Author SHA1 Message Date
renovate-bot b9a0bf1f83 chore(deps): update microsoftspeechversion to 1.52.0
PR and Push Build/Test / build-and-test (pull_request) Failing after 13m24s
PR and Push Build/Test / build-and-test (push) Successful in 11m59s
2026-09-29 02:42:03 +00:00
codex 7b2bcd3631 Preserve recovery transcripts and add user-approved agent requests
PR and Push Build/Test / build-and-test (push) Successful in 12m0s
Append successful offline recovery with a duplicate-warning marker and retain failed replay sources. Offer newly available microphones through confirmation notifications.

Add project discovery metadata and blocking association approval, plus nonblocking workflow requests that start a separate settings-agent conversation. Include regression tests, OpenSpec changes, and verified local Release startup.
2026-09-25 11:58:34 +02:00
codex e011e36ef7 Merge pull request 'chore(deps): update dependency microsoft.agents.ai.openai to 1.21.0' (#53) from renovate/microsoft.agents.ai.openai-1.x into main
PR and Push Build/Test / build-and-test (push) Successful in 12m29s
2026-09-18 09:59:28 +02:00
codex e17fb8b225 Merge pull request 'chore(deps): update microsoftspeechversion to 1.51.2' (#48) from renovate/microsoftspeechversion into main
PR and Push Build/Test / build-and-test (push) Canceled after 0s
2026-09-18 09:58:38 +02:00
codex c01213278e Merge pull request 'chore(deps): update dependency microsoft.entityframeworkcore.sqlite to 10.0.12' (#51) from renovate/microsoft.entityframeworkcore.sqlite-10.x into main
PR and Push Build/Test / build-and-test (push) Canceled after 0s
2026-09-18 09:57:58 +02:00
codex 7814f1c7c4 Merge pull request 'chore(deps): update dependency naudio to v3' (#45) from renovate/naudio-3.x into main
PR and Push Build/Test / build-and-test (push) Canceled after 0s
2026-09-18 09:57:23 +02:00
codex 88c3b21b2a Merge pull request 'chore(deps): update dependency naudio to 2.4.0' (#49) from renovate/naudio-2.x into main
PR and Push Build/Test / build-and-test (push) Failing after 20m17s
2026-09-18 09:56:40 +02:00
codex b249e5d977 Merge branch 'renovate/microsoftspeechversion' into renovate/microsoft.agents.ai.openai-1.x
PR and Push Build/Test / build-and-test (push) Successful in 12m41s
PR and Push Build/Test / build-and-test (pull_request) Successful in 12m42s
2026-09-17 13:36:23 +02:00
codex 3e4bf88931 Merge branch 'renovate/microsoft.entityframeworkcore.sqlite-10.x' into renovate/microsoftspeechversion
PR and Push Build/Test / build-and-test (push) Successful in 11m37s
PR and Push Build/Test / build-and-test (pull_request) Successful in 11m46s
2026-09-17 13:36:22 +02:00
codex 5e7c9f64d0 Merge remote-tracking branch 'origin/renovate/naudio-3.x' into renovate/microsoft.entityframeworkcore.sqlite-10.x
PR and Push Build/Test / build-and-test (push) Successful in 11m33s
PR and Push Build/Test / build-and-test (pull_request) Successful in 11m55s
# Conflicts:
#	MeetingAssistant/MeetingAssistant.csproj
2026-09-17 13:36:21 +02:00
codex fbd5a0852f Merge remote-tracking branch 'origin/renovate/naudio-2.x' into renovate/naudio-3.x
PR and Push Build/Test / build-and-test (push) Successful in 14m14s
PR and Push Build/Test / build-and-test (pull_request) Successful in 12m36s
# Conflicts:
#	MeetingAssistant/MeetingAssistant.csproj
2026-09-17 13:34:55 +02:00
codex 568bedb77a fix: isolate Windows sample playback for NAudio 3 2026-09-17 13:34:47 +02:00
codex f21c56b1c7 Merge remote-tracking branch 'origin/main' into renovate/naudio-3.x
# Conflicts:
#	MeetingAssistant/MeetingAssistant.csproj
2026-09-17 13:30:47 +02:00
codex c56de2b7ab Merge remote-tracking branch 'origin/renovate/microsoftspeechversion' into renovate/microsoftspeechversion
PR and Push Build/Test / build-and-test (push) Failing after 4s
PR and Push Build/Test / build-and-test (pull_request) Failing after 3s
2026-09-17 13:28:40 +02:00
codex 40d10d634a fix: reference WinMM explicitly for shared sample playback
PR and Push Build/Test / build-and-test (push) Successful in 11m36s
PR and Push Build/Test / build-and-test (pull_request) Successful in 12m9s
2026-09-17 13:27:49 +02:00
codex 264a3a823f Merge remote-tracking branch 'origin/main' into renovate/naudio-2.x
# Conflicts:
#	MeetingAssistant/MeetingAssistant.csproj
2026-09-17 13:26:04 +02:00
codex 072ebf38a3 Merge remote-tracking branch 'origin/main' into renovate/microsoft.entityframeworkcore.sqlite-10.x
PR and Push Build/Test / build-and-test (push) Failing after 1m47s
PR and Push Build/Test / build-and-test (pull_request) Failing after 1m11s
2026-09-17 13:24:45 +02:00
codex 4d66de943e fix: use Responses client options after Agent Framework update
PR and Push Build/Test / build-and-test (push) Failing after 1m48s
PR and Push Build/Test / build-and-test (pull_request) Failing after 1m49s
2026-09-17 13:23:57 +02:00
codex 150f11b6a7 Merge remote-tracking branch 'origin/main' into renovate/microsoft.agents.ai.openai-1.x 2026-09-17 13:21:45 +02:00
codex d7db64bec7 Merge pull request 'chore(deps): update dependency microsoft.net.test.sdk to 18.10.1' (#56) from renovate/microsoft.net.test.sdk-18.x into main
PR and Push Build/Test / build-and-test (push) Successful in 12m7s
2026-09-17 10:50:52 +02:00
codex cdeef1967e Merge pull request 'chore(deps): update dependency ncalcsync to 7.2.0' (#55) from renovate/ncalcsync-7.x into main
PR and Push Build/Test / build-and-test (push) Successful in 11m54s
2026-09-17 10:49:02 +02:00
codex aeb0cfe42f Merge pull request 'chore(deps): update dependency system.drawing.common to 10.0.12' (#52) from renovate/system.drawing.common-10.x into main
PR and Push Build/Test / build-and-test (push) Successful in 11m43s
2026-09-17 10:46:41 +02:00
renovate-bot ef0c0faba8 chore(deps): update dependency microsoft.net.test.sdk to 18.10.1
PR and Push Build/Test / build-and-test (push) Successful in 11m52s
PR and Push Build/Test / build-and-test (pull_request) Successful in 11m25s
2026-09-17 02:45:15 +00:00
codex 93eb3d4e98 fix: prevent profile-switch transcription deadlock
PR and Push Build/Test / build-and-test (push) Successful in 15m17s
2026-09-16 15:00:49 +02:00
Manuel d93d758a8f Merge branch 'main' into renovate/naudio-3.x
PR and Push Build/Test / build-and-test (push) Failing after 6m58s
PR and Push Build/Test / build-and-test (pull_request) Failing after 7m56s
2026-09-15 09:51:19 +02:00
Manuel 48ad4fb930 Merge branch 'main' into renovate/microsoftspeechversion
PR and Push Build/Test / build-and-test (push) Successful in 11m28s
PR and Push Build/Test / build-and-test (pull_request) Successful in 11m26s
2026-09-15 09:51:06 +02:00
Manuel cb5a300b44 Merge branch 'main' into renovate/microsoft.entityframeworkcore.sqlite-10.x
PR and Push Build/Test / build-and-test (push) Successful in 11m46s
PR and Push Build/Test / build-and-test (pull_request) Failing after 11m46s
2026-09-15 09:50:42 +02:00
Manuel b6acba8402 Merge branch 'main' into renovate/system.drawing.common-10.x
PR and Push Build/Test / build-and-test (push) Successful in 12m3s
PR and Push Build/Test / build-and-test (pull_request) Successful in 11m36s
2026-09-15 09:50:31 +02:00
Manuel a82e2fff03 Merge branch 'main' into renovate/microsoft.agents.ai.openai-1.x
PR and Push Build/Test / build-and-test (push) Failing after 7m36s
PR and Push Build/Test / build-and-test (pull_request) Failing after 7m2s
2026-09-15 09:50:21 +02:00
Manuel 336d940acd Merge branch 'main' into renovate/naudio-2.x
PR and Push Build/Test / build-and-test (push) Failing after 7m28s
PR and Push Build/Test / build-and-test (pull_request) Failing after 7m17s
2026-09-15 09:50:03 +02:00
codex cbf854fb2a Merge pull request 'chore(deps): update dependency microsoft.net.test.sdk to 18.10.0' (#54) from renovate/microsoft.net.test.sdk-18.x into main
PR and Push Build/Test / build-and-test (push) Failing after 12m12s
2026-09-14 08:55:43 +02:00
renovate-bot bacef060fe chore(deps): update dependency ncalcsync to 7.2.0
PR and Push Build/Test / build-and-test (pull_request) Successful in 10m47s
PR and Push Build/Test / build-and-test (push) Successful in 14m26s
2026-09-14 02:41:00 +00:00
renovate-bot a928382e51 chore(deps): update dependency microsoft.net.test.sdk to 18.10.0
PR and Push Build/Test / build-and-test (push) Successful in 13m30s
PR and Push Build/Test / build-and-test (pull_request) Successful in 15m33s
2026-09-13 02:40:52 +00:00
renovate-bot e23d04a38c chore(deps): update dependency naudio to v3
PR and Push Build/Test / build-and-test (push) Failing after 1m48s
PR and Push Build/Test / build-and-test (pull_request) Failing after 1m48s
2026-09-12 02:46:02 +00:00
renovate-bot c6a50d307e chore(deps): update dependency naudio to 2.4.0
PR and Push Build/Test / build-and-test (push) Failing after 1m48s
PR and Push Build/Test / build-and-test (pull_request) Failing after 1m48s
2026-09-12 02:45:52 +00:00
renovate-bot 436fa6c149 chore(deps): update dependency microsoft.agents.ai.openai to 1.21.0
PR and Push Build/Test / build-and-test (push) Failing after 1m48s
PR and Push Build/Test / build-and-test (pull_request) Failing after 1m48s
2026-09-12 02:45:39 +00:00
renovate-bot 5dce5fd350 chore(deps): update microsoftspeechversion to 1.51.2
PR and Push Build/Test / build-and-test (push) Failing after 1m48s
PR and Push Build/Test / build-and-test (pull_request) Failing after 1m48s
2026-09-12 02:45:33 +00:00
renovate-bot c37654a7d9 chore(deps): update dependency system.drawing.common to 10.0.12
PR and Push Build/Test / build-and-test (pull_request) Failing after 1m48s
PR and Push Build/Test / build-and-test (push) Failing after 1m47s
2026-09-12 02:45:27 +00:00
renovate-bot cc9044e091 chore(deps): update dependency microsoft.entityframeworkcore.sqlite to 10.0.12
PR and Push Build/Test / build-and-test (push) Failing after 3m58s
PR and Push Build/Test / build-and-test (pull_request) Failing after 1m48s
2026-09-12 02:45:18 +00:00
codex f86af983e8 feat: add local Resemblyzer speaker recognition
PR and Push Build/Test / build-and-test (push) Successful in 12m37s
2026-09-11 13:49:27 +02:00
codex 43fc8aaec0 Merge pull request 'chore(deps): update dependency xunit.runner.visualstudio to v4' (#46) from renovate/xunit.runner.visualstudio-4.x into main
PR and Push Build/Test / build-and-test (push) Failing after 1m49s
2026-09-10 10:36:56 +02:00
codex 8b543a2161 Merge pull request 'chore(deps): update dependency ncalcsync to 7.1.0' (#40) from renovate/ncalcsync-7.x into main
PR and Push Build/Test / build-and-test (push) Failing after 5m23s
2026-09-10 10:36:48 +02:00
codex 58896073c0 Merge pull request 'chore(deps): update dependency microsoft.aspnetcore.mvc.testing to 10.0.12' (#50) from renovate/microsoft.aspnetcore.mvc.testing-10.x into main
PR and Push Build/Test / build-and-test (push) Successful in 10m18s
2026-09-09 13:15:13 +02:00
renovate-bot 945543c078 chore(deps): update dependency microsoft.aspnetcore.mvc.testing to 10.0.12
PR and Push Build/Test / build-and-test (push) Successful in 9m58s
PR and Push Build/Test / build-and-test (pull_request) Successful in 10m8s
2026-09-09 02:40:27 +00:00
renovate-bot 3e1f447b20 chore(deps): update dependency xunit.runner.visualstudio to v4
PR and Push Build/Test / build-and-test (push) Successful in 9m42s
PR and Push Build/Test / build-and-test (pull_request) Successful in 9m54s
2026-09-04 02:42:29 +00:00
renovate-bot 497390c2ec chore(deps): update dependency ncalcsync to 7.1.0
PR and Push Build/Test / build-and-test (push) Successful in 11m57s
PR and Push Build/Test / build-and-test (pull_request) Successful in 10m33s
2026-09-04 02:42:18 +00:00
codex b1365b202d fix: unify pyannote validation toggle
PR and Push Build/Test / build-and-test (push) Successful in 12m8s
2026-09-02 13:46:51 +02:00
codex 70ffaa7357 feat: add transcription pause controls
PR and Push Build/Test / build-and-test (push) Successful in 13m19s
2026-09-02 12:22:11 +02:00
codex 714da456df Merge pull request 'chore(deps): update dependency microsoft.agents.ai.openai to 1.20.0' (#47) from renovate/microsoft.agents.ai.openai-1.x into main
PR and Push Build/Test / build-and-test (push) Successful in 12m55s
2026-09-01 09:32:08 +02:00
renovate-bot fa67f30c49 chore(deps): update dependency microsoft.agents.ai.openai to 1.20.0
PR and Push Build/Test / build-and-test (push) Successful in 17m6s
PR and Push Build/Test / build-and-test (pull_request) Successful in 17m10s
2026-09-01 02:50:34 +00:00
codex dc37719d46 Merge pull request 'chore(deps): update dependency microsoft.aspnetcore.mvc.testing to 10.0.11' (#41) from renovate/microsoft.aspnetcore.mvc.testing-10.x into main
PR and Push Build/Test / build-and-test (push) Successful in 10m35s
2026-08-18 13:13:40 +02:00
renovate-bot ee453e170e chore(deps): update dependency microsoft.aspnetcore.mvc.testing to 10.0.11
PR and Push Build/Test / build-and-test (push) Successful in 15m3s
PR and Push Build/Test / build-and-test (pull_request) Successful in 15m31s
2026-08-18 02:40:54 +00:00
codex d51463321b Merge pull request 'chore(deps): update dependency microsoft.net.test.sdk to 18.9.0' (#44) from renovate/microsoft.net.test.sdk-18.x into main
PR and Push Build/Test / build-and-test (push) Successful in 14m38s
2026-08-17 09:05:30 +02:00
renovate-bot 41ae47fbfe chore(deps): update dependency microsoft.net.test.sdk to 18.9.0
PR and Push Build/Test / build-and-test (push) Successful in 16m29s
PR and Push Build/Test / build-and-test (pull_request) Successful in 16m20s
2026-08-16 02:49:14 +00:00
codex f7ad2ce41a Merge pull request 'chore(deps): update dependency system.drawing.common to 10.0.11' (#43) from renovate/system.drawing.common-10.x into main
PR and Push Build/Test / build-and-test (push) Successful in 11m9s
2026-08-13 10:53:04 +02:00
renovate-bot bc76151926 chore(deps): update dependency system.drawing.common to 10.0.11
PR and Push Build/Test / build-and-test (pull_request) Successful in 15m43s
PR and Push Build/Test / build-and-test (push) Successful in 15m52s
2026-08-13 02:41:36 +00:00
codex af9f5b34dc Merge pull request 'chore(deps): update dependency microsoft.entityframeworkcore.sqlite to 10.0.11' (#42) from renovate/microsoft.entityframeworkcore.sqlite-10.x into main
PR and Push Build/Test / build-and-test (push) Successful in 11m25s
2026-08-12 13:19:26 +02:00
renovate-bot f3489f8e95 chore(deps): update dependency microsoft.entityframeworkcore.sqlite to 10.0.11
PR and Push Build/Test / build-and-test (push) Successful in 10m37s
PR and Push Build/Test / build-and-test (pull_request) Successful in 10m0s
2026-08-12 02:49:08 +00:00
codex 39591eef12 docs: refresh Meeting Assistant runtime boundaries
PR and Push Build/Test / build-and-test (push) Failing after 10m29s
2026-08-11 09:15:16 +02:00
codex b3aeef24eb Merge PR #34: update setup-dotnet to v6
PR and Push Build/Test / build-and-test (push) Failing after 16s
Attended CI control-plane merge after exact-head success and local combined validation; final main run will be monitored.
2026-08-05 13:03:50 +02:00
codex c91d956beb Merge PR #38: update NCalcSync to 7.0.2
PR and Push Build/Test / build-and-test (push) Failing after 16s
Attended major-version merge after API migration review and combined current-main workflow-expression test validation.
2026-08-05 13:03:39 +02:00
codex 4c858170d9 Merge PR #36: update Microsoft Agent Framework OpenAI
PR and Push Build/Test / build-and-test (push) Failing after 16s
Attended merge after upstream compatibility review and combined current-main validation.
2026-08-05 13:03:30 +02:00
codex 0b6643d87f Merge PR #37: update Microsoft Speech to 1.51.1
PR and Push Build/Test / build-and-test (push) Failing after 4m10s
Attended merge after fixing MAS model asset discovery and validating both targets plus 440 tests.
2026-08-05 13:03:18 +02:00
codex 44ce419c49 fix: copy available MAS model assets
PR and Push Build/Test / build-and-test (push) Successful in 11m38s
PR and Push Build/Test / build-and-test (pull_request) Failing after 11m41s
2026-08-05 13:01:50 +02:00
codex a087ff0cbf Merge remote-tracking branch 'origin/main' into renovate/microsoftspeechversion 2026-08-05 13:00:56 +02:00
codex 8dc1b70f8a test: stabilize summary retry background wait
PR and Push Build/Test / build-and-test (push) Successful in 11m32s
2026-08-05 12:35:00 +02:00
codex aaee123b9a merge: attach calendar metadata to active meetings
PR and Push Build/Test / build-and-test (push) Failing after 18m6s
2026-08-05 11:59:35 +02:00
codex 1a341f1eaa feat: attach calendar metadata to active meetings
PR and Push Build/Test / build-and-test (push) Successful in 15m2s
2026-08-05 11:58:25 +02:00
renovate-bot e886fac311 Update dependency NCalcSync to v7
PR and Push Build/Test / build-and-test (push) Failing after 16s
PR and Push Build/Test / build-and-test (pull_request) Failing after 16s
2026-08-05 02:40:11 +00:00
renovate-bot e35ccf661f Update dependency Microsoft.Agents.AI.OpenAI to 1.17.0
PR and Push Build/Test / build-and-test (push) Failing after 3m0s
PR and Push Build/Test / build-and-test (pull_request) Failing after 15s
2026-08-05 02:39:47 +00:00
codex aa42e8edda refactor: tighten tray menu section handling
PR and Push Build/Test / build-and-test (push) Successful in 10m51s
2026-08-04 10:11:43 +02:00
codex 2f12a96688 feat: prioritize finish meeting tray action
PR and Push Build/Test / build-and-test (push) Successful in 11m8s
2026-08-04 09:55:18 +02:00
renovate-bot 20eed38477 Update actions/setup-dotnet action to v6
PR and Push Build/Test / build-and-test (push) Successful in 10m25s
PR and Push Build/Test / build-and-test (pull_request) Successful in 10m46s
2026-08-04 02:39:34 +00:00
renovate-bot ada4a40f62 Update MicrosoftSpeechVersion to 1.51.1
PR and Push Build/Test / build-and-test (push) Successful in 10m0s
PR and Push Build/Test / build-and-test (pull_request) Successful in 11m11s
2026-08-04 02:39:30 +00:00
codex 5d0ae84426 fix: support streaming screenshot OCR 2026-08-03 16:14:01 +02:00
codex b9547ae4c4 fix: recover from microphone disconnects
PR and Push Build/Test / build-and-test (push) Successful in 12m29s
2026-08-03 15:57:30 +02:00
codex 75250f6041 Merge pull request 'Update dependency SQLitePCLRaw.bundle_e_sqlite3 to 3.0.5' (#35) from renovate/sqlitepclraw.bundle_e_sqlite3-3.x into main
PR and Push Build/Test / build-and-test (push) Successful in 10m19s
2026-07-30 15:42:00 +02:00
renovate-bot cd2446f620 Update dependency SQLitePCLRaw.bundle_e_sqlite3 to 3.0.5
PR and Push Build/Test / build-and-test (pull_request) Successful in 10m58s
PR and Push Build/Test / build-and-test (push) Successful in 13m32s
2026-07-28 02:38:03 +00:00
codex e192ae7cd8 fix: support LiteLLM Responses streaming
PR and Push Build/Test / build-and-test (push) Successful in 14m17s
2026-07-27 14:44:32 +02:00
codex 3cadf08fa2 feat: use assistant context as meeting memory
PR and Push Build/Test / build-and-test (push) Successful in 9m34s
2026-07-17 11:26:49 +02:00
Manuel 407db80413 Merge pull request 'Update dependency Microsoft.NET.Test.Sdk to 18.8.1' (#33) from renovate/microsoft.net.test.sdk-18.x into main
PR and Push Build/Test / build-and-test (push) Successful in 13m56s
2026-07-16 18:50:55 +02:00
renovate-bot f0aac40dfb Update dependency Microsoft.NET.Test.Sdk to 18.8.1
PR and Push Build/Test / build-and-test (push) Successful in 9m38s
PR and Push Build/Test / build-and-test (pull_request) Successful in 9m28s
2026-07-16 02:36:52 +00:00
codex 0e0feedad2 Fix YAML escaping for meeting note frontmatter
PR and Push Build/Test / build-and-test (push) Successful in 10m21s
2026-07-15 14:55:37 +02:00
codex 89d81fa4c6 fix: encode spaces in summary image links
PR and Push Build/Test / build-and-test (push) Successful in 11m2s
2026-07-15 14:55:25 +02:00
Manuel dc49bc3330 Merge pull request 'Update dependency System.Drawing.Common to 10.0.10' (#32) from renovate/system.drawing.common-10.x into main
PR and Push Build/Test / build-and-test (push) Successful in 9m38s
2026-07-15 13:29:06 +02:00
Manuel aff3528406 Merge pull request 'Update dependency Microsoft.EntityFrameworkCore.Sqlite to 10.0.10' (#31) from renovate/microsoft.entityframeworkcore.sqlite-10.x into main
PR and Push Build/Test / build-and-test (push) Failing after 9m50s
2026-07-15 13:27:51 +02:00
Manuel d92da18d08 Merge pull request 'Update dependency Microsoft.AspNetCore.Mvc.Testing to 10.0.10' (#30) from renovate/microsoft.aspnetcore.mvc.testing-10.x into main
PR and Push Build/Test / build-and-test (push) Successful in 9m41s
2026-07-15 13:27:20 +02:00
renovate-bot ec59464340 Update dependency System.Drawing.Common to 10.0.10
PR and Push Build/Test / build-and-test (pull_request) Successful in 9m19s
PR and Push Build/Test / build-and-test (push) Successful in 9m33s
2026-07-15 09:58:37 +00:00
renovate-bot 5667102272 Update dependency Microsoft.EntityFrameworkCore.Sqlite to 10.0.10
PR and Push Build/Test / build-and-test (pull_request) Successful in 9m33s
PR and Push Build/Test / build-and-test (push) Successful in 9m9s
2026-07-15 02:35:11 +00:00
renovate-bot 2f62ed467a Update dependency Microsoft.AspNetCore.Mvc.Testing to 10.0.10
PR and Push Build/Test / build-and-test (push) Successful in 9m8s
PR and Push Build/Test / build-and-test (pull_request) Successful in 9m15s
2026-07-15 02:35:02 +00:00
Manuel 0439a819b2 Merge pull request 'Update actions/checkout action to v7' (#22) from renovate/actions-checkout-7.x into main
PR and Push Build/Test / build-and-test (push) Successful in 11m32s
Reviewed-on: #22
2026-07-05 01:43:27 +02:00
Manuel 2cfe4e4ef0 Merge pull request 'Update dependency Microsoft.Agents.AI.OpenAI to 1.13.0' (#28) from renovate/microsoft.agents.ai.openai-1.x into main
PR and Push Build/Test / build-and-test (push) Successful in 11m11s
Reviewed-on: #28
2026-07-05 01:39:59 +02:00
renovate-bot b5ccf2125c Update dependency Microsoft.Agents.AI.OpenAI to 1.13.0
PR and Push Build/Test / build-and-test (push) Successful in 10m18s
PR and Push Build/Test / build-and-test (pull_request) Successful in 10m8s
2026-07-04 02:33:36 +00:00
codex 0a7fc240d9 Merge pull request 'Update dependency NCalcSync to 6.4.0' (#29) from renovate/ncalcsync-6.x into main
PR and Push Build/Test / build-and-test (push) Successful in 9m38s
2026-07-03 13:26:22 +02:00
renovate-bot d81c2731da Update actions/checkout action to v7
PR and Push Build/Test / build-and-test (push) Failing after 9m39s
PR and Push Build/Test / build-and-test (pull_request) Successful in 9m50s
2026-07-03 02:33:23 +00:00
renovate-bot aa8ac26bbb Update dependency NCalcSync to 6.4.0
PR and Push Build/Test / build-and-test (pull_request) Successful in 9m46s
PR and Push Build/Test / build-and-test (push) Successful in 9m42s
2026-07-03 02:33:17 +00:00
codex 6ee2d0ea08 Merge pull request 'Update dependency NCalcSync to 6.3.3' (#27) from renovate/ncalcsync-6.x into main
PR and Push Build/Test / build-and-test (push) Successful in 14m17s
2026-07-02 10:10:32 +02:00
codex e8742f544d Merge pull request 'Update dependency YamlDotNet to 18.1.0' (#26) from renovate/yamldotnet-18.x into main
PR and Push Build/Test / build-and-test (push) Successful in 10m12s
2026-07-02 10:10:07 +02:00
renovate-bot 18bed0532a Update dependency NCalcSync to 6.3.3
PR and Push Build/Test / build-and-test (push) Successful in 9m54s
PR and Push Build/Test / build-and-test (pull_request) Successful in 10m1s
2026-07-02 02:32:16 +00:00
codex 8e2f266e66 Close stale transcript workflow follow-up
PR and Push Build/Test / build-and-test (push) Successful in 9m35s
2026-07-01 11:44:07 +02:00
codex 5c67738939 Add attendee transformation workflows 2026-07-01 11:10:36 +02:00
codex 92e359646b Update meeting summary agent UI 2026-07-01 10:30:28 +02:00
renovate-bot c6b2add4dd Update dependency YamlDotNet to 18.1.0
PR and Push Build/Test / build-and-test (push) Successful in 9m19s
PR and Push Build/Test / build-and-test (pull_request) Successful in 9m21s
2026-06-27 02:30:48 +00:00
codex 4787bf8cec Upgrade NuGet dependencies
PR and Push Build/Test / build-and-test (push) Successful in 9m11s
2026-06-26 13:31:53 +02:00
codex aecef30627 Archive completed meeting assistant changes
PR and Push Build/Test / build-and-test (push) Successful in 9m19s
2026-06-26 13:15:47 +02:00
codex 7d16b0cad7 Merge pull request 'Update dependency Microsoft.Agents.AI.OpenAI to 1.11.1' (#25) from renovate/microsoft.agents.ai.openai-1.x into main
PR and Push Build/Test / build-and-test (push) Successful in 9m57s
2026-06-26 11:17:28 +02:00
codex 91ce5f5aa0 Merge pull request 'Update dependency Microsoft.AspNetCore.Mvc.Testing to 10.0.9' (#19) from renovate/microsoft.aspnetcore.mvc.testing-10.x into main
PR and Push Build/Test / build-and-test (push) Failing after 9m25s
2026-06-26 11:17:21 +02:00
renovate-bot 19d7c28dcc Update dependency Microsoft.AspNetCore.Mvc.Testing to 10.0.9
PR and Push Build/Test / build-and-test (push) Successful in 9m5s
PR and Push Build/Test / build-and-test (pull_request) Successful in 9m13s
2026-06-26 02:31:45 +00:00
renovate-bot 9729e644af Update dependency Microsoft.Agents.AI.OpenAI to 1.11.1
PR and Push Build/Test / build-and-test (push) Successful in 9m4s
PR and Push Build/Test / build-and-test (pull_request) Successful in 9m15s
2026-06-26 02:31:35 +00:00
232 changed files with 16846 additions and 1864 deletions
+2 -2
View File
@@ -15,10 +15,10 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v7
- name: Setup .NET
uses: actions/setup-dotnet@v5
uses: actions/setup-dotnet@v6
with:
dotnet-version: "10.0.x"
@@ -102,6 +102,23 @@ public sealed class AudioMixingTests
Assert.Equal(2_000, BitConverter.ToInt16(chunks[0].Pcm));
}
[Fact]
public async Task CompositeAudioSourceKeepsSystemAudioWhileMicrophoneIsRecovering()
{
var microphone = new WaitingAudioSource();
var system = new FixedAudioSource(Pcm16(10_000));
var source = CreateSource(microphone, system);
using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5));
await using var chunks = source
.CaptureAsync(new MeetingAssistantOptions(), cancellation.Token)
.GetAsyncEnumerator(cancellation.Token);
Assert.True(await chunks.MoveNextAsync());
Assert.Equal(10_000, BitConverter.ToInt16(chunks.Current.Pcm));
await cancellation.CancelAsync();
}
[Fact]
public void AdaptiveEchoCancellerReducesEchoFromMicrophoneSignal()
{
@@ -221,6 +238,16 @@ public sealed class AudioMixingTests
}
}
private sealed class WaitingAudioSource : IMeetingAudioSource
{
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
yield break;
}
}
private sealed class OptionsCapturingAudioSource : IMeetingAudioSource
{
private readonly AudioChunk chunk;
@@ -1,4 +1,5 @@
using MeetingAssistant.Calendar;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Recording;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
@@ -22,6 +23,94 @@ public sealed class CalendarRecordingPromptSchedulerTests
Assert.Equal(0, harness.Recorder.StopCount);
}
[Fact]
public async Task AcceptedPromptStartsRecordingWithPromptedMeetingMetadata()
{
var harness = CreateHarness(isRecording: false);
var selectedMetadata = new MeetingMetadata(
"Selected planning",
["Ada"],
"Selected agenda",
DateTimeOffset.Parse("2026-06-03T10:30:00+00:00"));
var otherMetadata = new MeetingMetadata(
"Other planning",
["Grace"],
"Other agenda",
DateTimeOffset.Parse("2026-06-03T10:40:00+00:00"));
var selectedMeeting = CreateMeeting(
harness.Clock,
id: "teams-selected",
subject: "Selected planning",
metadata: selectedMetadata);
var otherMeeting = CreateMeeting(
harness.Clock,
id: "teams-other",
subject: "Other planning",
startOffset: TimeSpan.FromMinutes(40),
metadata: otherMetadata);
harness.Provider.Meetings = [selectedMeeting, otherMeeting];
await SyncThenPromptAsync(harness, selectedMeeting);
Assert.Equal(1, harness.Recorder.StartCount);
Assert.Same(selectedMetadata, harness.Recorder.StartMetadata.Single());
Assert.Equal(1, harness.Recorder.PromptStartCount);
}
[Fact]
public async Task AcceptedPromptWithoutMetadataStillUsesPromptStartPath()
{
var harness = CreateHarness(isRecording: false);
var meeting = CreateMeeting(
harness.Clock,
id: "teams-without-metadata",
subject: "No metadata",
metadata: null);
harness.Provider.Meetings = [meeting];
await SyncThenPromptAsync(harness, meeting);
Assert.Equal(1, harness.Recorder.StartCount);
Assert.Null(harness.Recorder.StartMetadata.Single());
Assert.Equal(1, harness.Recorder.PromptStartCount);
}
[Fact]
public async Task AcceptingDisplayedPromptsOutOfOrderUsesAcceptedPromptMetadata()
{
var harness = CreateHarness(isRecording: false, autoAcceptPrompts: false);
var firstMetadata = new MeetingMetadata(
"First planning",
["Ada"],
"First agenda",
DateTimeOffset.Parse("2026-06-03T10:30:00+00:00"));
var secondMetadata = new MeetingMetadata(
"Second planning",
["Grace"],
"Second agenda",
DateTimeOffset.Parse("2026-06-03T10:30:00+00:00"));
var firstMeeting = CreateMeeting(
harness.Clock,
id: "teams-first",
subject: "First planning",
metadata: firstMetadata);
var secondMeeting = CreateMeeting(
harness.Clock,
id: "teams-second",
subject: "Second planning",
metadata: secondMetadata);
harness.Provider.Meetings = [firstMeeting, secondMeeting];
await harness.Scheduler.SyncOnceAsync(CancellationToken.None);
harness.Clock.Now = firstMeeting.Start;
await harness.Scheduler.CheckDuePromptsAsync(CancellationToken.None);
await harness.PromptService.RespondAsync(secondMeeting, MeetingStartPromptResponse.Record);
Assert.Equal([firstMeeting, secondMeeting], harness.PromptService.PromptedMeetings);
Assert.Equal(1, harness.Recorder.StartCount);
Assert.Same(secondMetadata, harness.Recorder.StartMetadata.Single());
}
[Fact]
public async Task AcceptedPromptStopsActiveRecordingBeforeStartingNewRecording()
{
@@ -34,6 +123,68 @@ public sealed class CalendarRecordingPromptSchedulerTests
Assert.Equal(["stop", "start"], harness.Recorder.Commands);
}
[Fact]
public async Task ActiveRecordingPromptAttachesTheSelectedAppointmentMetadataWithoutRestarting()
{
var harness = CreateHarness(isRecording: true, autoAcceptPrompts: false);
var firstMetadata = new MeetingMetadata(
"First planning",
["Ada"],
"First agenda",
DateTimeOffset.Parse("2026-06-03T10:30:00+00:00"));
var secondMetadata = new MeetingMetadata(
"Second planning",
["Grace"],
"Second agenda",
DateTimeOffset.Parse("2026-06-03T10:30:00+00:00"));
var firstMeeting = CreateMeeting(
harness.Clock,
id: "teams-first",
subject: "First planning",
metadata: firstMetadata);
var secondMeeting = CreateMeeting(
harness.Clock,
id: "teams-second",
subject: "Second planning",
metadata: secondMetadata);
harness.Provider.Meetings = [firstMeeting, secondMeeting];
await harness.Scheduler.SyncOnceAsync(CancellationToken.None);
harness.Clock.Now = firstMeeting.Start;
await harness.Scheduler.CheckDuePromptsAsync(CancellationToken.None);
await harness.PromptService.RespondAsync(
secondMeeting,
MeetingStartPromptResponse.AttachMetadataToCurrentMeeting);
Assert.All(
harness.PromptService.PromptRequests,
request => Assert.True(request.CanAttachToCurrentMeeting));
Assert.Equal(["attach-metadata"], harness.Recorder.Commands);
Assert.Same(secondMetadata, harness.Recorder.AttachedMetadata.Single());
Assert.Equal(0, harness.Recorder.StopCount);
Assert.Equal(0, harness.Recorder.StartCount);
Assert.True(harness.Recorder.CurrentStatus.IsRecording);
}
[Fact]
public async Task CanceledCachedMeetingDoesNotPromptRecording()
{
var harness = CreateHarness(isRecording: false);
var canceledMeeting = CreateMeeting(
harness.Clock,
id: "teams-canceled",
subject: "Canceled project sync",
isCanceled: true);
harness.Provider.Meetings = [canceledMeeting];
await harness.Scheduler.SyncOnceAsync(CancellationToken.None);
harness.Clock.Now = canceledMeeting.Start;
await harness.Scheduler.CheckDuePromptsAsync(CancellationToken.None);
Assert.Empty(harness.PromptService.PromptedMeetings);
Assert.Equal(0, harness.Recorder.StartCount);
}
[Fact]
public async Task DisabledCalendarPromptsDoNotQueryCalendar()
{
@@ -115,23 +266,28 @@ public sealed class CalendarRecordingPromptSchedulerTests
string id = "teams-1",
string subject = "Project sync",
TimeSpan? startOffset = null,
TimeSpan? duration = null)
TimeSpan? duration = null,
MeetingMetadata? metadata = null,
bool isCanceled = false)
{
var start = clock.Now.Add(startOffset ?? TimeSpan.FromMinutes(30));
return new CalendarMeeting(
id,
subject,
start,
start.Add(duration ?? TimeSpan.FromMinutes(30)));
start.Add(duration ?? TimeSpan.FromMinutes(30)),
metadata,
isCanceled);
}
private static SchedulerHarness CreateHarness(
bool isRecording = false,
bool enabled = true)
bool enabled = true,
bool autoAcceptPrompts = true)
{
var clock = new ManualCalendarPromptClock(new DateTimeOffset(2026, 6, 3, 9, 30, 0, TimeSpan.Zero));
var provider = new RecordingCalendarMeetingProvider([]);
var promptService = new AcceptingMeetingStartPromptService();
var promptService = new CapturingMeetingStartPromptService(autoAcceptPrompts);
var recorder = new RecordingPromptRecorder(isRecording);
var scheduler = new CalendarRecordingPromptScheduler(
Options.Create(new MeetingAssistantOptions
@@ -155,7 +311,7 @@ public sealed class CalendarRecordingPromptSchedulerTests
private sealed record SchedulerHarness(
CalendarRecordingPromptScheduler Scheduler,
RecordingCalendarMeetingProvider Provider,
AcceptingMeetingStartPromptService PromptService,
CapturingMeetingStartPromptService PromptService,
RecordingPromptRecorder Recorder,
ManualCalendarPromptClock Clock);
@@ -179,18 +335,45 @@ public sealed class CalendarRecordingPromptSchedulerTests
}
}
private sealed class AcceptingMeetingStartPromptService : IMeetingStartPromptService
private sealed class CapturingMeetingStartPromptService : IMeetingStartPromptService
{
public List<CalendarMeeting> PromptedMeetings { get; } = [];
private readonly bool autoAccept;
private readonly List<PendingPrompt> pendingPrompts = [];
public CapturingMeetingStartPromptService(bool autoAccept)
{
this.autoAccept = autoAccept;
}
public List<MeetingStartPromptRequest> PromptRequests { get; } = [];
public IReadOnlyList<CalendarMeeting> PromptedMeetings =>
PromptRequests.Select(request => request.Meeting).ToList();
public async Task ShowPromptAsync(
MeetingStartPromptRequest request,
Func<MeetingStartPromptResponse, CancellationToken, Task> handleResponseAsync,
CancellationToken cancellationToken)
{
PromptedMeetings.Add(request.Meeting);
await handleResponseAsync(MeetingStartPromptResponse.Record, cancellationToken);
PromptRequests.Add(request);
pendingPrompts.Add(new PendingPrompt(request.Meeting, handleResponseAsync));
if (autoAccept)
{
await handleResponseAsync(MeetingStartPromptResponse.Record, cancellationToken);
}
}
public Task RespondAsync(
CalendarMeeting meeting,
MeetingStartPromptResponse response)
{
var prompt = pendingPrompts.Single(pending => ReferenceEquals(pending.Meeting, meeting));
return prompt.HandleResponseAsync(response, CancellationToken.None);
}
private sealed record PendingPrompt(
CalendarMeeting Meeting,
Func<MeetingStartPromptResponse, CancellationToken, Task> HandleResponseAsync);
}
private sealed class RecordingPromptRecorder : IMeetingPromptRecordingController
@@ -206,16 +389,37 @@ public sealed class CalendarRecordingPromptSchedulerTests
public int StopCount { get; private set; }
public int PromptStartCount { get; private set; }
public List<string> Commands { get; } = [];
public List<MeetingMetadata?> StartMetadata { get; } = [];
public List<MeetingMetadata> AttachedMetadata { get; } = [];
public Task<RecordingStatus> StartAsync(CancellationToken cancellationToken)
{
return StartRecordingAsync(null);
}
private Task<RecordingStatus> StartRecordingAsync(
MeetingMetadata? metadata)
{
StartCount++;
Commands.Add("start");
StartMetadata.Add(metadata);
CurrentStatus = Status(isRecording: true);
return Task.FromResult(CurrentStatus);
}
public Task<RecordingStatus> StartFromPromptAsync(
MeetingMetadata? metadata,
CancellationToken cancellationToken)
{
PromptStartCount++;
return StartRecordingAsync(metadata);
}
public Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
{
StopCount++;
@@ -224,6 +428,15 @@ public sealed class CalendarRecordingPromptSchedulerTests
return Task.FromResult(CurrentStatus);
}
public Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
MeetingMetadata metadata,
CancellationToken cancellationToken)
{
Commands.Add("attach-metadata");
AttachedMetadata.Add(metadata);
return Task.FromResult(CurrentStatus);
}
private static RecordingStatus Status(bool isRecording)
{
return new RecordingStatus(
@@ -1,6 +1,7 @@
using System.Net;
using System.Net.Http.Json;
using MeetingAssistant.Recording;
using MeetingAssistant.Workflow;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
@@ -41,6 +42,17 @@ public sealed class HealthEndpointTests : IClassFixture<WebApplicationFactory<Pr
Assert.Equal("ok", body?.Status);
}
[Fact]
public async Task NeutralBuildReportsSpeakerPlaybackUnavailable()
{
var playback = factory.Services.GetRequiredService<IWorkflowRulesEditorSamplePlaybackQueue>();
var result = await playback.QueueAsync(1, [1, 2]);
Assert.Contains("Windows build", result);
Assert.DoesNotContain("Queued", result);
}
[Fact]
public async Task ApplicationStartupDeletesStaleTemporaryRecordings()
{
@@ -58,6 +58,64 @@ public sealed class LaunchProfileOptionsProviderTests
Assert.Equal("Ctrl+Alt+L", profile.Options.Hotkey.Toggle);
}
[Fact]
public void CheckedInConfigurationKeepsResemblyzerOptInWithDocumentedDefaults()
{
var profile = CreateProviderFromAppsettings().GetRequiredProfile(null);
Assert.False(profile.Options.SpeakerIdentification.Resemblyzer.Enabled);
Assert.Equal(5, profile.Options.SpeakerIdentification.Resemblyzer.RequiredVectorsPerSpeaker);
Assert.Equal(1000, profile.Options.SpeakerIdentification.Resemblyzer.MaxVectorsPerIdentity);
Assert.Equal(20, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumVectors);
Assert.Equal(0.75, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningNeighborSimilarity);
Assert.Equal(3, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumNeighbors);
Assert.Equal(0.60, profile.Options.SpeakerIdentification.Resemblyzer.OutlierPruningMinimumClusterRatio);
}
[Fact]
public void InvalidResemblyzerOutlierPruningSettingsAreRejected()
{
var provider = CreateProvider(new Dictionary<string, string?>
{
["MeetingAssistant:SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio"] = "1.1"
});
var exception = Assert.Throws<InvalidOperationException>(
() => provider.GetRequiredProfile(null));
Assert.Contains("OutlierPruningMinimumClusterRatio", exception.Message);
}
[Fact]
public void NonFiniteResemblyzerSimilaritySettingsAreRejected()
{
var provider = CreateProvider(new Dictionary<string, string?>
{
["MeetingAssistant:SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity"] = "NaN"
});
var exception = Assert.Throws<InvalidOperationException>(
() => provider.GetRequiredProfile(null));
Assert.Contains("MinimumIdentitySimilarity", exception.Message);
}
[Fact]
public void ContradictorySpeakerSampleDurationsAreRejected()
{
var provider = CreateProvider(new Dictionary<string, string?>
{
["MeetingAssistant:SpeakerIdentification:MinimumSampleSpeechDuration"] = "00:01:01",
["MeetingAssistant:SpeakerIdentification:MaximumSampleDuration"] = "00:01:00"
});
var exception = Assert.Throws<InvalidOperationException>(
() => provider.GetRequiredProfile(null));
Assert.Contains("MinimumSampleSpeechDuration", exception.Message);
Assert.Contains("MaximumSampleDuration", exception.Message);
}
[Fact]
public void DuplicateProfileHotkeysAreRejected()
{
@@ -1,103 +1,253 @@
using MeetingAssistant.Summary;
using Microsoft.Extensions.AI;
using System.Net;
using System.Text;
using System.Text.Json;
namespace MeetingAssistant.Tests;
public sealed class LiteLlmResponsesChatClientTests
{
[Fact]
public void ParserIgnoresReasoningItemsWithNullStatusAndReadsText()
public async Task ClientAssemblesStreamedTextResponseWithMetadataAndUsage()
{
const string json = """
var handler = new SequencedHttpMessageHandler(
new HttpResponseMessage(HttpStatusCode.OK)
{
"id": "resp_test",
"created_at": 1779147100,
"model": "gpt-5.5-2026-04-23",
"output": [
{
"type": "reasoning",
"summary": [],
"status": null
},
{
"type": "message",
"role": "assistant",
"status": "completed",
"content": [
{
"type": "output_text",
"text": "OK"
}
]
}
]
}
""";
Content = new StringContent(
"""
data: {"type":"response.created","response":{"id":"resp_stream","created_at":1779147100,"model":"gpt-5.5","object":"response","output":[],"parallel_tool_calls":true,"status":"in_progress","store":false},"sequence_number":0}
var response = LiteLlmResponsesChatClient.ParseResponseJson(json);
data: {"type":"response.output_item.added","output_index":0,"item":{"id":"msg_stream","type":"message","status":"in_progress","content":[],"role":"assistant"},"sequence_number":1}
Assert.Equal("OK", response.Text);
Assert.Equal("resp_test", response.ResponseId);
Assert.Equal("gpt-5.5-2026-04-23", response.ModelId);
data: {"type":"response.content_part.added","item_id":"msg_stream","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"text":""},"sequence_number":2}
data: {"type":"response.output_text.delta","item_id":"msg_stream","output_index":0,"content_index":0,"delta":"Streamed OK","sequence_number":3}
data: {"type":"response.output_text.done","item_id":"msg_stream","output_index":0,"content_index":0,"text":"Streamed OK","sequence_number":4}
data: {"type":"response.content_part.done","item_id":"msg_stream","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"text":"Streamed OK"},"sequence_number":5}
data: {"type":"response.output_item.done","output_index":0,"item":{"id":"msg_stream","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"Streamed OK"}],"role":"assistant"},"sequence_number":6}
data: {"type":"response.completed","response":{"id":"resp_stream","created_at":1779147100,"model":"gpt-5.5","object":"response","output":[],"parallel_tool_calls":true,"status":"completed","store":false,"usage":{"input_tokens":12,"output_tokens":3,"total_tokens":15}},"sequence_number":7}
data: [DONE]
""",
Encoding.UTF8,
"text/event-stream")
});
using var client = CreateClient(handler, reconnectionAttempts: 0);
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "reply")]);
Assert.Equal("Streamed OK", response.Text);
Assert.Equal("resp_stream", response.ResponseId);
Assert.Equal("gpt-5.5", response.ModelId);
Assert.Equal(DateTimeOffset.FromUnixTimeSeconds(1779147100), response.CreatedAt);
Assert.NotNull(response.Usage);
Assert.Equal(12, response.Usage.InputTokenCount);
Assert.Equal(3, response.Usage.OutputTokenCount);
Assert.Equal(15, response.Usage.TotalTokenCount);
}
[Fact]
public void ParserReadsFunctionCalls()
public async Task ClientAssemblesStreamedFunctionCall()
{
const string json = """
var handler = new SequencedHttpMessageHandler(
new HttpResponseMessage(HttpStatusCode.OK)
{
"output": [
{
"type": "function_call",
"call_id": "call_1",
"name": "write_summary",
"arguments": "{\"markdown\":\"# Summary\\nDone\"}",
"status": "completed"
}
]
}
""";
Content = new StringContent(
"""
data: {"type":"response.output_item.done","output_index":0,"item":{"type":"function_call","call_id":"call_stream","name":"write_summary","arguments":"{\"markdown\":\"# Summary\\nDone\"}","status":"completed"}}
var response = LiteLlmResponsesChatClient.ParseResponseJson(json);
data: {"type":"response.completed","response":{"id":"resp_stream","model":"gpt-5.5","output":[]}}
data: [DONE]
""",
Encoding.UTF8,
"text/event-stream")
});
using var client = CreateClient(handler, reconnectionAttempts: 0);
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "write summary")]);
var call = Assert.IsType<FunctionCallContent>(Assert.Single(response.Messages[0].Contents));
Assert.Equal("call_1", call.CallId);
Assert.Equal("call_stream", call.CallId);
Assert.Equal("write_summary", call.Name);
Assert.NotNull(call.Arguments);
Assert.Equal("# Summary\nDone", call.Arguments["markdown"]);
Assert.Equal("# Summary\nDone", call.Arguments["markdown"]?.ToString());
}
[Fact]
public void ParserReadsUsage()
public async Task ClientUsesNonStreamingResponsesWhenConfigured()
{
const string json = """
{
"usage": {
"input_tokens": 123,
"output_tokens": 45,
"total_tokens": 168
},
"output": [
var handler = new RecordingHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(
"""
{
"type": "message",
"content": [
"id": "resp_nonstream",
"created_at": 1779147100,
"model": "gpt-5.5",
"object": "response",
"output": [
{
"type": "output_text",
"text": "OK"
"id": "msg_nonstream",
"type": "message",
"status": "completed",
"content": [
{
"type": "output_text",
"annotations": [],
"text": "Non-streamed OK"
}
],
"role": "assistant"
}
]
],
"parallel_tool_calls": true,
"status": "completed",
"store": false,
"usage": {
"input_tokens": 12,
"output_tokens": 4,
"total_tokens": 16
}
}
]
}
""";
""",
Encoding.UTF8,
"application/json")
});
using var client = CreateClient(handler, reconnectionAttempts: 0, useStreaming: false);
var response = LiteLlmResponsesChatClient.ParseResponseJson(json);
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "reply")]);
Assert.NotNull(response.Usage);
Assert.Equal(123, response.Usage.InputTokenCount);
Assert.Equal(45, response.Usage.OutputTokenCount);
Assert.Equal(168, response.Usage.TotalTokenCount);
Assert.Equal("Non-streamed OK", response.Text);
Assert.Contains("\"stream\":false", Assert.Single(handler.RequestBodies));
}
[Fact]
public async Task ClientSendsChatMessagesAsResponsesMessageItems()
{
var handler = new RecordingHttpMessageHandler(_ => CreateStreamedTextResponse("Done."));
using var client = CreateClient(handler, reconnectionAttempts: 0);
await client.GetResponseAsync(
[
new ChatMessage(ChatRole.User, "write summary"),
new ChatMessage(ChatRole.Assistant, "I will inspect the meeting.")
]);
using var request = JsonDocument.Parse(Assert.Single(handler.RequestBodies));
var inputItems = request.RootElement.GetProperty("input").EnumerateArray().ToArray();
Assert.Equal(2, inputItems.Length);
Assert.All(inputItems, item => Assert.Equal("message", item.GetProperty("type").GetString()));
Assert.Equal("user", inputItems[0].GetProperty("role").GetString());
Assert.Equal(
"input_text",
Assert.Single(inputItems[0].GetProperty("content").EnumerateArray())
.GetProperty("type")
.GetString());
Assert.Equal("assistant", inputItems[1].GetProperty("role").GetString());
Assert.Equal(
"output_text",
Assert.Single(inputItems[1].GetProperty("content").EnumerateArray())
.GetProperty("type")
.GetString());
Assert.DoesNotContain("\"type\":\"unknown\"", handler.RequestBodies[0]);
}
[Fact]
public async Task AgentLoopReturnsMalformedFunctionArgumentsWithoutInvokingTool()
{
var responses = new Queue<HttpResponseMessage>(
[
new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(
"""
data: {"type":"response.output_item.done","output_index":0,"item":{"type":"function_call","call_id":"call_invalid","name":"write_summary","arguments":"{not-json","status":"completed"}}
data: {"type":"response.completed","response":{"id":"resp_invalid","model":"gpt-5.5","output":[]}}
data: [DONE]
""",
Encoding.UTF8,
"text/event-stream")
},
CreateStreamedTextResponse("Recovered.")
]);
var handler = new RecordingHttpMessageHandler(_ => responses.Dequeue());
var toolInvoked = false;
var tool = AIFunctionFactory.Create(
(string markdown) =>
{
toolInvoked = true;
return markdown;
},
"write_summary",
"Writes a summary.");
using var innerClient = CreateClient(handler, reconnectionAttempts: 0);
using var functionClient = innerClient
.AsBuilder()
.UseFunctionInvocation(
loggerFactory: null,
client => client.FunctionInvoker = FunctionInvocationGuard.InvokeAsync)
.Build();
var response = await functionClient.GetResponseAsync(
[new ChatMessage(ChatRole.User, "write summary")],
new ChatOptions { Tools = [tool] });
Assert.False(toolInvoked);
Assert.Equal("Recovered.", response.Text);
Assert.Equal(2, handler.RequestBodies.Count);
Assert.Contains("invalid_tool_arguments", handler.RequestBodies[1]);
Assert.Contains("call_invalid", handler.RequestBodies[1]);
}
[Fact]
public async Task ClientReportsVisibleReasoningSummariesSeparatelyFromResponseText()
{
var handler = new SequencedHttpMessageHandler(
CreateStreamedTextResponse(
"Done.",
"Checked the configured rules file.",
"Prepared a targeted update."));
var reasoningSummaries = new List<string>();
using var client = CreateClient(
handler,
reconnectionAttempts: 0,
reasoningSummaryChanged: reasoningSummaries.Add);
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "update rules")]);
Assert.Equal("Done.", response.Text);
Assert.Equal(
[
"Checked the configured rules file.",
"Prepared a targeted update."
],
reasoningSummaries);
}
[Fact]
public async Task ClientDoesNotFailResponseWhenReasoningSummaryCallbackFails()
{
var handler = new SequencedHttpMessageHandler(
CreateStreamedTextResponse("Done.", "Checked the configured rules file."));
using var client = CreateClient(
handler,
reconnectionAttempts: 0,
reasoningSummaryChanged: _ => throw new InvalidOperationException("UI failed"));
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "update rules")]);
Assert.Equal("Done.", response.Text);
}
[Fact]
@@ -108,24 +258,7 @@ public sealed class LiteLlmResponsesChatClientTests
{
Content = new StringContent("Internal Server Error")
},
new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("""
{
"output": [
{
"type": "message",
"content": [
{
"type": "output_text",
"text": "Done."
}
]
}
]
}
""")
});
CreateStreamedTextResponse("Done."));
var retryCount = 0;
using var client = CreateClient(handler, reconnectionAttempts: 1, retrying: () => retryCount++);
@@ -156,24 +289,7 @@ public sealed class LiteLlmResponsesChatClientTests
[Fact]
public async Task ClientSendsUserInitiatorOnceThenAgentInitiator()
{
var handler = new RecordingHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("""
{
"output": [
{
"type": "message",
"content": [
{
"type": "output_text",
"text": "Done."
}
]
}
]
}
""")
});
var handler = new RecordingHttpMessageHandler(_ => CreateStreamedTextResponse("Done."));
using var client = CreateClient(handler, reconnectionAttempts: 0);
await client.GetResponseAsync([new ChatMessage(ChatRole.User, "write summary")]);
@@ -205,24 +321,7 @@ public sealed class LiteLlmResponsesChatClientTests
};
}
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("""
{
"output": [
{
"type": "message",
"content": [
{
"type": "output_text",
"text": "Done."
}
]
}
]
}
""")
};
return CreateStreamedTextResponse("Done.");
});
using var client = CreateClient(
handler,
@@ -257,24 +356,7 @@ public sealed class LiteLlmResponsesChatClientTests
};
}
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("""
{
"output": [
{
"type": "message",
"content": [
{
"type": "output_text",
"text": "Done."
}
]
}
]
}
""")
};
return CreateStreamedTextResponse("Done.");
});
using var client = CreateClient(
handler,
@@ -306,7 +388,9 @@ public sealed class LiteLlmResponsesChatClientTests
HttpMessageHandler handler,
int reconnectionAttempts,
LiteLlmResponsesCompactionOptions? compactionOptions = null,
Action? retrying = null)
Action? retrying = null,
Action<string>? reasoningSummaryChanged = null,
bool useStreaming = true)
{
return new LiteLlmResponsesChatClient(
new HttpClient(handler)
@@ -320,7 +404,50 @@ public sealed class LiteLlmResponsesChatClientTests
reconnectionAttempts,
TimeSpan.Zero,
compactionOptions,
retrying: retrying);
retrying: retrying,
reasoningSummaryChanged: reasoningSummaryChanged,
useStreaming: useStreaming);
}
private static HttpResponseMessage CreateStreamedTextResponse(
string text,
params string[] reasoningSummaries)
{
var events = new List<string>();
for (var index = 0; index < reasoningSummaries.Length; index++)
{
events.Add("data: " + JsonSerializer.Serialize(new
{
type = "response.reasoning_summary_text.delta",
item_id = "reasoning_stream",
output_index = 0,
summary_index = index,
delta = reasoningSummaries[index]
}));
}
var outputIndex = reasoningSummaries.Length > 0 ? 1 : 0;
events.Add("data: " + JsonSerializer.Serialize(new
{
type = "response.output_text.delta",
item_id = "msg_stream",
output_index = outputIndex,
content_index = 0,
delta = text
}));
events.Add(
"""data: {"type":"response.completed","response":{"id":"resp_stream","created_at":1779147100,"model":"gpt-5.5","object":"response","output":[],"parallel_tool_calls":true,"status":"completed","store":false,"usage":{"input_tokens":12,"output_tokens":3,"total_tokens":15}}}""");
events.Add("data: [DONE]");
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(
string.Join($"{Environment.NewLine}{Environment.NewLine}", events)
+ Environment.NewLine
+ Environment.NewLine,
Encoding.UTF8,
"text/event-stream")
};
}
private sealed class SequencedHttpMessageHandler : HttpMessageHandler
@@ -13,23 +13,50 @@ namespace MeetingAssistant.Tests;
public sealed class LiteLlmScreenshotOcrClientTests
{
[Fact]
public async Task ExtractUsesAgentStreamingTransportByDefault()
{
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
var handler = new RecordingHandler(
CreateStreamedTextResponse("Streamed OCR text"),
"text/event-stream");
var client = new LiteLlmScreenshotOcrClient(
() => handler,
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
var options = new MeetingAssistantOptions
{
Agent =
{
Endpoint = "https://summary.local",
Model = "vision-model",
Key = "agent-key",
UseStreaming = true
}
};
var result = await client.ExtractAsync(
screenshotPath,
"Extract screenshot.",
options,
CancellationToken.None);
Assert.Equal("Streamed OCR text", result.Text);
using var payload = JsonDocument.Parse(handler.RequestBody!);
Assert.True(payload.RootElement.GetProperty("stream").GetBoolean());
var message = Assert.Single(payload.RootElement.GetProperty("input").EnumerateArray());
Assert.Equal("message", message.GetProperty("type").GetString());
var content = message.GetProperty("content").EnumerateArray().ToArray();
Assert.Equal("input_text", content[0].GetProperty("type").GetString());
Assert.Equal("Extract screenshot.", content[0].GetProperty("text").GetString());
Assert.Equal("input_image", content[1].GetProperty("type").GetString());
Assert.Equal("data:image/png;base64,AQID", content[1].GetProperty("image_url").GetString());
}
[Fact]
public async Task ExtractUsesAgentEndpointAndModelWhenOcrEndpointAndModelAreBlank()
{
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
await File.WriteAllBytesAsync(screenshotPath, [1, 2, 3]);
var handler = new RecordingHandler("""
{
"output": [
{
"content": [
{ "type": "output_text", "text": "Visible slide text" }
]
}
]
}
""");
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
var handler = new RecordingHandler(CreateNonStreamingTextResponse("Visible slide text"));
var client = new LiteLlmScreenshotOcrClient(
() => handler,
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
@@ -39,7 +66,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
{
Endpoint = "https://summary.local",
Model = "summary-model",
Key = "agent-key"
Key = "agent-key",
UseStreaming = false
},
Screenshots =
{
@@ -64,6 +92,7 @@ public sealed class LiteLlmScreenshotOcrClientTests
Assert.Equal("Bearer", handler.Authorization?.Scheme);
Assert.Equal("ocr-key", handler.Authorization?.Parameter);
using var payload = JsonDocument.Parse(handler.RequestBody!);
Assert.False(payload.RootElement.GetProperty("stream").GetBoolean());
Assert.Equal("summary-model", payload.RootElement.GetProperty("model").GetString());
var content = payload.RootElement
.GetProperty("input")[0]
@@ -75,10 +104,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
[Fact]
public async Task ExtractUsesScreenshotOcrEndpointAndModelWhenConfigured()
{
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
await File.WriteAllBytesAsync(screenshotPath, [4, 5, 6]);
var handler = new RecordingHandler("""{ "output_text": "OCR result" }""");
var screenshotPath = await CreateScreenshotAsync([4, 5, 6]);
var handler = new RecordingHandler(CreateNonStreamingTextResponse("OCR result"));
var client = new LiteLlmScreenshotOcrClient(
() => handler,
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
@@ -88,7 +115,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
{
Endpoint = "https://summary.local",
Model = "summary-model",
Key = "agent-key"
Key = "agent-key",
UseStreaming = false
},
Screenshots =
{
@@ -116,14 +144,15 @@ public sealed class LiteLlmScreenshotOcrClientTests
[Fact]
public async Task ExtractParsesCropMetadataAndOmitsMetadataFromReturnedText()
{
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
await File.WriteAllBytesAsync(screenshotPath, CreatePngBytes(8, 6));
var handler = new RecordingHandler("""
{
"output_text": "Slide text\n\n```json\n{ \"crop\": { \"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4 } }\n```"
}
""");
var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6));
var handler = new RecordingHandler(CreateNonStreamingTextResponse(
"""
Slide text
```json
{ "crop": { "x": 1, "y": 2, "width": 3, "height": 4 } }
```
"""));
var client = new LiteLlmScreenshotOcrClient(
() => handler,
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
@@ -131,7 +160,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
{
Agent =
{
Key = "agent-key"
Key = "agent-key",
UseStreaming = false
}
};
@@ -150,13 +180,84 @@ public sealed class LiteLlmScreenshotOcrClientTests
StringComparison.Ordinal);
}
[Fact]
public async Task ExtractParsesAttendeeMetadataAndOmitsMetadataFromReturnedText()
{
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
var handler = new RecordingHandler(CreateNonStreamingTextResponse(
"""
Visible participant tiles: Ada and Grace.
```json
{ "crop": null, "attendees": ["Ada Lovelace", "Grace Hopper"] }
```
"""));
var client = new LiteLlmScreenshotOcrClient(
() => handler,
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
var options = new MeetingAssistantOptions
{
Agent =
{
Key = "agent-key",
UseStreaming = false
}
};
var result = await client.ExtractAsync(
screenshotPath,
"Extract screenshot.",
options,
CancellationToken.None);
Assert.Equal("Visible participant tiles: Ada and Grace.", result.Text);
Assert.Equal(["Ada Lovelace", "Grace Hopper"], result.Attendees);
}
[Fact]
public async Task ExtractIgnoresMalformedAttendeesMetadataAndStillParsesCrop()
{
var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6));
var handler = new RecordingHandler(CreateNonStreamingTextResponse(
"""
Slide text
```json
{ "crop": { "x": 1, "y": 2, "width": 3, "height": 4 }, "attendees": "Ada" }
```
"""));
var client = new LiteLlmScreenshotOcrClient(
() => handler,
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
var options = new MeetingAssistantOptions
{
Agent =
{
Key = "agent-key",
UseStreaming = false
}
};
var result = await client.ExtractAsync(
screenshotPath,
"Extract screenshot.",
options,
CancellationToken.None);
Assert.Equal("Slide text", result.Text);
Assert.Equal(new ScreenshotCropCoordinates(1, 2, 3, 4), result.Crop);
Assert.Empty(result.Attendees);
}
private sealed class RecordingHandler : HttpMessageHandler
{
private readonly string responseBody;
private readonly string mediaType;
public RecordingHandler(string responseBody)
public RecordingHandler(string responseBody, string mediaType = "application/json")
{
this.responseBody = responseBody;
this.mediaType = mediaType;
}
public Uri? RequestUri { get; private set; }
@@ -176,7 +277,7 @@ public sealed class LiteLlmScreenshotOcrClientTests
: await request.Content.ReadAsStringAsync(cancellationToken);
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(responseBody, Encoding.UTF8, "application/json")
Content = new StringContent(responseBody, Encoding.UTF8, mediaType)
};
}
}
@@ -193,6 +294,63 @@ public sealed class LiteLlmScreenshotOcrClientTests
bitmap.Save(stream, ImageFormat.Png);
return stream.ToArray();
}
private static string CreateNonStreamingTextResponse(string text)
{
return JsonSerializer.Serialize(new
{
id = "resp_ocr",
created_at = 1779147100,
model = "vision-model",
@object = "response",
output = new[]
{
new
{
id = "msg_ocr",
type = "message",
status = "completed",
content = new[]
{
new
{
type = "output_text",
annotations = Array.Empty<object>(),
text
}
},
role = "assistant"
}
},
parallel_tool_calls = true,
status = "completed",
store = false
});
}
private static string CreateStreamedTextResponse(string text)
{
var delta = JsonSerializer.Serialize(new
{
type = "response.output_text.delta",
item_id = "msg_ocr",
output_index = 0,
content_index = 0,
delta = text
});
return
$"data: {delta}{Environment.NewLine}{Environment.NewLine}" +
"""data: {"type":"response.completed","response":{"id":"resp_ocr","created_at":1779147100,"model":"vision-model","object":"response","output":[],"parallel_tool_calls":true,"status":"completed","store":false}}""" +
$"{Environment.NewLine}{Environment.NewLine}data: [DONE]{Environment.NewLine}{Environment.NewLine}";
}
private static async Task<string> CreateScreenshotAsync(byte[] bytes)
{
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
await File.WriteAllBytesAsync(screenshotPath, bytes);
return screenshotPath;
}
}
#pragma warning restore CA1416
@@ -9,10 +9,10 @@
<ItemGroup>
<PackageReference Include="coverlet.collector" Version="10.0.1" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.8" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.7.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.12" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.10.1" />
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5" />
<PackageReference Include="xunit.runner.visualstudio" Version="4.0.0" />
</ItemGroup>
<ItemGroup>
@@ -135,6 +135,72 @@ public sealed class MeetingNoteStoreTests
Assert.Equal("User notes.", loaded.UserNotes);
}
[Fact]
public async Task StoreEscapesApostrophePrefixedAttendeesBeforeWritingFrontmatter()
{
var (store, saved) = await SaveNoteAsync(
title: "Escaped Attendees",
attendees: ["'Ada Lovelace"],
projects: [],
userNotes: "Discuss attendee import.");
var content = await File.ReadAllTextAsync(saved.Path);
var loaded = await store.ReadAsync(saved.Path, CancellationToken.None);
Assert.Contains("- \"'Ada Lovelace\"", content);
Assert.Equal(["'Ada Lovelace"], loaded.Frontmatter.Attendees);
Assert.Equal("Discuss attendee import.", loaded.UserNotes);
}
[Theory]
[InlineData("Ada # platform lead")]
[InlineData("- Ada Lovelace")]
[InlineData("? Ada Lovelace")]
[InlineData("{Ada: Platform}")]
[InlineData("*Ada")]
[InlineData("&Ada")]
[InlineData("!Ada")]
[InlineData("| Ada")]
[InlineData("> Ada")]
[InlineData("@Ada")]
[InlineData("`Ada")]
[InlineData("true")]
[InlineData("null")]
[InlineData("2026-07-08")]
[InlineData("Ada \"The Architect\" Lovelace")]
[InlineData("C:\\People\\Ada")]
public async Task StoreEscapesYamlSensitiveAttendeesBeforeWritingFrontmatter(string attendee)
{
var (store, saved) = await SaveNoteAsync(
title: "Escaped Attendees",
attendees: [attendee],
projects: [],
userNotes: "Discuss attendee import.");
var loaded = await store.ReadAsync(saved.Path, CancellationToken.None);
Assert.Equal([attendee], loaded.Frontmatter.Attendees);
Assert.Equal("Discuss attendee import.", loaded.UserNotes);
}
[Theory]
[InlineData("Planning # Q3")]
[InlineData("- Planning")]
[InlineData("{Planning: Q3}")]
[InlineData("true")]
[InlineData("2026-07-08")]
public async Task StoreEscapesYamlSensitiveScalarAndProjectFrontmatterValues(string value)
{
var (store, saved) = await SaveNoteAsync(
title: value,
attendees: [],
projects: [value],
userNotes: "Discuss YAML escaping.");
var loaded = await store.ReadAsync(saved.Path, CancellationToken.None);
Assert.Equal(value, loaded.Frontmatter.Title);
Assert.Equal([value], loaded.Frontmatter.Projects);
Assert.Equal("Discuss YAML escaping.", loaded.UserNotes);
}
[Fact]
public void ActionLinkEscapesSummaryFileName()
{
@@ -147,6 +213,26 @@ public sealed class MeetingNoteStoreTests
link);
}
private static async Task<(MarkdownMeetingNoteStore Store, MeetingNote Saved)> SaveNoteAsync(
string title,
IReadOnlyList<string> attendees,
IReadOnlyList<string> projects,
string userNotes)
{
var (vaultRoot, store) = CreateStore();
var note = MeetingNoteTemplate.Create(
title: title,
attendees: attendees,
projects: projects,
transcriptPath: Path.Combine(vaultRoot, "Meetings", "Transcripts", "20260519-transcript.md"),
assistantContextPath: Path.Combine(vaultRoot, "Meetings", "Assistant Context", "20260519-context.md"),
summaryPath: Path.Combine(vaultRoot, "Meetings", "Summaries", "20260519-summary.md"),
userNotes: userNotes);
var saved = await store.SaveAsync(note, CancellationToken.None);
return (store, saved);
}
private static (string VaultRoot, MarkdownMeetingNoteStore Store) CreateStore()
{
var vaultRoot = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
@@ -1,6 +1,9 @@
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Screenshots;
using MeetingAssistant.Speakers;
using MeetingAssistant.Workflow;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using System.Drawing;
using System.Drawing.Imaging;
@@ -130,6 +133,194 @@ public sealed class MeetingScreenshotServiceTests
Assert.Contains("Shared screen text", context);
}
[Fact]
public async Task CaptureAddsOcrAttendeesToMeetingNoteThroughCanonicalizer()
{
var fixture = await ScreenshotFixture.CreateAsync(
options =>
{
options.Screenshots.Ocr.Enabled = true;
},
attendees: ["Ada Lovelace"]);
var ocr = new CapturingScreenshotOcrClient(
"Visible participant tiles: Ada, Grace, and Ada again.",
attendees: ["Ada L.", "Grace Hopper", "Ada Lovelace"]);
var canonicalizer = new MappingAttendeeCanonicalizer(new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["Ada L."] = "Ada Lovelace"
});
var service = fixture.CreateService(
new FixedScreenshotCapture([1, 2, 3]),
ocr,
canonicalizer);
await service.CaptureAsync(
fixture.Artifacts,
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
fixture.Options,
CancellationToken.None);
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
var meeting = await fixture.NoteStore.ReadAsync(fixture.Artifacts.MeetingNotePath, CancellationToken.None);
Assert.Equal(["Ada Lovelace", "Grace Hopper"], meeting.Frontmatter.Attendees);
Assert.Contains(canonicalizer.Requests, request =>
request.SequenceEqual(["Ada Lovelace", "Ada L.", "Grace Hopper", "Ada Lovelace"]));
}
[Fact]
public async Task CaptureTransformsOcrAttendeesBeforeWritingMeetingNote()
{
var fixture = await ScreenshotFixture.CreateAsync(
options =>
{
options.Screenshots.Ocr.Enabled = true;
});
var workflowEngine = new TransformingAttendeeWorkflowEngine("Ada Lovelace (Contoso)", "Ada Lovelace");
var service = fixture.CreateService(
new FixedScreenshotCapture([1, 2, 3]),
new CapturingScreenshotOcrClient(
"Visible participant tile: Ada.",
attendees: ["Ada Lovelace (Contoso)"]),
meetingWorkflowEngine: workflowEngine);
await service.CaptureAsync(
fixture.Artifacts,
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
fixture.Options,
CancellationToken.None);
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
var meeting = await fixture.NoteStore.ReadAsync(fixture.Artifacts.MeetingNotePath, CancellationToken.None);
Assert.Equal(["Ada Lovelace"], meeting.Frontmatter.Attendees);
Assert.Equal(["Ada Lovelace (Contoso)"], workflowEngine.AttendeeRequests);
}
[Fact]
public async Task CaptureWritesRetryLinkWhenOcrFails()
{
var fixture = await ScreenshotFixture.CreateAsync(options =>
{
options.Api.PublicBaseUrl = "http://localhost:5090";
options.Screenshots.Ocr.Enabled = true;
});
var service = fixture.CreateService(
new FixedScreenshotCapture([1, 2, 3]),
new ThrowingScreenshotOcrClient("vision offline"));
var result = await service.CaptureAsync(
fixture.Artifacts,
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
fixture.Options,
CancellationToken.None);
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
var screenshotId = ExtractScreenshotOcrId(context);
Assert.Contains("_OCR failed: vision offline_", context);
Assert.Contains("<!-- screenshot-ocr:", context);
Assert.Contains("<!-- /screenshot-ocr:", context);
Assert.Contains("[Retry screenshot OCR](http://localhost:5090/meetings/screenshot-ocr/retry?", context);
Assert.Contains($"screenshotId={screenshotId}", context);
Assert.Contains($"screenshotPath={Uri.EscapeDataString(result.ScreenshotPath)}", context);
Assert.Contains($"assistantContextPath={Uri.EscapeDataString(fixture.Artifacts.AssistantContextPath)}", context);
}
[Fact]
public async Task RetryOcrReplacesFailureForSameScreenshot()
{
var fixture = await ScreenshotFixture.CreateAsync(options =>
{
options.Screenshots.Ocr.Enabled = true;
});
var ocr = new SequencedScreenshotOcrClient(
new InvalidOperationException("vision offline"),
new ScreenshotOcrResult("Retried OCR text", null, ["Grace Hopper"]));
var service = fixture.CreateService(
new FixedScreenshotCapture([1, 2, 3]),
ocr);
var result = await service.CaptureAsync(
fixture.Artifacts,
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
fixture.Options,
CancellationToken.None);
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
var failedContext = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
var screenshotId = ExtractScreenshotOcrId(failedContext);
var retry = await service.TriggerOcrRetryAsync(
fixture.Artifacts,
result.ScreenshotPath,
screenshotId,
fixture.Options,
CancellationToken.None);
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
Assert.NotNull(retry);
Assert.Equal(result.ScreenshotPath, retry.ScreenshotPath);
Assert.Equal(screenshotId, retry.ScreenshotId);
Assert.Equal([result.ScreenshotPath, result.ScreenshotPath], ocr.ScreenshotPaths);
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
Assert.Contains("### OCR", context);
Assert.Contains("Retried OCR text", context);
Assert.DoesNotContain("vision offline", context);
Assert.DoesNotContain("Retry screenshot OCR", context);
Assert.DoesNotContain("<!-- screenshot-ocr:", context);
}
[Fact]
public async Task ProcessMeetingNoteImageEmbedsAppendsContextAndRunsOcrWithoutCropOrAttendees()
{
var fixture = await ScreenshotFixture.CreateAsync(
options =>
{
options.Screenshots.Ocr.Enabled = true;
},
attendees: ["Ada Lovelace"],
userNotes: "Discussed ![[whiteboard.png]] and ![Diagram](attachments/diagram.png).");
var noteFolder = Path.GetDirectoryName(fixture.Artifacts.MeetingNotePath)!;
var attachmentsFolder = Path.Combine(noteFolder, "attachments");
Directory.CreateDirectory(attachmentsFolder);
var whiteboardPath = Path.Combine(noteFolder, "whiteboard.png");
var diagramPath = Path.Combine(attachmentsFolder, "diagram.png");
await File.WriteAllBytesAsync(whiteboardPath, [1, 2, 3]);
await File.WriteAllBytesAsync(diagramPath, [4, 5, 6]);
var ocr = new CapturingScreenshotOcrClient(
"Manual image OCR",
new ScreenshotCropCoordinates(1, 1, 2, 2),
["Grace Hopper"]);
var service = fixture.CreateService(
new FixedScreenshotCapture([1, 2, 3]),
ocr);
var originalMeetingNote = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
var result = await service.ProcessMeetingNoteImageEmbedsAsync(
fixture.Artifacts,
fixture.Options,
CancellationToken.None);
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
Assert.Equal(2, result.QueuedCount);
Assert.Equal(2, ocr.CallCount);
Assert.Equal([whiteboardPath, diagramPath], ocr.ScreenshotPaths);
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
Assert.Contains("## Meeting Note Image", context);
Assert.Contains("Image from meeting note.", context);
Assert.Contains("Original embed: `![[whiteboard.png]]`", context);
Assert.Contains("Original embed: `![Diagram](attachments/diagram.png)`", context);
Assert.Contains("whiteboard.png", context);
Assert.Contains("diagram.png", context);
Assert.Contains("Manual image OCR", context);
Assert.DoesNotContain("Cropped screenshot", context);
Assert.Equal(originalMeetingNote, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
var meeting = await fixture.NoteStore.ReadAsync(fixture.Artifacts.MeetingNotePath, CancellationToken.None);
Assert.Equal(["Ada Lovelace"], meeting.Frontmatter.Attendees);
}
[Fact]
public async Task WaitForPendingOcrWaitsForRunningScreenshotOcr()
{
@@ -192,11 +383,13 @@ public sealed class MeetingScreenshotServiceTests
private ScreenshotFixture(
MeetingAssistantOptions options,
MeetingSessionArtifacts artifacts,
MarkdownMeetingArtifactStore artifactStore)
MarkdownMeetingArtifactStore artifactStore,
MarkdownMeetingNoteStore noteStore)
{
Options = options;
Artifacts = artifacts;
ArtifactStore = artifactStore;
NoteStore = noteStore;
}
public MeetingAssistantOptions Options { get; }
@@ -205,8 +398,12 @@ public sealed class MeetingScreenshotServiceTests
public MarkdownMeetingArtifactStore ArtifactStore { get; }
public MarkdownMeetingNoteStore NoteStore { get; }
public static async Task<ScreenshotFixture> CreateAsync(
Action<MeetingAssistantOptions>? configure = null)
Action<MeetingAssistantOptions>? configure = null,
IReadOnlyList<string>? attendees = null,
string userNotes = "")
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var options = new MeetingAssistantOptions
@@ -218,6 +415,9 @@ public sealed class MeetingScreenshotServiceTests
}
};
configure?.Invoke(options);
var noteStore = new MarkdownMeetingNoteStore(
Microsoft.Extensions.Options.Options.Create(options),
NullLogger<MarkdownMeetingNoteStore>.Instance);
var artifacts = new MeetingSessionArtifacts(
Path.Combine(root, "Notes", "meeting.md"),
Path.Combine(root, "Transcripts", "transcript.md"),
@@ -225,6 +425,7 @@ public sealed class MeetingScreenshotServiceTests
Path.Combine(root, "Summaries", "summary.md"));
var artifactStore = new MarkdownMeetingArtifactStore(
NullLogger<MarkdownMeetingArtifactStore>.Instance);
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
var meeting = MeetingNoteTemplate.Create(
"Planning",
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
@@ -232,26 +433,37 @@ public sealed class MeetingScreenshotServiceTests
assistantContextPath: artifacts.AssistantContextPath,
summaryPath: artifacts.SummaryPath) with
{
Path = artifacts.MeetingNotePath
Path = artifacts.MeetingNotePath,
UserNotes = userNotes
};
meeting.Frontmatter.Attendees = attendees?.ToList() ?? [];
var savedMeeting = await noteStore.SaveAsync(
meeting,
options,
CancellationToken.None);
await artifactStore.CreateAssistantContextAsync(
artifacts,
meeting,
savedMeeting,
"",
null,
CancellationToken.None);
return new ScreenshotFixture(options, artifacts, artifactStore);
return new ScreenshotFixture(options, artifacts, artifactStore, noteStore);
}
public MeetingScreenshotService CreateService(
IActiveWindowScreenshotCapture capture,
IScreenshotOcrClient ocrClient)
IScreenshotOcrClient ocrClient,
ISpeakerIdentityAttendeeCanonicalizer? attendeeCanonicalizer = null,
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
{
return new MeetingScreenshotService(
capture,
ArtifactStore,
NoteStore,
attendeeCanonicalizer ?? PassthroughSpeakerIdentityAttendeeCanonicalizer.Instance,
ocrClient,
NullLogger<MeetingScreenshotService>.Instance);
NullLogger<MeetingScreenshotService>.Instance,
meetingWorkflowEngine);
}
}
@@ -276,15 +488,18 @@ public sealed class MeetingScreenshotServiceTests
public CapturingScreenshotOcrClient(
string text = "",
ScreenshotCropCoordinates? crop = null)
ScreenshotCropCoordinates? crop = null,
IReadOnlyList<string>? attendees = null)
{
result = new ScreenshotOcrResult(text, crop);
result = new ScreenshotOcrResult(text, crop, attendees ?? []);
}
public int CallCount { get; private set; }
public string? Prompt { get; private set; }
public List<string> ScreenshotPaths { get; } = [];
public Task<ScreenshotOcrResult> ExtractAsync(
string screenshotPath,
string prompt,
@@ -293,6 +508,7 @@ public sealed class MeetingScreenshotServiceTests
{
CallCount++;
Prompt = prompt;
ScreenshotPaths.Add(screenshotPath);
return Task.FromResult(result);
}
}
@@ -320,7 +536,79 @@ public sealed class MeetingScreenshotServiceTests
public void Release(string value)
{
result.TrySetResult(new ScreenshotOcrResult(value, null));
result.TrySetResult(new ScreenshotOcrResult(value, null, []));
}
}
private sealed class ThrowingScreenshotOcrClient : IScreenshotOcrClient
{
private readonly string message;
public ThrowingScreenshotOcrClient(string message)
{
this.message = message;
}
public Task<ScreenshotOcrResult> ExtractAsync(
string screenshotPath,
string prompt,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
throw new InvalidOperationException(message);
}
}
private sealed class SequencedScreenshotOcrClient : IScreenshotOcrClient
{
private readonly Queue<object> results;
public SequencedScreenshotOcrClient(params object[] results)
{
this.results = new Queue<object>(results);
}
public List<string> ScreenshotPaths { get; } = [];
public Task<ScreenshotOcrResult> ExtractAsync(
string screenshotPath,
string prompt,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
ScreenshotPaths.Add(screenshotPath);
var result = results.Dequeue();
if (result is Exception exception)
{
throw exception;
}
return Task.FromResult((ScreenshotOcrResult)result);
}
}
private sealed class MappingAttendeeCanonicalizer : ISpeakerIdentityAttendeeCanonicalizer
{
private readonly IReadOnlyDictionary<string, string> aliases;
public MappingAttendeeCanonicalizer(IReadOnlyDictionary<string, string> aliases)
{
this.aliases = aliases;
}
public List<IReadOnlyList<string>> Requests { get; } = [];
public Task<IReadOnlyList<string>> CanonicalizeAsync(
IReadOnlyList<string> attendees,
CancellationToken cancellationToken)
{
Requests.Add(attendees.ToList());
var result = attendees
.Select(attendee => aliases.TryGetValue(attendee, out var canonical) ? canonical : attendee)
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
return Task.FromResult<IReadOnlyList<string>>(result);
}
}
@@ -336,6 +624,17 @@ public sealed class MeetingScreenshotServiceTests
bitmap.Save(stream, ImageFormat.Png);
return stream.ToArray();
}
private static string ExtractScreenshotOcrId(string context)
{
const string prefix = "<!-- screenshot-ocr:";
var start = context.IndexOf(prefix, StringComparison.Ordinal);
Assert.True(start >= 0);
start += prefix.Length;
var end = context.IndexOf(" -->", start, StringComparison.Ordinal);
Assert.True(end > start);
return context[start..end];
}
}
#pragma warning restore CA1416
@@ -7,6 +7,37 @@ namespace MeetingAssistant.Tests;
public sealed class MeetingSummaryInstructionBuilderTests
{
[Fact]
public async Task BuilderAddsBoundMetadataWithoutAgentsAndRequestGuidanceToCustomPrompt()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectsRoot = Path.Combine(root, "Projects");
var artifacts = CreateArtifacts(root);
await WriteMeetingNoteAsync(artifacts.MeetingNotePath, ["Alpha"]);
Directory.CreateDirectory(Path.Combine(projectsRoot, "Alpha"));
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "PROJECT.md"),
"---\nname: Alpha Platform\ndescription: Builds meeting automation\n---\nInternal notes");
await WriteProjectAgentsAsync(projectsRoot, "Unbound", "Do not expose these instructions");
var builder = new MeetingSummaryInstructionBuilder(Options.Create(new MeetingAssistantOptions
{
Agent = new AgentOptions { InitialPrompt = "Custom base." },
Vault = new VaultOptions { ProjectsFolder = projectsRoot }
}));
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
Assert.StartsWith("Custom base.", instructions);
Assert.Contains("# Alpha", instructions);
Assert.Contains("Alpha Platform", instructions);
Assert.Contains("Builds meeting automation", instructions);
Assert.Contains("PROJECT.md", instructions);
Assert.Contains("256", instructions);
Assert.Contains("request_project_association", instructions);
Assert.Contains("request_workflow_change", instructions);
Assert.DoesNotContain("Internal notes", instructions);
Assert.DoesNotContain("Do not expose", instructions);
}
[Fact]
public async Task BuilderUsesDefaultPromptWhenConfiguredPromptIsBlank()
{
@@ -24,6 +55,7 @@ public sealed class MeetingSummaryInstructionBuilderTests
Assert.Contains("You are the Meeting Assistant summary agent.", instructions);
Assert.Contains("include only the most relevant cropped screenshots", instructions);
Assert.Contains("Do not include every cropped screenshot", instructions);
Assert.Contains("encode spaces in image-link targets as `%20`", instructions);
Assert.Contains("Use add_attendee and remove_attendee", instructions);
Assert.Contains("partial screenshot", instructions);
Assert.Contains("override_speaker", instructions);
@@ -39,6 +71,27 @@ public sealed class MeetingSummaryInstructionBuilderTests
Assert.Contains("title parameter", instructions);
}
[Fact]
public async Task DefaultPromptTreatsAssistantContextAsMeetingMemoryForUncertainty()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var artifacts = CreateArtifacts(root);
await WriteMeetingNoteAsync(artifacts.MeetingNotePath, []);
var builder = new MeetingSummaryInstructionBuilder(Options.Create(new MeetingAssistantOptions
{
Agent = new AgentOptions { InitialPrompt = " " },
Vault = new VaultOptions { ProjectsFolder = Path.Combine(root, "Projects") }
}));
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
Assert.Contains("meeting-specific memory", instructions);
Assert.Contains("unexpected problems", instructions);
Assert.Contains("missing information", instructions);
Assert.Contains("assumptions", instructions);
Assert.Contains("write_context", instructions);
}
[Fact]
public async Task BuilderUsesConfiguredPrompt()
{
@@ -53,7 +106,8 @@ public sealed class MeetingSummaryInstructionBuilderTests
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
Assert.Equal("Custom summarizer instructions.", instructions);
Assert.StartsWith("Custom summarizer instructions.", instructions);
Assert.Contains("request_project_association", instructions);
}
[Fact]
@@ -74,21 +128,12 @@ public sealed class MeetingSummaryInstructionBuilderTests
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
var expected = """
Base.
---
projects:
# Alpha
Alpha instructions.
# Beta
Beta instructions.
""";
Assert.Equal(NormalizeLineEndings(expected), NormalizeLineEndings(instructions));
Assert.StartsWith("Base.", instructions);
Assert.Contains("---\nprojects:", NormalizeLineEndings(instructions));
Assert.Contains("# Alpha", instructions);
Assert.Contains("Alpha instructions.", instructions);
Assert.Contains("# Beta", instructions);
Assert.Contains("Beta instructions.", instructions);
}
private static MeetingSessionArtifacts CreateArtifacts(string root)
@@ -189,7 +189,9 @@ public sealed class MeetingSummaryRetryRunnerTests
return;
}
await stateChanged.Task.WaitAsync(TimeSpan.FromMilliseconds(100));
await Task.WhenAny(
stateChanged.Task,
Task.Delay(TimeSpan.FromMilliseconds(100)));
}
throw new TimeoutException($"Expected {count} state changes.");
@@ -293,6 +293,44 @@ public sealed class MeetingSummaryToolTests
Assert.Contains("transcript: \"[[../Transcripts/transcript|Transcript]]\"", meetingNote);
}
[Fact]
public async Task ToolsTransformAddedAttendeeBeforeWritingMeetingNote()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var artifacts = new MeetingSessionArtifacts(
MeetingNotePath: Path.Combine(root, "Meetings", "Notes", "meeting.md"),
TranscriptPath: Path.Combine(root, "Meetings", "Transcripts", "transcript.md"),
AssistantContextPath: Path.Combine(root, "Meetings", "Assistant Context", "context.md"),
SummaryPath: Path.Combine(root, "Meetings", "Summaries", "summary.md"));
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
await File.WriteAllTextAsync(
artifacts.MeetingNotePath,
"""
---
title: Meeting
attendees: []
projects: []
transcript: "[[../Transcripts/transcript|Transcript]]"
assistant_context: "[[../Assistant Context/context|Assistant Context]]"
summary: "[[../Summaries/summary|Summary]]"
---
User note line.
""");
var workflowEngine = new TransformingAttendeeWorkflowEngine("Ada Lovelace (Contoso)", "Ada Lovelace");
var tools = new MeetingSummaryTools(
artifacts,
new MeetingAssistantOptions(),
meetingWorkflowEngine: workflowEngine);
Assert.Equal("Added attendee Ada Lovelace.", await tools.AddAttendee("Ada Lovelace (Contoso)"));
var meetingNote = await File.ReadAllTextAsync(artifacts.MeetingNotePath);
Assert.Contains("- Ada Lovelace", meetingNote);
Assert.DoesNotContain("Ada Lovelace (Contoso)", meetingNote);
Assert.Equal(["Ada Lovelace (Contoso)"], workflowEngine.AttendeeRequests);
}
[Fact]
public async Task ToolsOverrideSpeakerInTranscriptAndRecordOverride()
{
@@ -799,4 +837,5 @@ public sealed class MeetingSummaryToolTests
Assert.False(tools.SummaryWasWritten);
Assert.False(File.Exists(artifacts.SummaryPath));
}
}
@@ -310,6 +310,10 @@ public sealed class MeetingWorkflowEngineTests
[InlineData("speaker_identified:\n name: ADA", "speaker", null, null, "ada", true)]
[InlineData("speaker_identified:\n name: Ada", "speaker", null, null, "Grace", false)]
[InlineData("speaker_identified: {}", "speaker", null, null, "Grace", true)]
[InlineData("attendee_added:\n equals: Ada Lovelace", "attendee", null, null, "ada lovelace", true)]
[InlineData("attendee_added:\n contains: contoso", "attendee", null, null, "Ada (Contoso)", true)]
[InlineData("attendee_added:\n regex: '^Ada .+Contoso\\)$'", "attendee", null, null, "Ada (Contoso)", true)]
[InlineData("attendee_added:\n regex: '^Grace'", "attendee", null, null, "Ada (Contoso)", false)]
public async Task TriggerMatchingScenarios(
string triggerYaml,
string eventKind,
@@ -318,6 +322,16 @@ public sealed class MeetingWorkflowEngineTests
string? speaker,
bool shouldRun)
{
var stepYaml = eventKind == "attendee"
? """
- uses: set_property
property: attendee.name
value: Triggered
"""
: """
- uses: add_project
value: Triggered
""";
var fixture = await WorkflowFixture.CreateAsync(
$$"""
rules:
@@ -325,14 +339,21 @@ public sealed class MeetingWorkflowEngineTests
on:
- {{triggerYaml}}
steps:
- uses: add_project
value: Triggered
{{stepYaml}}
""");
await fixture.Engine.RunAsync(
CreateEvent(eventKind, fixture.Artifacts, from, to, speaker),
fixture.Options,
CancellationToken.None);
var workflowEvent = CreateEvent(eventKind, fixture.Artifacts, from, to, speaker);
if (eventKind == "attendee")
{
var transformed = await fixture.Engine.TransformAttendeeAsync(
workflowEvent,
fixture.Options,
CancellationToken.None);
Assert.Equal(shouldRun ? "Triggered" : speaker, transformed);
return;
}
await fixture.Engine.RunAsync(workflowEvent, fixture.Options, CancellationToken.None);
var meeting = await fixture.ReadMeetingAsync();
Assert.Equal(shouldRun, meeting.Frontmatter.Projects.Contains("Triggered"));
@@ -603,6 +624,34 @@ public sealed class MeetingWorkflowEngineTests
Assert.Equal(["Ada Lovelace"], meeting.Frontmatter.Attendees);
}
[Fact]
public async Task AttendeeAddedRuleCanTransformWorkflowAddedAttendee()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: add-raw-attendee
on:
- created: {}
steps:
- uses: add_attendee
value: 'Ada Lovelace (Contoso)'
- name: clean-contoso-attendee
on:
- attendee_added:
contains: 'Contoso'
steps:
- uses: set_property
property: attendee.name
value: '@Model.Attendee.Name.Replace(" (Contoso)", "")'
""");
await RunCreatedAsync(fixture);
var meeting = await fixture.ReadMeetingAsync();
Assert.Equal(["Ada Lovelace"], meeting.Frontmatter.Attendees);
}
[Fact]
public async Task AddAttendeeDoesNotDuplicateExistingDisplayNameFromEmailAddress()
{
@@ -792,6 +841,7 @@ public sealed class MeetingWorkflowEngineTests
{
"created" => MeetingWorkflowEvent.Created(artifacts),
"speaker" => MeetingWorkflowEvent.SpeakerIdentified(artifacts, speaker ?? "Ada"),
"attendee" => MeetingWorkflowEvent.AttendeeAdded(artifacts, speaker ?? "Ada"),
"state" => MeetingWorkflowEvent.StateTransition(
artifacts,
ParseState(from ?? "collecting metadata"),
@@ -0,0 +1,248 @@
using MeetingAssistant.Recording;
using Microsoft.Extensions.Logging.Abstractions;
namespace MeetingAssistant.Tests;
public sealed class MicrophoneAudioSourceTests
{
[Fact]
public async Task CaptureMovesToNewlyResolvedMicrophoneWhenCurrentCaptureFails()
{
var captureSources = new SequenceMicrophoneCaptureSourceFactory(
new FailingAfterChunkAudioSource(Pcm16(1_000)),
new ActiveAudioSource(Pcm16(2_000)));
var source = new MicrophoneAudioSource(
captureSources,
NullLogger<MicrophoneAudioSource>.Instance,
TimeSpan.Zero);
using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5));
await using var chunks = source
.CaptureAsync(new MeetingAssistantOptions(), cancellation.Token)
.GetAsyncEnumerator(cancellation.Token);
Assert.True(await chunks.MoveNextAsync());
Assert.Equal(1_000, BitConverter.ToInt16(chunks.Current.Pcm));
Assert.True(await chunks.MoveNextAsync());
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
Assert.Equal(2, captureSources.CaptureCreationCount);
await cancellation.CancelAsync();
}
[Fact]
public async Task CaptureWaitsForMicrophoneToBecomeAvailable()
{
var captureSources = new InitiallyUnavailableMicrophoneCaptureSourceFactory(
new ActiveAudioSource(Pcm16(3_000)));
var source = new MicrophoneAudioSource(
captureSources,
NullLogger<MicrophoneAudioSource>.Instance,
TimeSpan.Zero);
using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(5));
await using var chunks = source
.CaptureAsync(new MeetingAssistantOptions(), cancellation.Token)
.GetAsyncEnumerator(cancellation.Token);
Assert.True(await chunks.MoveNextAsync());
Assert.Equal(3_000, BitConverter.ToInt16(chunks.Current.Pcm));
Assert.Equal(2, captureSources.CaptureCreationCount);
await cancellation.CancelAsync();
}
[Fact]
public async Task AcceptedNewMicrophoneReplacesCaptureWithinTheSameStream()
{
var selection = new MicrophoneDeviceSelection();
var devices = new SelectableCaptureDevices(selection);
var prompts = new AcceptingPrompt();
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
var source = new MicrophoneAudioSource(devices, NullLogger<MicrophoneAudioSource>.Instance,
TimeSpan.Zero, monitor, selection);
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
await using var chunks = source.CaptureAsync(new MeetingAssistantOptions(), lifetime.Token).GetAsyncEnumerator();
try
{
Assert.True(await chunks.MoveNextAsync());
Assert.Equal(1_000, BitConverter.ToInt16(chunks.Current.Pcm));
devices.Available = [new("old", "Built-in"), new("new", "USB headset")];
Assert.Equal("new", (await prompts.Shown.Task.WaitAsync(TimeSpan.FromSeconds(2))).Id);
var nextChunk = chunks.MoveNextAsync().AsTask();
Assert.False(nextChunk.IsCompleted);
Assert.Null(selection.SelectedDeviceId);
prompts.Response.SetResult(true);
Assert.True(await nextChunk.WaitAsync(TimeSpan.FromSeconds(2)));
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
Assert.Equal("new", selection.SelectedDeviceId);
Assert.False(lifetime.IsCancellationRequested);
}
finally
{
await lifetime.CancelAsync();
}
}
[Fact]
public async Task EndingMeetingDuringCreationDisposesAbandonedCapture()
{
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var abandonedCapture = new DisposableAudioSource();
var factory = new CallbackCaptureFactory(() =>
{
lifetime.Cancel();
return abandonedCapture;
});
var source = new MicrophoneAudioSource(factory, NullLogger<MicrophoneAudioSource>.Instance);
await using var chunks = source.CaptureAsync(lifetime.Token).GetAsyncEnumerator();
Assert.False(await chunks.MoveNextAsync());
Assert.True(abandonedCapture.Disposed);
}
private sealed class CallbackCaptureFactory(Func<IMeetingAudioSource> create) : IMicrophoneCaptureSourceFactory
{
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options) => create();
}
[Fact]
public async Task AcceptedSwitchDuringCreationDisposesAbandonedCapture()
{
var selection = new MicrophoneDeviceSelection();
var devices = new SwitchDuringCreationDevices(selection);
var prompts = new AcceptingPrompt();
prompts.Response.SetResult(true);
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
var source = new MicrophoneAudioSource(devices, NullLogger<MicrophoneAudioSource>.Instance,
TimeSpan.Zero, monitor, selection);
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
await using var chunks = source.CaptureAsync(new MeetingAssistantOptions(), lifetime.Token).GetAsyncEnumerator();
try
{
Assert.True(await chunks.MoveNextAsync());
Assert.Equal("new", selection.SelectedDeviceId);
Assert.Equal(2_000, BitConverter.ToInt16(chunks.Current.Pcm));
Assert.True(devices.AbandonedCapture.Disposed);
}
finally
{
await lifetime.CancelAsync();
}
}
private sealed class SwitchDuringCreationDevices(MicrophoneDeviceSelection selection)
: IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
{
private MicrophoneDevice[] available = [new("old", "Built-in")];
private int creations;
private volatile bool switchObserved;
public DisposableAudioSource AbandonedCapture { get; } = new();
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
{
if (selection.SelectedDeviceId == "new") switchObserved = true;
return Volatile.Read(ref available);
}
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
=> new(GetAvailableMicrophones(), available[0]);
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
{
if (++creations > 1) return new ActiveAudioSource(Pcm16(2_000));
Volatile.Write(ref available, [new("old", "Built-in"), new("new", "USB headset")]);
// Let the acceptance callback finish before returning the already-opened source.
if (!SpinWait.SpinUntil(() => switchObserved, TimeSpan.FromSeconds(2)))
throw new TimeoutException("The new microphone was not selected during capture creation.");
return AbandonedCapture;
}
}
private sealed class DisposableAudioSource : IMeetingAudioSource, IDisposable
{
public bool Disposed { get; private set; }
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
=> new ActiveAudioSource(Pcm16(1_000)).CaptureAsync(cancellationToken);
public void Dispose() => Disposed = true;
}
private sealed class AcceptingPrompt : IMicrophoneSwitchPromptService
{
public TaskCompletionSource<MicrophoneDevice> Shown { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
{
Shown.TrySetResult(microphone);
return Response.Task.WaitAsync(cancellationToken);
}
}
private sealed class SelectableCaptureDevices(MicrophoneDeviceSelection selection)
: IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
{
public volatile MicrophoneDevice[] Available = [new("old", "Built-in")];
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones() => Available;
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
=> new(Available, selection.Resolve(options.Recording.MicrophoneDeviceId, Available[0], Available));
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
=> new ActiveAudioSource(Pcm16(GetMicrophoneSnapshot(options).Current!.Id == "new" ? (short)2_000 : (short)1_000));
}
private static byte[] Pcm16(short sample)
{
return BitConverter.GetBytes(sample);
}
private sealed class SequenceMicrophoneCaptureSourceFactory(params IMeetingAudioSource[] sources)
: IMicrophoneCaptureSourceFactory
{
private readonly Queue<IMeetingAudioSource> sources = new(sources);
public int CaptureCreationCount { get; private set; }
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
{
CaptureCreationCount++;
return sources.Dequeue();
}
}
private sealed class InitiallyUnavailableMicrophoneCaptureSourceFactory(IMeetingAudioSource availableSource)
: IMicrophoneCaptureSourceFactory
{
public int CaptureCreationCount { get; private set; }
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
{
CaptureCreationCount++;
if (CaptureCreationCount == 1)
{
throw new InvalidOperationException("No microphone is currently available.");
}
return availableSource;
}
}
private sealed class FailingAfterChunkAudioSource(byte[] pcm) : IMeetingAudioSource
{
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
await Task.Yield();
yield return new AudioChunk(pcm, 16000, 1);
throw new InvalidOperationException("The active microphone was disconnected.");
}
}
private sealed class ActiveAudioSource(byte[] pcm) : IMeetingAudioSource
{
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
yield return new AudioChunk(pcm, 16000, 1);
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
}
}
}
@@ -0,0 +1,67 @@
using MeetingAssistant.Recording;
namespace MeetingAssistant.Tests;
public sealed class MicrophoneSelectionTests
{
[Fact]
public void BlankConfiguredMicrophoneUsesDefaultDevice()
{
var selection = new MicrophoneDeviceSelection();
var selected = selection.Resolve(
configuredDeviceId: null,
new MicrophoneDevice("default-id", "default microphone"),
[
new MicrophoneDevice("default-id", "default microphone"),
new MicrophoneDevice("other-id", "other microphone")
]);
Assert.Equal("default-id", selected?.Id);
}
[Fact]
public void ConfiguredMicrophoneUsesMatchingDevice()
{
var selection = new MicrophoneDeviceSelection();
var selected = selection.Resolve(
"other-id",
new MicrophoneDevice("default-id", "default microphone"),
[
new MicrophoneDevice("default-id", "default microphone"),
new MicrophoneDevice("other-id", "other microphone")
]);
Assert.Equal("other-id", selected?.Id);
}
[Fact]
public void RuntimeMicrophoneSelectionOverridesConfiguredDevice()
{
var selection = new MicrophoneDeviceSelection();
selection.Select("runtime-id");
var selected = selection.Resolve(
"configured-id",
new MicrophoneDevice("default-id", "default microphone"),
[
new MicrophoneDevice("configured-id", "configured microphone"),
new MicrophoneDevice("runtime-id", "runtime microphone")
]);
Assert.Equal("runtime-id", selected?.Id);
}
[Fact]
public void UnavailableDefaultMicrophoneFallsBackToAnotherActiveDevice()
{
var selection = new MicrophoneDeviceSelection();
var selected = selection.Resolve(
configuredDeviceId: null,
new MicrophoneDevice("disconnected-id", "disconnected microphone"),
[new MicrophoneDevice("backup-id", "backup microphone")]);
Assert.Equal("backup-id", selected?.Id);
}
}
@@ -0,0 +1,163 @@
using System.Threading.Channels;
using MeetingAssistant.Recording;
using Microsoft.Extensions.Logging.Abstractions;
namespace MeetingAssistant.Tests;
public sealed class MicrophoneSwitchMonitorTests
{
[Fact]
public async Task CancellationDuringConfirmationDeviceLookupCannotSwitchMicrophone()
{
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var devices = new MutableDevices();
var prompts = new ImmediatelyAcceptedPrompt(() => devices.BeforeRead = lifetime.Cancel);
var monitor = new MicrophoneSwitchMonitor(devices, prompts,
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10), TimeSpan.FromMinutes(1));
var switched = false;
var monitoring = monitor.RunAsync((_, _) => switched = true, lifetime.Token);
devices.Available = [new("usb", "USB microphone")];
await monitoring.WaitAsync(TimeSpan.FromSeconds(2));
Assert.True(prompts.WasShown);
Assert.False(switched);
}
[Fact]
public async Task OffersOnlyNewDevicesOnceAndSwitchesOnlyAfterAcceptance()
{
var oldDevice = new MicrophoneDevice("old", "Built-in microphone");
var newDevice = new MicrophoneDevice("new", "USB headset");
var devices = new MutableDevices { Available = [oldDevice] };
var prompts = new ControlledPrompts();
var monitor = CreateMonitor(devices, prompts);
var switched = new TaskCompletionSource<MicrophoneDevice>(TaskCreationOptions.RunContinuationsAsynchronously);
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var monitoring = monitor.RunAsync((device, _) => switched.TrySetResult(device), lifetime.Token);
try
{
devices.Available = [oldDevice, newDevice];
var prompt = await prompts.NextAsync();
Assert.Equal(newDevice, prompt.Device);
Assert.False(switched.Task.IsCompleted);
await devices.WaitForPollsAsync();
Assert.False(prompts.HasMore);
prompt.Response.SetResult(true);
Assert.Equal(newDevice, await switched.Task.WaitAsync(TimeSpan.FromSeconds(2)));
await devices.WaitForPollsAsync();
Assert.False(prompts.HasMore);
}
finally
{
await lifetime.CancelAsync();
await monitoring;
}
}
[Theory]
[InlineData("declined")]
[InlineData("removed")]
[InlineData("expired")]
[InlineData("ended")]
public async Task InvalidOrDeclinedPromptsCannotChangeSelection(string outcome)
{
var device = new MicrophoneDevice("usb", "USB microphone");
var devices = new MutableDevices();
var prompts = new ControlledPrompts();
var monitor = CreateMonitor(devices, prompts,
outcome == "expired" ? TimeSpan.FromMilliseconds(30) : null);
var switched = false;
using var lifetime = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var monitoring = monitor.RunAsync((_, _) => switched = true, lifetime.Token);
try
{
devices.Available = [device];
var prompt = await prompts.NextAsync();
if (outcome == "declined")
{
prompt.Response.SetResult(false);
await devices.WaitForPollsAsync();
}
else
{
if (outcome == "removed") devices.Available = [];
if (outcome == "ended") await lifetime.CancelAsync();
var canceled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
using var registration = prompt.Cancellation.Register(() => canceled.TrySetResult());
await canceled.Task.WaitAsync(TimeSpan.FromSeconds(2));
prompt.Response.TrySetResult(true);
}
await lifetime.CancelAsync();
await monitoring;
Assert.False(switched);
}
finally
{
await lifetime.CancelAsync();
await monitoring;
}
}
private static MicrophoneSwitchMonitor CreateMonitor(MutableDevices devices, ControlledPrompts prompts,
TimeSpan? promptLifetime = null) => new(devices, prompts,
NullLogger<MicrophoneSwitchMonitor>.Instance, TimeSpan.FromMilliseconds(10),
promptLifetime ?? TimeSpan.FromMinutes(1));
private sealed class MutableDevices : IMicrophoneDeviceProvider
{
private readonly Channel<bool> polls = Channel.CreateUnbounded<bool>();
public volatile MicrophoneDevice[] Available = [];
public Action? BeforeRead { get; set; }
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
{
BeforeRead?.Invoke();
polls.Writer.TryWrite(true);
return Available;
}
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
=> new(Available, Available.FirstOrDefault());
public async Task WaitForPollsAsync()
{
while (polls.Reader.TryRead(out _)) { }
for (var i = 0; i < 3; i++)
await polls.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
}
}
private sealed class ImmediatelyAcceptedPrompt(Action beforeAcceptance) : IMicrophoneSwitchPromptService
{
public bool WasShown { get; private set; }
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
{
WasShown = true;
beforeAcceptance();
return Task.FromResult(true);
}
}
private sealed class ControlledPrompts : IMicrophoneSwitchPromptService
{
private readonly Channel<Prompt> requests = Channel.CreateUnbounded<Prompt>();
public bool HasMore => requests.Reader.TryPeek(out _);
public Task<Prompt> NextAsync() => requests.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
public async Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
{
var prompt = new Prompt(microphone, cancellationToken);
requests.Writer.TryWrite(prompt);
return await prompt.Response.Task.WaitAsync(cancellationToken);
}
}
private sealed record Prompt(MicrophoneDevice Device, CancellationToken Cancellation)
{
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
}
}
@@ -25,4 +25,16 @@ public sealed class NotificationActivationArgumentsTests
Assert.Equal("abc123", arguments["promptId"]);
Assert.Equal("continue", arguments["response"]);
}
[Fact]
public void InactivityPromptActionsIncludePauseAndParseItsResponse()
{
Assert.Contains(
MeetingInactivityPromptActions.All,
action => action.Content == "Pause transcription" &&
action.ResponseArgument == "pause");
Assert.Equal(
MeetingInactivityPromptResponse.Pause,
MeetingInactivityPromptActions.ParseResponse("pause"));
}
}
@@ -8,7 +8,7 @@ public sealed class OutlookClassicMeetingMetadataProviderTests
[Fact]
public void ExtractAgendaStopsBeforeTeamsJoinInformation()
{
var agenda = OutlookClassicMeetingMetadataProvider.ExtractAgenda(
var agenda = OutlookClassicAppointmentMetadata.ExtractAgenda(
"""
Review current prototype
Decide next backend
@@ -25,7 +25,7 @@ public sealed class OutlookClassicMeetingMetadataProviderTests
[Fact]
public void NormalizeAttendeesDeduplicatesOrganizerAndRecipientEmail()
{
var attendees = OutlookClassicMeetingMetadataProvider.NormalizeAttendees([
var attendees = OutlookClassicAppointmentMetadata.NormalizeAttendees([
"Marcus.Altmann@sew-eurodrive.de",
"Marcus.Altmann@sew-eurodrive.de <Marcus.Altmann@sew-eurodrive.de>",
"Schweigert, Manuel",
@@ -37,5 +37,14 @@ public sealed class OutlookClassicMeetingMetadataProviderTests
"Schweigert, Manuel"
], attendees);
}
[Theory]
[InlineData("Canceled: Project sync")]
[InlineData("Cancelled: Project sync")]
[InlineData("Abgesagt: Project sync")]
public void SubjectIndicatesCancellationRecognizesOutlookCanceledPrefixes(string subject)
{
Assert.True(OutlookClassicCom.SubjectIndicatesCancellation(subject));
}
}
#endif
@@ -40,7 +40,7 @@ public sealed class OutlookMeetingCandidateSelectorTests
{
var now = new DateTime(2026, 5, 20, 10, 0, 0);
var endingOverlap = new Candidate(now.AddMinutes(-25), now.AddMinutes(2));
var upcoming = new Candidate(now.AddMinutes(5), now.AddMinutes(35));
var upcoming = new Candidate(now.AddMinutes(1), now.AddMinutes(31));
var selected = OutlookMeetingCandidateSelector.Select(
[endingOverlap, upcoming],
@@ -1,10 +1,90 @@
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Summary;
using System.Text.Json;
namespace MeetingAssistant.Tests;
public sealed class ProjectKnowledgeToolTests
{
[Theory]
[InlineData("name: Incomplete project")]
[InlineData("description: Missing name")]
[InlineData("name: ' '\ndescription: Missing name")]
public async Task CatalogUsesLegacyFallbackForIncompleteMetadata(string frontmatter)
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectRoot = Path.Combine(root, "Projects", "Alpha");
Directory.CreateDirectory(projectRoot);
await File.WriteAllTextAsync(Path.Combine(projectRoot, "PROJECT.md"), $"---\n{frontmatter}\n---\nNotes");
var tools = new MeetingSummaryTools(CreateArtifacts(root),
new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
using var catalog = JsonDocument.Parse(await tools.ListProjects());
var project = Assert.Single(catalog.RootElement.EnumerateArray());
Assert.Equal("Alpha", project.GetProperty("displayname").GetString());
Assert.Equal("", project.GetProperty("description").GetString());
}
[Fact]
public async Task ProjectMetadataWriteEnforcesDescriptionLimitAndPreservesOtherContent()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectRoot = Path.Combine(root, "Projects", "Alpha");
Directory.CreateDirectory(projectRoot);
var artifacts = CreateArtifacts(root);
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: Alpha\n---");
var tools = new MeetingSummaryTools(artifacts,
new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
var original = $"---\nname: Alpha Platform\ndescription: {new string('a', 256)}\nowner: Ada\n---\nUser-authored project notes.";
Assert.Equal("Alpha/PROJECT.md", await tools.WriteProjectFile("Alpha", "PROJECT.md", original));
var rejected = await tools.WriteProjectFile("Alpha", "PROJECT.md",
$"description: {new string('b', 257)}", from: 3, to: 3);
Assert.StartsWith("Refused:", rejected);
Assert.Equal(original, await File.ReadAllTextAsync(Path.Combine(projectRoot, "PROJECT.md")));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("Alpha", "PROJECT.md", "No metadata", replace_file: true));
Assert.Equal("Alpha/PROJECT.md", await tools.WriteProjectFile("Alpha", "PROJECT.md",
"description: Updated project purpose", from: 3, to: 3));
var updated = await tools.ReadProjectFile("Alpha", "PROJECT.md");
Assert.Contains("owner: Ada", updated);
Assert.Contains("User-authored project notes.", updated);
Assert.Contains("Updated project purpose", await tools.ListProjects());
}
[Fact]
public async Task CatalogListsAllProjectsWithMetadataAndLegacyFallbackWithoutGrantingReadAccess()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectsRoot = Path.Combine(root, "Projects");
Directory.CreateDirectory(Path.Combine(projectsRoot, "Alpha"));
Directory.CreateDirectory(Path.Combine(projectsRoot, "Legacy"));
Directory.CreateDirectory(Path.Combine(projectsRoot, "Malformed"));
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "PROJECT.md"),
"---\nname: Alpha Platform\ndescription: Meeting automation\n---\nPrivate project notes");
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Alpha", "AGENTS.md"), "Private instructions");
await File.WriteAllTextAsync(Path.Combine(projectsRoot, "Malformed", "PROJECT.md"), "---\nname: [broken\n---");
var tools = new MeetingSummaryTools(CreateArtifacts(root),
new MeetingAssistantOptions { Vault = { ProjectsFolder = projectsRoot } });
var catalog = await tools.ListProjects();
using var json = JsonDocument.Parse(catalog);
var projects = json.RootElement.EnumerateArray().ToArray();
Assert.Equal(3, projects.Length);
Assert.Equal("Alpha", projects[0].GetProperty("id").GetString());
Assert.Equal("Alpha Platform", projects[0].GetProperty("displayname").GetString());
Assert.Equal("Meeting automation", projects[0].GetProperty("description").GetString());
Assert.Equal("Legacy", projects[1].GetProperty("displayname").GetString());
Assert.Equal("", projects[1].GetProperty("description").GetString());
Assert.Equal("Malformed", projects[2].GetProperty("displayname").GetString());
Assert.Equal("", projects[2].GetProperty("description").GetString());
Assert.DoesNotContain("Private", catalog);
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
}
[Fact]
public async Task ToolsOperateOnProjectsBoundInMeetingFrontmatter()
{
@@ -48,7 +128,7 @@ public sealed class ProjectKnowledgeToolTests
}
});
Assert.Equal("MeetingAssistant", await tools.ListProjects());
Assert.Contains("MeetingAssistant", await tools.ListProjects());
Assert.Equal("README.md\nnotes/context.md", await tools.ListProjectFiles("MeetingAssistant"));
Assert.Equal("Second alpha line\nThird beta line", await tools.ReadProjectFile("MeetingAssistant", "README.md", 2, 99));
@@ -56,15 +136,15 @@ public sealed class ProjectKnowledgeToolTests
Assert.Equal("MeetingAssistant/notes/summary.md", writeResult);
Assert.Equal("# Project Update", await File.ReadAllTextAsync(Path.Combine(meetingAssistantRoot, "notes", "summary.md")));
Assert.Equal("IgnoredProject/ignored.md", await tools.WriteProjectFile("IgnoredProject", "ignored.md", "changed"));
Assert.Equal("alpha should not be searched\nchanged", await File.ReadAllTextAsync(Path.Combine(ignoredRoot, "ignored.md")));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("IgnoredProject", "ignored.md", "changed"));
Assert.Equal("alpha should not be searched", await File.ReadAllTextAsync(Path.Combine(ignoredRoot, "ignored.md")));
Assert.Equal(
"README.md:2 Second alpha line",
await tools.Search("alpha"));
}
[Fact]
public async Task ListProjectsAcceptsScalarProjectFrontmatter()
public async Task ProjectFileAccessAcceptsScalarProjectFrontmatter()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectsRoot = Path.Combine(root, "Projects");
@@ -88,7 +168,8 @@ public sealed class ProjectKnowledgeToolTests
}
});
Assert.Equal("MeetingAssistant", await tools.ListProjects());
Assert.Equal("MeetingAssistant/notes.md", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "Bound project"));
Assert.Equal("Bound project", await tools.ReadProjectFile("MeetingAssistant", "notes.md"));
}
[Fact]
@@ -100,8 +181,11 @@ public sealed class ProjectKnowledgeToolTests
Directory.CreateDirectory(projectRoot);
var projectFile = Path.Combine(projectRoot, "notes.md");
await File.WriteAllTextAsync(projectFile, "one\ntwo\nthree\nfour");
var artifacts = CreateArtifacts(root);
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: MeetingAssistant\n---\n");
var tools = new MeetingSummaryTools(
CreateArtifacts(root),
artifacts,
new MeetingAssistantOptions
{
Vault =
@@ -132,8 +216,13 @@ public sealed class ProjectKnowledgeToolTests
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var projectsRoot = Path.Combine(root, "Projects");
Directory.CreateDirectory(Path.Combine(projectsRoot, "MeetingAssistant"));
var artifacts = CreateArtifacts(root);
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\nprojects: MeetingAssistant\n---\n");
var outsidePath = Path.Combine(projectsRoot, "outside.md");
await File.WriteAllTextAsync(outsidePath, "Preserve outside content");
var tools = new MeetingSummaryTools(
CreateArtifacts(root),
artifacts,
new MeetingAssistantOptions
{
Vault =
@@ -144,6 +233,7 @@ public sealed class ProjectKnowledgeToolTests
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MissingProject", "notes.md", "content"));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "../outside.md", "content"));
Assert.Equal("Preserve outside content", await File.ReadAllTextAsync(outsidePath));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, insert: 1));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("MeetingAssistant", "notes.md", "content", from: 1, to: 1, replace_file: true));
@@ -18,31 +18,8 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
NullLogger<PyannoteTranscriptFinalizer>.Instance);
var service = new PyannoteDiarizationWarmupHostedService(
finalizer,
new FakeLaunchProfileOptionsProvider(new MeetingAssistantOptions
{
Recording = { TranscriptionProvider = "azure-speech" },
SpeakerIdentification =
{
PyannoteValidation =
{
Enabled = true,
Diarization =
{
Enabled = true,
DockerCommand = "docker",
Image = "meeting-assistant-pyannote-validation:local",
ModelsFolder = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"models"),
Token = "hf_test",
TokenEnv = "",
CommandTimeout = TimeSpan.FromMinutes(1)
}
}
}
}),
new FakeLaunchProfileOptionsProvider(
CreateValidationOptions("meeting-assistant-pyannote-validation:local")),
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
@@ -54,23 +31,109 @@ public sealed class PyannoteDiarizationWarmupHostedServiceTests
Assert.True(commandRunner.RunCancellationWasObserved);
}
[Fact]
public async Task HostedServiceUsesApplicationValidationRuntimeInsteadOfNamedProfileOverride()
{
var commandRunner = new BlockingCommandRunner();
var finalizer = new PyannoteTranscriptFinalizer(
commandRunner,
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
var defaultOptions = CreateValidationOptions("meeting-assistant-pyannote-default:local");
var namedOptions = CreateValidationOptions("meeting-assistant-pyannote-named:local");
var service = new PyannoteDiarizationWarmupHostedService(
finalizer,
new FakeLaunchProfileOptionsProvider(
[
new LaunchProfile("english", namedOptions),
new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, defaultOptions)
]),
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
await commandRunner.WaitForRunAsync();
await service.StopAsync(CancellationToken.None);
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-default:local"));
Assert.DoesNotContain(commandRunner.Commands, command => command.Arguments.Contains("meeting-assistant-pyannote-named:local"));
}
[Fact]
public async Task HostedServiceDoesNotWarmIdentityValidationWhenResemblyzerIsSelected()
{
var commandRunner = new BlockingCommandRunner();
var finalizer = new PyannoteTranscriptFinalizer(
commandRunner,
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
var options = CreateValidationOptions("meeting-assistant-pyannote-validation:local");
options.SpeakerIdentification.Resemblyzer.Enabled = true;
var service = new PyannoteDiarizationWarmupHostedService(
finalizer,
new FakeLaunchProfileOptionsProvider(options),
NullLogger<PyannoteDiarizationWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None);
await Task.Delay(TimeSpan.FromMilliseconds(50));
await service.StopAsync(CancellationToken.None);
Assert.Empty(commandRunner.Commands);
}
private static MeetingAssistantOptions CreateValidationOptions(string image)
{
return new MeetingAssistantOptions
{
Recording = { TranscriptionProvider = "azure-speech" },
SpeakerIdentification =
{
PyannoteValidation =
{
Enabled = true,
Diarization =
{
DockerCommand = "docker",
Image = image,
ModelsFolder = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"models"),
Token = "hf_test",
TokenEnv = "",
CommandTimeout = TimeSpan.FromMinutes(1)
}
}
}
};
}
private sealed class FakeLaunchProfileOptionsProvider : ILaunchProfileOptionsProvider
{
private readonly MeetingAssistantOptions options;
private readonly IReadOnlyList<LaunchProfile> profiles;
public FakeLaunchProfileOptionsProvider(MeetingAssistantOptions options)
: this([new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options)])
{
this.options = options;
}
public FakeLaunchProfileOptionsProvider(IReadOnlyList<LaunchProfile> profiles)
{
this.profiles = profiles;
}
public LaunchProfile GetRequiredProfile(string? name)
{
return new LaunchProfile(ConfigurationLaunchProfileOptionsProvider.DefaultProfileName, options);
var profileName = string.IsNullOrWhiteSpace(name)
? ConfigurationLaunchProfileOptionsProvider.DefaultProfileName
: name;
return profiles.Single(profile => profile.Name.Equals(profileName, StringComparison.OrdinalIgnoreCase));
}
public IReadOnlyList<LaunchProfile> GetProfiles()
{
return [GetRequiredProfile(null)];
return profiles;
}
public IReadOnlyList<LaunchProfileHotkey> GetHotkeys()
@@ -2,6 +2,7 @@ using MeetingAssistant;
using MeetingAssistant.Recording;
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NAudio.Wave;
@@ -57,17 +58,49 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
Assert.Empty(commandRunner.Commands);
}
[Fact]
public async Task OuterToggleControlsValidationWhenLegacyNestedToggleIsFalse()
{
var commandRunner = new CapturingCommandRunner(
"""
__MEETING_ASSISTANT_PYANNOTE_JSON_START__
[{"start":0.0,"end":20.0,"speaker":"SPEAKER_00"}]
__MEETING_ASSISTANT_PYANNOTE_JSON_END__
""");
var modelsFolder = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"models");
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
{
["SpeakerIdentification:PyannoteValidation:Enabled"] = "true",
["SpeakerIdentification:PyannoteValidation:Diarization:Enabled"] = "false",
["SpeakerIdentification:PyannoteValidation:Diarization:BuildImage"] = "false",
["SpeakerIdentification:PyannoteValidation:Diarization:ModelsFolder"] = modelsFolder,
["SpeakerIdentification:PyannoteValidation:Diarization:Token"] = "hf_test",
["SpeakerIdentification:PyannoteValidation:Diarization:TokenEnv"] = ""
})
.Build();
var configuredOptions = configuration.Get<MeetingAssistantOptions>()!;
var validator = CreateValidator(commandRunner, configuredOptions);
var valid = await validator.ValidateSampleAsync(
CreateWav(TimeSpan.FromSeconds(20)),
CancellationToken.None);
Assert.True(valid);
Assert.Contains(commandRunner.Commands, command => command.Arguments.Contains("run"));
}
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
CapturingCommandRunner commandRunner,
bool enabled = true)
{
var finalizer = new PyannoteTranscriptFinalizer(
return CreateValidator(
commandRunner,
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
return new PyannoteSpeakerIdentityMatchValidator(
finalizer,
Options.Create(new MeetingAssistantOptions
new MeetingAssistantOptions
{
SpeakerIdentification = new SpeakerIdentificationOptions
{
@@ -76,9 +109,8 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
Enabled = enabled,
MinimumSingleSpeakerCoverage = 0.90,
MinimumMatchingKnownSnippetRatio = 1,
Diarization = new PyannoteDiarizationOptions
Diarization = new PyannoteRuntimeOptions
{
Enabled = true,
BuildImage = false,
DockerCommand = "docker",
Image = "meeting-assistant-pyannote:local",
@@ -94,7 +126,20 @@ public sealed class PyannoteSpeakerIdentityMatchValidatorTests
}
}
}
}),
});
}
private static PyannoteSpeakerIdentityMatchValidator CreateValidator(
CapturingCommandRunner commandRunner,
MeetingAssistantOptions configuredOptions)
{
var finalizer = new PyannoteTranscriptFinalizer(
commandRunner,
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
return new PyannoteSpeakerIdentityMatchValidator(
finalizer,
Options.Create(configuredOptions),
NullLogger<PyannoteSpeakerIdentityMatchValidator>.Instance);
}
@@ -156,9 +156,8 @@ public sealed class PyannoteTranscriptFinalizerTests
commandRunner,
Options.Create(new MeetingAssistantOptions()),
NullLogger<PyannoteTranscriptFinalizer>.Instance);
var explicitDiarization = new PyannoteDiarizationOptions
var explicitDiarization = new PyannoteRuntimeOptions
{
Enabled = true,
DockerCommand = "docker",
BaseImage = "python:3.11-slim",
Image = "meeting-assistant-pyannote-azure:local",
@@ -169,7 +168,7 @@ public sealed class PyannoteTranscriptFinalizerTests
CommandTimeout = TimeSpan.FromMinutes(1)
};
await finalizer.FinalizeAsync(
await finalizer.FinalizeEnabledAsync(
audioPath,
[new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(1), "Unknown", "hello")],
explicitDiarization,
@@ -186,9 +185,8 @@ public sealed class PyannoteTranscriptFinalizerTests
{
var commandRunner = new CapturingCommandRunner("");
var finalizer = CreateFinalizer(commandRunner, token: "hf_test");
var diarization = new PyannoteDiarizationOptions
var diarization = new PyannoteRuntimeOptions
{
Enabled = true,
DockerCommand = "docker",
BaseImage = "python:3.11-slim",
Image = "meeting-assistant-pyannote-warmup:local",
@@ -218,9 +216,8 @@ public sealed class PyannoteTranscriptFinalizerTests
var commandRunner = new CapturingCommandRunner("");
var finalizer = CreateFinalizer(commandRunner, token: null);
await finalizer.WarmUpAsync(new PyannoteDiarizationOptions
await finalizer.WarmUpAsync(new PyannoteRuntimeOptions
{
Enabled = true,
Token = "",
TokenEnv = "",
ModelsFolder = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"), "models")
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,596 @@
using MeetingAssistant;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerSpeakerIdentificationServiceTests
{
[Fact]
public async Task LiveMatchRelabelsSpeakerAndStoresFiveVectorsWithoutWavSnippets()
{
await using var fixture = await Fixture.CreateAsync();
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
var result = await service.IdentifyKnownSpeakersAsync(
fixture.CreateRequest("Guest03", sampleCount: 5),
CancellationToken.None);
Assert.Equal("Chris", result.Segments.Single().Speaker);
Assert.Equal("Chris", result.SpeakerMappings["Guest03"]);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Equal(7, saved.VoiceVectors.Count);
Assert.Empty(saved.Snippets);
Assert.Single(saved.References);
Assert.Single(fixture.Encoder.Requests);
Assert.Equal(5, fixture.Encoder.Requests[0].Count);
}
[Fact]
public async Task SummaryOverrideCreatesNamedIdentityFromThreeAvailableVectors()
{
await using var fixture = await Fixture.CreateAsync();
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
var request = fixture.CreateRequest("Guest-01", sampleCount: 3);
await service.ApplySpeakerOverrideAsync(
request,
"Guest-01",
"Sabrina",
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Equal("Sabrina", saved.CanonicalName);
Assert.Equal(3, saved.VoiceVectors.Count);
Assert.Empty(saved.Snippets);
Assert.Single(saved.References);
Assert.Equal(3, fixture.Encoder.Requests.Single().Count);
}
[Fact]
public async Task FinalProcessingLearnsUnmatchedSpeakerFromFiveVectorsAndAttendees()
{
await using var fixture = await Fixture.CreateAsync();
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["John", "Mike"]),
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Null(saved.CanonicalName);
Assert.Equal(["John", "Mike"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
Assert.Equal(5, saved.VoiceVectors.Count);
Assert.Empty(saved.Snippets);
Assert.Single(saved.References);
}
[Fact]
public async Task FiveVectorThresholdDoesNotCapQualifyingCurrentRunEvidence()
{
await using var fixture = await Fixture.CreateAsync();
fixture.Encoder.Vectors = Enumerable.Range(0, 8)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest-01", sampleCount: 8, attendees: ["John", "Mike"]),
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Equal(8, saved.VoiceVectors.Count);
Assert.Equal(8, fixture.Encoder.Requests.Single().Count);
}
[Fact]
public async Task FinalProcessingPrunesForeignVectorsFromNewIdentityAtMinimum()
{
await using var fixture = await Fixture.CreateAsync(options =>
{
options.OutlierPruningMinimumVectors = 20;
options.OutlierPruningNeighborSimilarity = 0.90;
options.OutlierPruningMinimumNeighbors = 3;
options.OutlierPruningMinimumClusterRatio = 0.60;
});
fixture.Encoder.Vectors = Enumerable.Range(0, 16)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest-01", sampleCount: 20, attendees: ["John", "Mike"]),
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Equal(16, saved.VoiceVectors.Count);
Assert.All(
saved.VoiceVectors,
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task FinalProcessingStoresVectorsCollectedAfterLiveMatch(bool transcriptAlreadyRelabeled)
{
await using var fixture = await Fixture.CreateAsync();
var identity = await fixture.AddIdentityAsync(
"Chris",
[UnitVector(0), UnitVector(0, 20, 0.02f)]);
var currentRunVectors = Enumerable.Range(0, 8)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
fixture.Encoder.Vectors = currentRunVectors.Take(5).ToList();
var service = fixture.CreateService();
var liveResult = await service.IdentifyKnownSpeakersAsync(
fixture.CreateRequest("Guest03", sampleCount: 5),
CancellationToken.None);
fixture.Encoder.Vectors = currentRunVectors;
var finalRequest = fixture.CreateRequest("Guest03", sampleCount: 8) with
{
KnownSpeakerMappings = liveResult.SpeakerMappings
};
if (transcriptAlreadyRelabeled)
{
finalRequest = finalRequest with { Segments = liveResult.Segments };
}
await service.ProcessFinishedTranscriptAsync(finalRequest, CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Equal(10, saved.VoiceVectors.Count);
Assert.Equal([5, 8], fixture.Encoder.Requests.Select(request => request.Count));
}
[Fact]
public async Task RepeatedMatchDeduplicatesVectorsAndHonorsConfiguredLimit()
{
await using var fixture = await Fixture.CreateAsync(options => options.MaxVectorsPerIdentity = 6);
var identity = await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
fixture.Encoder.Vectors =
[
UnitVector(0),
UnitVector(0, 2, 0.01f),
UnitVector(0, 3, 0.02f),
UnitVector(0, 4, 0.03f),
UnitVector(0, 5, 0.04f)
];
var service = fixture.CreateService();
var request = fixture.CreateRequest("Guest03", sampleCount: 5);
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
await service.IdentifyKnownSpeakersAsync(request, CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Equal(6, saved.VoiceVectors.Count);
Assert.Single(saved.References);
}
[Fact]
public async Task FinishedMatchingExtractsFiveNonOverlappingSamplesWhenLiveSamplesAreMissing()
{
await using var fixture = await Fixture.CreateAsync();
await fixture.AddIdentityAsync("Chris", [UnitVector(0), UnitVector(0, 20, 0.02f)]);
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
var result = await service.IdentifyFinishedSpeakersAsync(
fixture.CreateRequest("Guest03", sampleCount: 0, segmentCount: 5),
CancellationToken.None);
Assert.Equal("Chris", result.Segments[0].Speaker);
Assert.Equal(5, fixture.SnippetExtractor.Requests.Count);
Assert.All(
fixture.SnippetExtractor.Requests.Zip(fixture.SnippetExtractor.Requests.Skip(1)),
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
Assert.Equal(5, fixture.Encoder.Requests.Single().Count);
}
[Fact]
public async Task FinalMatchPromotesTheRemainingCandidateAndAuditsTheTranscript()
{
await using var fixture = await Fixture.CreateAsync();
var identity = await fixture.AddIdentityAsync(
null,
[UnitVector(0), UnitVector(0, 20, 0.02f)],
candidates: ["John", "Mike"]);
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
var result = await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "John", "Chris"]),
CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Equal("John", saved.CanonicalName);
Assert.Equal(["John"], saved.CandidateNames.Select(candidate => candidate.Name));
Assert.Equal("John", result.Segments.Single().Speaker);
Assert.All(saved.References, reference =>
Assert.Contains("Guest03 was identified as John", File.ReadAllText(reference.TranscriptPath)));
}
[Fact]
public async Task FinalMatchResetsCandidatesWhenAttendeesDoNotIntersect()
{
await using var fixture = await Fixture.CreateAsync();
var identity = await fixture.AddIdentityAsync(
null,
[UnitVector(0), UnitVector(0, 20, 0.02f)],
candidates: ["John", "Mike"]);
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: ["Jane", "Chris"]),
CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(identity.Id);
Assert.Null(saved.CanonicalName);
Assert.Equal(["Chris", "Jane"], saved.CandidateNames.Select(candidate => candidate.Name).Order());
}
[Fact]
public async Task FinalUnmatchedSpeakerWithOneCandidateIsAuditedWhenLearned()
{
await using var fixture = await Fixture.CreateAsync();
fixture.Encoder.Vectors = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, index + 2, 0.03f))
.ToList();
var service = fixture.CreateService();
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest("Guest-01", sampleCount: 5, attendees: ["Manuel"]),
CancellationToken.None);
var saved = await fixture.LoadOnlyIdentityAsync();
Assert.Equal("Manuel", saved.CanonicalName);
Assert.Contains("Guest-01 was identified as Manuel", File.ReadAllText(saved.References.Single().TranscriptPath));
}
[Fact]
public async Task CandidateLimitCountsOnlyIdentitiesWithCompatibleVectors()
{
await using var fixture = await Fixture.CreateAsync(
configureSpeaker: options => options.MaxMatchCandidates = 1);
await fixture.AddIdentityAsync("Legacy WAV identity", []);
await fixture.AddIdentityAsync("Chris", [UnitVector(0)]);
fixture.Encoder.Vectors = Enumerable.Repeat(UnitVector(0), 5).ToList();
var service = fixture.CreateService();
var result = await service.IdentifyKnownSpeakersAsync(
fixture.CreateRequest("Guest03", sampleCount: 5, attendees: []),
CancellationToken.None);
Assert.Equal("Chris", result.Segments.Single().Speaker);
}
[Fact]
public async Task SummaryOverrideMergePreservesAllEvidenceFromCurrentRunCandidate()
{
await using var fixture = await Fixture.CreateAsync();
var target = await fixture.AddIdentityAsync("Sabrina", [UnitVector(0)]);
var request = fixture.CreateRequest("Guest-01", sampleCount: 3, attendees: ["Sabrina"]);
await fixture.AddCurrentRunCandidateAsync(request, "Sabrina");
fixture.Encoder.Vectors = Enumerable.Range(0, 3)
.Select(index => UnitVector(0, index + 2, 0.02f))
.ToList();
var service = fixture.CreateService();
await service.ApplySpeakerOverrideAsync(
request,
"Guest-01",
"Sabrina",
CancellationToken.None);
var saved = await fixture.LoadIdentityAsync(target.Id);
Assert.Single(saved.Snippets);
Assert.Equal(5, saved.VoiceVectors.Count);
Assert.Contains(saved.VoiceVectors, vector => vector.ModelId == "older-model");
await using var context = new TestDbContextFactory(fixture.DatabasePath).CreateDbContext();
Assert.Single(await context.SpeakerIdentities.ToListAsync());
}
private static float[] UnitVector(
int primaryDimension,
int? secondaryDimension = null,
float secondaryValue = 0)
{
var vector = new float[256];
vector[primaryDimension] = 1;
if (secondaryDimension is { } dimension)
{
vector[dimension] = secondaryValue;
}
return vector;
}
private static byte[] ToBytes(float[] vector)
{
var bytes = new byte[vector.Length * sizeof(float)];
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
return bytes;
}
private sealed class Fixture : IAsyncDisposable
{
private readonly string directory;
private readonly string databasePath;
private readonly SpeakerIdentificationOptions speakerOptions;
private Fixture(string directory, string databasePath, SpeakerIdentificationOptions speakerOptions)
{
this.directory = directory;
this.databasePath = databasePath;
this.speakerOptions = speakerOptions;
}
public FakeEncoder Encoder { get; } = new();
public FakeSnippetExtractor SnippetExtractor { get; } = new();
public string DatabasePath => databasePath;
public static async Task<Fixture> CreateAsync(
Action<ResemblyzerSpeakerRecognitionOptions>? configure = null,
Action<SpeakerIdentificationOptions>? configureSpeaker = null)
{
var directory = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(directory);
var databasePath = Path.Combine(directory, "speaker-identities.db");
var speakerOptions = new SpeakerIdentificationOptions
{
DatabasePath = databasePath,
MatchBatchSize = 6,
MaxMatchCandidates = 100,
MatchIdentityActiveAge = TimeSpan.FromDays(365),
MinimumSampleSpeechDuration = TimeSpan.Zero,
Resemblyzer = new ResemblyzerSpeakerRecognitionOptions
{
Enabled = true,
RequiredVectorsPerSpeaker = 5,
MaxVectorsPerIdentity = 1000,
MinimumClusterCohesion = 0.75,
MinimumIdentitySimilarity = 0.75,
MinimumSimilarityMargin = 0.05,
ModelId = "resemblyzer-0.1.4-pretrained"
}
};
configure?.Invoke(speakerOptions.Resemblyzer);
configureSpeaker?.Invoke(speakerOptions);
await using var context = new SpeakerIdentityDbContext(
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath};Pooling=False")
.Options);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
return new Fixture(directory, databasePath, speakerOptions);
}
public ResemblyzerSpeakerIdentificationService CreateService()
{
var appOptions = new MeetingAssistantOptions { SpeakerIdentification = speakerOptions };
return new ResemblyzerSpeakerIdentificationService(
new TestDbContextFactory(databasePath),
SnippetExtractor,
Encoder,
new ResemblyzerVoiceClusterMatcher(
speakerOptions.Resemblyzer,
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
new ResemblyzerVoiceVectorOutlierPruner(
speakerOptions.Resemblyzer,
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
Options.Create(appOptions),
NullLogger<ResemblyzerSpeakerIdentificationService>.Instance);
}
public SpeakerIdentificationRequest CreateRequest(
string speaker,
int sampleCount,
IReadOnlyList<string>? attendees = null,
int segmentCount = 1)
{
var transcriptPath = Path.Combine(directory, "transcript.md");
File.WriteAllText(transcriptPath, "Transcript");
var segments = Enumerable.Range(0, segmentCount)
.Select(index => new TranscriptionSegment(
TimeSpan.FromSeconds(index * 30),
TimeSpan.FromSeconds((index + 1) * 30),
speaker,
"enough useful words for a speaker sample"))
.ToList();
var segment = segments[0];
return new SpeakerIdentificationRequest(
Path.Combine(directory, "meeting.wav"),
new MeetingNote(
Path.Combine(directory, "meeting.md"),
new MeetingNoteFrontmatter
{
Title = "Test",
Attendees = attendees?.ToList() ?? ["Chris"],
Transcript = transcriptPath,
AssistantContext = Path.Combine(directory, "context.md"),
Summary = Path.Combine(directory, "summary.md")
},
""),
segments,
Enumerable.Range(0, sampleCount)
.Select(index => new SpeakerAudioSample(speaker, segment, [(byte)(index + 1)], 100 - index))
.ToList());
}
public async Task<SpeakerIdentity> AddIdentityAsync(
string? name,
IReadOnlyList<float[]> vectors,
IReadOnlyList<string>? candidates = null,
IReadOnlyList<string>? aliases = null)
{
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
var now = DateTimeOffset.UtcNow;
var identity = new SpeakerIdentity
{
CanonicalName = name,
CreatedAt = now,
UpdatedAt = now,
CandidateNames = candidates?.Select(candidate => new SpeakerCandidateName { Name = candidate }).ToList() ?? [],
Aliases = aliases?.Select(alias => new SpeakerAlias { Name = alias }).ToList() ?? [],
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
{
ModelId = speakerOptions.Resemblyzer.ModelId,
Dimensions = 256,
VectorBytes = ToBytes(vector),
Fingerprint = $"known-{index}",
CreatedAt = now.AddMinutes(index)
}).ToList()
};
context.SpeakerIdentities.Add(identity);
await context.SaveChangesAsync();
return identity;
}
public async Task AddCurrentRunCandidateAsync(
SpeakerIdentificationRequest request,
string candidateName)
{
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
var now = DateTimeOffset.UtcNow;
context.SpeakerIdentities.Add(new SpeakerIdentity
{
CreatedAt = now,
UpdatedAt = now,
CandidateNames = [new SpeakerCandidateName { Name = candidateName }],
Snippets = [new SpeakerSnippet { WavBytes = [9, 8, 7], CreatedAt = now }],
VoiceVectors =
[
new SpeakerVoiceVector
{
ModelId = "older-model",
Dimensions = 256,
VectorBytes = ToBytes(UnitVector(1)),
Fingerprint = "older-model-vector",
CreatedAt = now
}
],
References =
[
SpeakerIdentityReferences.Create(
request.MeetingNote.Path,
request.MeetingNote.Frontmatter.Transcript,
now)
]
});
await context.SaveChangesAsync();
}
public async Task<SpeakerIdentity> LoadIdentityAsync(int id)
{
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
return await context.SpeakerIdentities
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Aliases)
.Include(identity => identity.References)
.SingleAsync(identity => identity.Id == id);
}
public async Task<SpeakerIdentity> LoadOnlyIdentityAsync()
{
await using var context = new TestDbContextFactory(databasePath).CreateDbContext();
return await context.SpeakerIdentities
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.References)
.SingleAsync();
}
public ValueTask DisposeAsync()
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, recursive: true);
}
return ValueTask.CompletedTask;
}
}
private sealed class FakeEncoder : IResemblyzerVoiceEncoder
{
public IReadOnlyList<float[]> Vectors { get; set; } = [];
public List<IReadOnlyList<byte[]>> Requests { get; } = [];
public Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken)
{
Requests.Add(wavSamples.Select(sample => sample.ToArray()).ToList());
return Task.FromResult<IReadOnlyList<float[]>>(Vectors.Take(wavSamples.Count).ToList());
}
public Task WarmUpAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
private sealed class FakeSnippetExtractor : ISpeakerSnippetExtractor
{
public List<IReadOnlyList<TranscriptionSegment>> Requests { get; } = [];
public Task<byte[]> ExtractSnippetAsync(
string audioPath,
IReadOnlyList<TranscriptionSegment> speakerSegments,
CancellationToken cancellationToken)
{
Requests.Add(speakerSegments.ToList());
return Task.FromResult<byte[]>([checked((byte)Requests.Count)]);
}
}
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
{
private readonly string databasePath;
public TestDbContextFactory(string databasePath)
{
this.databasePath = databasePath;
}
public SpeakerIdentityDbContext CreateDbContext()
{
return new SpeakerIdentityDbContext(
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath};Pooling=False")
.Options);
}
}
}
@@ -0,0 +1,174 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerSpeakerIdentityMergeServiceTests
{
[Fact]
public async Task MergeRecentIdentitiesRequiresTwoDisjointMatchingVectorClusters()
{
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(directory);
try
{
var databasePath = Path.Combine(directory, "identities.db");
var factory = new TestDbContextFactory(databasePath);
await using (var context = factory.CreateDbContext())
{
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
context.SpeakerIdentities.Add(CreateIdentity(
"Chris",
DateTimeOffset.UtcNow.AddMonths(-2),
Enumerable.Range(0, 10)
.Select(index => UnitVector(0, index + 20, 0.02f))
.ToList(),
directory));
context.SpeakerIdentities.Add(CreateIdentity(
"Chris duplicate",
DateTimeOffset.UtcNow.AddDays(-1),
Enumerable.Range(0, 10)
.Select(index => UnitVector(0, index + 2, 0.02f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 40, 0.02f)))
.ToList(),
directory));
context.SpeakerIdentities.Add(CreateIdentity(
"Decoy",
DateTimeOffset.UtcNow.AddMonths(-1),
[UnitVector(0)],
directory));
await context.SaveChangesAsync();
}
var resemblyzerOptions = new ResemblyzerSpeakerRecognitionOptions
{
Enabled = true,
RequiredVectorsPerSpeaker = 5,
MaxVectorsPerIdentity = 1000,
MinimumClusterCohesion = 0.75,
MinimumIdentitySimilarity = 0.75,
MinimumSimilarityMargin = 0.05,
ModelId = "resemblyzer-0.1.4-pretrained"
};
var service = new ResemblyzerSpeakerIdentityMergeService(
factory,
new ResemblyzerVoiceClusterMatcher(
resemblyzerOptions,
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance),
new ResemblyzerVoiceVectorOutlierPruner(
resemblyzerOptions,
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
Options.Create(new MeetingAssistantOptions
{
SpeakerIdentification = new SpeakerIdentificationOptions
{
MergeRecentIdentityAge = TimeSpan.FromDays(14),
MaxMatchCandidates = 1,
MaxSnippetsPerSpeaker = 3,
Resemblyzer = resemblyzerOptions
}
}),
NullLogger<ResemblyzerSpeakerIdentityMergeService>.Instance);
var result = await service.MergeRecentIdentitiesAsync(TimeSpan.FromDays(14), CancellationToken.None);
Assert.Equal(2, result.MatchAttempts);
Assert.Equal(1, result.MergedPairs);
await using var verification = factory.CreateDbContext();
var saved = await verification.SpeakerIdentities
.Include(identity => identity.Aliases)
.Include(identity => identity.VoiceVectors)
.SingleAsync(identity => identity.CanonicalName == "Chris");
Assert.Equal(2, await verification.SpeakerIdentities.CountAsync());
Assert.Equal("Chris", saved.CanonicalName);
Assert.Contains(saved.Aliases, alias => alias.Name == "Chris duplicate");
Assert.Equal(20, saved.VoiceVectors.Count);
Assert.All(saved.VoiceVectors, vector => Assert.True(ToVector(vector.VectorBytes)[0] > 0.9f));
}
finally
{
Directory.Delete(directory, recursive: true);
}
}
private static SpeakerIdentity CreateIdentity(
string name,
DateTimeOffset createdAt,
IReadOnlyList<float[]> vectors,
string directory)
{
var transcriptPath = Path.Combine(directory, $"{Guid.NewGuid():N}.md");
File.WriteAllText(transcriptPath, "Transcript");
return new SpeakerIdentity
{
CanonicalName = name,
CreatedAt = createdAt,
UpdatedAt = createdAt,
VoiceVectors = vectors.Select((vector, index) => new SpeakerVoiceVector
{
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = ToBytes(vector),
Fingerprint = $"{name}-{index}",
CreatedAt = createdAt.AddMinutes(index)
}).ToList(),
References =
[
new SpeakerIdentityReference
{
MeetingNotePath = Path.Combine(directory, $"{Guid.NewGuid():N}.md"),
TranscriptPath = transcriptPath,
CreatedAt = createdAt
}
]
};
}
private static float[] UnitVector(int primary, int? secondary = null, float secondaryValue = 0)
{
var vector = new float[256];
vector[primary] = 1;
if (secondary is { } index)
{
vector[index] = secondaryValue;
}
return vector;
}
private static byte[] ToBytes(float[] vector)
{
var bytes = new byte[vector.Length * sizeof(float)];
Buffer.BlockCopy(vector, 0, bytes, 0, bytes.Length);
return bytes;
}
private static float[] ToVector(byte[] bytes)
{
var vector = new float[bytes.Length / sizeof(float)];
Buffer.BlockCopy(bytes, 0, vector, 0, bytes.Length);
return vector;
}
private sealed class TestDbContextFactory : IDbContextFactory<SpeakerIdentityDbContext>
{
private readonly string databasePath;
public TestDbContextFactory(string databasePath)
{
this.databasePath = databasePath;
}
public SpeakerIdentityDbContext CreateDbContext()
{
return new SpeakerIdentityDbContext(
new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath};Pooling=False")
.Options);
}
}
}
@@ -0,0 +1,122 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using Microsoft.Extensions.Logging.Abstractions;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerVoiceClusterMatcherTests
{
[Fact]
public void CoherentSimilarClusterSelectsUnambiguousIdentity()
{
var matcher = CreateMatcher();
var query = Enumerable.Range(0, 5)
.Select(index => UnitVector(0, secondaryDimension: index + 2, secondaryValue: 0.05f))
.ToList();
var candidates = new[]
{
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0), UnitVector(0, 10, 0.02f)]),
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(1), UnitVector(1, 11, 0.02f)])
};
var result = matcher.Match(query, candidates);
Assert.True(result.Accepted);
Assert.Equal(42, result.IdentityId);
Assert.True(result.Cohesion >= 0.99);
Assert.True(result.BestSimilarity >= 0.99);
Assert.True(result.RunnerUpSimilarity < 0.1);
}
[Fact]
public void FourVectorsDoNotTriggerAutomaticMatching()
{
var result = CreateMatcher().Match(
Enumerable.Repeat(UnitVector(0), 4).ToList(),
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
Assert.False(result.Accepted);
Assert.Contains("4/5", result.Reason);
}
[Fact]
public void IncoherentClusterIsRejectedBeforeIdentityScoring()
{
var result = CreateMatcher().Match(
Enumerable.Range(0, 5).Select(index => UnitVector(index)).ToList(),
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])]);
Assert.False(result.Accepted);
Assert.Contains("cohesion", result.Reason);
Assert.Null(result.BestSimilarity);
}
[Fact]
public void SimilarityBelowThresholdIsRejected()
{
var result = CreateMatcher().Match(
Enumerable.Repeat(UnitVector(0), 5).ToList(),
[new ResemblyzerVoiceVectorCandidate(42, [UnitVector(1)])]);
Assert.False(result.Accepted);
Assert.Contains("best similarity", result.Reason);
Assert.Equal(0, result.BestSimilarity);
}
[Fact]
public void SimilarCandidatesWithinMarginAreRejected()
{
var result = CreateMatcher().Match(
Enumerable.Repeat(UnitVector(0), 5).ToList(),
[
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)]),
new ResemblyzerVoiceVectorCandidate(77, [UnitVector(0, 1, 0.01f)])
]);
Assert.False(result.Accepted);
Assert.Contains("margin", result.Reason);
Assert.NotNull(result.RunnerUpSimilarity);
}
[Fact]
public void InvalidStoredCandidateDoesNotAbortScoringOtherIdentities()
{
var result = CreateMatcher().Match(
Enumerable.Repeat(UnitVector(0), 5).ToList(),
[
new ResemblyzerVoiceVectorCandidate(13, [new float[256]]),
new ResemblyzerVoiceVectorCandidate(42, [UnitVector(0)])
]);
Assert.True(result.Accepted);
Assert.Equal(42, result.IdentityId);
}
private static ResemblyzerVoiceClusterMatcher CreateMatcher()
{
return new ResemblyzerVoiceClusterMatcher(
new ResemblyzerSpeakerRecognitionOptions
{
RequiredVectorsPerSpeaker = 5,
MinimumClusterCohesion = 0.75,
MinimumIdentitySimilarity = 0.75,
MinimumSimilarityMargin = 0.05
},
NullLogger<ResemblyzerVoiceClusterMatcher>.Instance);
}
private static float[] UnitVector(
int primaryDimension,
int? secondaryDimension = null,
float secondaryValue = 0)
{
var vector = new float[256];
vector[primaryDimension] = 1;
if (secondaryDimension is { } dimension)
{
vector[dimension] = secondaryValue;
}
return vector;
}
}
@@ -0,0 +1,141 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using Microsoft.Extensions.Logging.Abstractions;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerVoiceVectorOutlierPrunerTests
{
[Fact]
public void DominantDensityClusterRemovesForeignSpeakerVectorsAtThreshold()
{
var identity = IdentityWithVectors(
Enumerable.Range(0, 16)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList());
var pruner = CreatePruner();
var result = pruner.Prune(identity);
Assert.Equal(4, result);
Assert.Equal(16, identity.VoiceVectors.Count);
Assert.All(
identity.VoiceVectors,
stored => Assert.True(SpeakerVoiceVectors.Decode(stored)[0] > 0.9f));
}
[Fact]
public void AmbiguousDenseClustersPreserveAllVectors()
{
var identity = IdentityWithVectors(
Enumerable.Range(0, 10)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 10)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList());
var pruner = CreatePruner();
var result = pruner.Prune(identity);
Assert.Equal(0, result);
Assert.Equal(20, identity.VoiceVectors.Count);
}
[Fact]
public void BelowMinimumVectorCountPreservesAllVectorsWithoutEvaluation()
{
var identity = IdentityWithVectors(
Enumerable.Range(0, 15)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList());
var pruner = CreatePruner();
var result = pruner.Prune(identity);
Assert.Equal(0, result);
Assert.Equal(19, identity.VoiceVectors.Count);
}
[Fact]
public void PruningPreservesOtherModelsAndMalformedStoredRows()
{
var identity = IdentityWithVectors(
Enumerable.Range(0, 16)
.Select(index => UnitVector(0, index + 2, 0.04f))
.Concat(Enumerable.Range(0, 4)
.Select(index => UnitVector(1, index + 30, 0.04f)))
.ToList());
SpeakerVoiceVectors.AddDistinct(
identity,
[UnitVector(2, 60, 0.04f)],
"older-model",
1000,
DateTimeOffset.UtcNow);
identity.VoiceVectors.Add(new SpeakerVoiceVector
{
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = [1],
Fingerprint = "malformed",
CreatedAt = DateTimeOffset.UtcNow
});
identity.VoiceVectors.Add(new SpeakerVoiceVector
{
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = new byte[256 * sizeof(float)],
Fingerprint = "zero-magnitude",
CreatedAt = DateTimeOffset.UtcNow
});
var pruner = CreatePruner();
var result = pruner.Prune(identity);
Assert.Equal(4, result);
Assert.Equal(19, identity.VoiceVectors.Count);
Assert.Contains(identity.VoiceVectors, vector => vector.ModelId == "older-model");
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "malformed");
Assert.Contains(identity.VoiceVectors, vector => vector.Fingerprint == "zero-magnitude");
}
private static ResemblyzerVoiceVectorOutlierPruner CreatePruner()
{
return new ResemblyzerVoiceVectorOutlierPruner(
new ResemblyzerSpeakerRecognitionOptions
{
ModelId = "resemblyzer-0.1.4-pretrained",
OutlierPruningMinimumVectors = 20,
OutlierPruningNeighborSimilarity = 0.90,
OutlierPruningMinimumNeighbors = 3,
OutlierPruningMinimumClusterRatio = 0.60
},
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance);
}
private static SpeakerIdentity IdentityWithVectors(IReadOnlyList<float[]> vectors)
{
var identity = new SpeakerIdentity();
SpeakerVoiceVectors.AddDistinct(
identity,
vectors,
"resemblyzer-0.1.4-pretrained",
1000,
DateTimeOffset.UtcNow);
return identity;
}
private static float[] UnitVector(
int primaryDimension,
int secondaryDimension,
float secondaryValue)
{
var vector = new float[256];
vector[primaryDimension] = 1;
vector[secondaryDimension] = secondaryValue;
return vector;
}
}
@@ -0,0 +1,78 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Tests;
public sealed class ResemblyzerWarmupHostedServiceTests
{
[Fact]
public async Task EnabledWarmupStartsWithoutBlockingApplicationStartup()
{
var encoder = new BlockingEncoder();
var options = new MeetingAssistantOptions();
options.SpeakerIdentification.Resemblyzer.Enabled = true;
var service = new ResemblyzerWarmupHostedService(
encoder,
Options.Create(options),
NullLogger<ResemblyzerWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(1));
await encoder.WaitForWarmupAsync();
await service.StopAsync(CancellationToken.None);
Assert.True(encoder.CancellationObserved);
}
[Fact]
public async Task DisabledWarmupDoesNotInvokeEncoder()
{
var encoder = new BlockingEncoder();
var service = new ResemblyzerWarmupHostedService(
encoder,
Options.Create(new MeetingAssistantOptions()),
NullLogger<ResemblyzerWarmupHostedService>.Instance);
await service.StartAsync(CancellationToken.None);
await service.StopAsync(CancellationToken.None);
Assert.False(encoder.WarmupStarted);
}
private sealed class BlockingEncoder : IResemblyzerVoiceEncoder
{
private readonly TaskCompletionSource started = new(TaskCreationOptions.RunContinuationsAsynchronously);
public bool WarmupStarted { get; private set; }
public bool CancellationObserved { get; private set; }
public Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken)
{
return Task.FromResult<IReadOnlyList<float[]>>([]);
}
public async Task WarmUpAsync(CancellationToken cancellationToken)
{
WarmupStarted = true;
started.TrySetResult();
try
{
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
CancellationObserved = true;
}
}
public Task WaitForWarmupAsync()
{
return started.Task.WaitAsync(TimeSpan.FromSeconds(1));
}
}
}
@@ -9,5 +9,6 @@ public sealed class ScreenshotOcrOptionsTests
{
Assert.Contains("exactly who is in the meeting", ScreenshotOcrOptions.DefaultPrompt);
Assert.Contains("partial", ScreenshotOcrOptions.DefaultPrompt);
Assert.Contains("\"attendees\"", ScreenshotOcrOptions.DefaultPrompt);
}
}
@@ -8,13 +8,22 @@ namespace MeetingAssistant.Tests;
public sealed class SpeakerAudioSampleCollectorTests
{
[Fact]
public void CollectorWaitsForConfiguredUninterruptedSpeechDuration()
public void DefaultSampleDurationsAreTenAndSixtySeconds()
{
var options = new SpeakerIdentificationOptions();
Assert.Equal(TimeSpan.FromSeconds(10), options.MinimumSampleSpeechDuration);
Assert.Equal(TimeSpan.FromSeconds(60), options.MaximumSampleDuration);
}
[Fact]
public void CollectorWaitsForConfiguredSpeakerAudioDuration()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
maximumSegmentGap: TimeSpan.FromSeconds(1));
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
maximumSampleDuration: TimeSpan.FromSeconds(60));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
collector.TryAdd(Segment(0, 12, "Guest01", "one two three four five."));
@@ -37,8 +46,8 @@ public sealed class SpeakerAudioSampleCollectorTests
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
maximumSegmentGap: TimeSpan.FromSeconds(1));
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
maximumSampleDuration: TimeSpan.FromSeconds(60));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(50)));
collector.TryAdd(Segment(0, 20, "Guest01", "one two three four five."));
@@ -48,6 +57,61 @@ public sealed class SpeakerAudioSampleCollectorTests
Assert.DoesNotContain(collector.Snapshot(), sample => sample.Speaker == "Guest01");
}
[Fact]
public void CollectorCombinesConsecutiveSameSpeakerSegmentsAcrossProviderPauses()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
maximumSampleDuration: TimeSpan.FromSeconds(60));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(20)));
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
collector.TryAdd(Segment(7, 13, "Guest01", "five six seven eight."));
var sample = Assert.Single(collector.Snapshot());
Assert.Equal(TimeSpan.Zero, sample.Segment.Start);
Assert.Equal(TimeSpan.FromSeconds(13), sample.Segment.End);
}
[Fact]
public void CollectorDoesNotCountProviderPausesAsSpeakerAudio()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
maximumSampleDuration: TimeSpan.FromSeconds(60));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(30)));
collector.TryAdd(Segment(0, 4, "Guest01", "one two three four."));
collector.TryAdd(Segment(20, 25, "Guest01", "five six seven eight."));
Assert.Empty(collector.Snapshot());
}
[Fact]
public void CollectorCapsRecognitionWavAtConfiguredMaximumDuration()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(10),
maximumSampleDuration: TimeSpan.FromSeconds(60));
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(75)));
var sample = collector.TryAdd(Segment(
0,
70,
"Guest01",
"one two three four five six seven eight nine ten."));
Assert.NotNull(sample);
Assert.Equal(TimeSpan.FromSeconds(60), sample.Segment.End);
Assert.True(ReadDuration(sample.WavBytes) <= TimeSpan.FromSeconds(60));
}
[Fact]
public void CollectorLogsWhenSampleIsDiscardedBecauseSpeechIsTooShort()
{
@@ -55,8 +119,8 @@ public sealed class SpeakerAudioSampleCollectorTests
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
maximumSegmentGap: TimeSpan.FromSeconds(1),
minimumSampleSpeechDuration: TimeSpan.FromSeconds(30),
maximumSampleDuration: TimeSpan.FromSeconds(60),
logger: logger);
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
@@ -69,6 +133,47 @@ public sealed class SpeakerAudioSampleCollectorTests
Assert.Contains("minimum duration", message);
}
[Fact]
public void CollectorCanResetThePendingSpanAfterEachAcceptedSample()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 5,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
maximumSampleDuration: TimeSpan.FromSeconds(60),
requireNonOverlappingSamples: true);
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(6)));
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
collector.TryAdd(Segment(2, 4, "Guest01", "four five six."));
collector.TryAdd(Segment(4, 6, "Guest01", "seven eight nine."));
var samples = collector.Snapshot().OrderBy(sample => sample.Segment.Start).ToList();
Assert.Collection(
samples,
sample => Assert.Equal((TimeSpan.Zero, TimeSpan.FromSeconds(2)), (sample.Segment.Start, sample.Segment.End)),
sample => Assert.Equal((TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4)), (sample.Segment.Start, sample.Segment.End)),
sample => Assert.Equal((TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(6)), (sample.Segment.Start, sample.Segment.End)));
}
[Fact]
public void ResettingCollectorRejectsASegmentThatOverlapsAnAcceptedSample()
{
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 5,
minimumSampleSpeechDuration: TimeSpan.FromSeconds(2),
maximumSampleDuration: TimeSpan.FromSeconds(60),
requireNonOverlappingSamples: true);
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(4)));
collector.TryAdd(Segment(0, 2, "Guest01", "one two three."));
collector.TryAdd(Segment(1, 3, "Guest01", "four five six."));
Assert.Single(collector.Snapshot());
}
private static TranscriptionSegment Segment(
double start,
double end,
@@ -107,6 +107,9 @@ public sealed class SpeakerIdentityMergeServiceTests
return new SpeakerIdentityMergeService(
new TestSpeakerIdentityDbContextFactory(dbPath),
Matcher,
new ResemblyzerVoiceVectorOutlierPruner(
new ResemblyzerSpeakerRecognitionOptions(),
NullLogger<ResemblyzerVoiceVectorOutlierPruner>.Instance),
Options.Create(new MeetingAssistantOptions
{
SpeakerIdentification = new SpeakerIdentificationOptions
@@ -0,0 +1,47 @@
using MeetingAssistant.Speakers;
namespace MeetingAssistant.Tests;
public sealed class SpeakerIdentityMergerTests
{
[Fact]
public void MergeIntoRetainsNewestDistinctVoiceVectorsUpToConfiguredLimit()
{
var now = DateTimeOffset.UtcNow;
var target = new SpeakerIdentity
{
CanonicalName = "Chris",
VoiceVectors =
[
Vector("oldest", 1, now.AddMinutes(-4)),
Vector("duplicate", 2, now.AddMinutes(-3))
]
};
var source = new SpeakerIdentity
{
VoiceVectors =
[
Vector("duplicate", 2, now.AddMinutes(-2)),
Vector("newer", 3, now.AddMinutes(-1)),
Vector("newest", 4, now)
]
};
SpeakerIdentityMerger.MergeInto(target, source, maxSnippets: 3, maxVoiceVectors: 3);
Assert.Equal(["duplicate", "newer", "newest"], target.VoiceVectors.Select(vector => vector.Fingerprint).Order());
Assert.All(target.VoiceVectors, vector => Assert.Same(target, vector.SpeakerIdentity));
}
private static SpeakerVoiceVector Vector(string fingerprint, byte value, DateTimeOffset createdAt)
{
return new SpeakerVoiceVector
{
ModelId = "model",
Dimensions = 256,
VectorBytes = Enumerable.Repeat(value, 256 * sizeof(float)).ToArray(),
Fingerprint = fingerprint,
CreatedAt = createdAt
};
}
}
@@ -0,0 +1,80 @@
using MeetingAssistant.Speakers;
using Microsoft.Data.Sqlite;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
namespace MeetingAssistant.Tests;
public sealed class SpeakerRecognitionFeatureSelectionTests
{
[Theory]
[InlineData(false, typeof(SpeakerIdentityService), typeof(SpeakerIdentityMergeService), 1, false)]
[InlineData(true, typeof(ResemblyzerSpeakerIdentificationService), typeof(ResemblyzerSpeakerIdentityMergeService), 1000, true)]
public async Task FeatureFlagSelectsExactlyOneIdentificationBackend(
bool enabled,
Type expectedIdentificationType,
Type expectedMergeType,
int expectedMinimumSamples,
bool expectedNonOverlappingSamples)
{
var directory = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(directory);
try
{
await using var factory = new WebApplicationFactory<Program>()
.WithWebHostBuilder(builder =>
{
builder.ConfigureAppConfiguration((_, configuration) =>
{
configuration.AddInMemoryCollection(new Dictionary<string, string?>
{
["MeetingAssistant:FunAsr:Backend:Enabled"] = "false",
["MeetingAssistant:SpeakerIdentification:DatabasePath"] = Path.Combine(directory, "identities.db"),
["MeetingAssistant:SpeakerIdentification:PyannoteValidation:Enabled"] = "false",
["MeetingAssistant:SpeakerIdentification:Resemblyzer:Enabled"] = enabled.ToString()
});
});
builder.ConfigureTestServices(services =>
{
services.RemoveAll<IResemblyzerVoiceEncoder>();
services.AddSingleton<IResemblyzerVoiceEncoder, NoopEncoder>();
});
});
var service = factory.Services.GetRequiredService<ISpeakerIdentificationService>();
Assert.IsType(expectedIdentificationType, service);
Assert.Single(factory.Services.GetServices<ISpeakerIdentificationService>());
var mergeService = factory.Services.GetRequiredService<ISpeakerIdentityMergeService>();
Assert.IsType(expectedMergeType, mergeService);
Assert.Single(factory.Services.GetServices<ISpeakerIdentityMergeService>());
var policy = factory.Services.GetRequiredService<SpeakerSampleCollectionPolicy>();
Assert.Equal(expectedMinimumSamples, policy.MinimumRetainedSamples);
Assert.Equal(expectedNonOverlappingSamples, policy.RequireNonOverlappingSamples);
}
finally
{
SqliteConnection.ClearAllPools();
Directory.Delete(directory, recursive: true);
}
}
private sealed class NoopEncoder : IResemblyzerVoiceEncoder
{
public Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken)
{
return Task.FromResult<IReadOnlyList<float[]>>([]);
}
public Task WarmUpAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
}
@@ -0,0 +1,142 @@
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
namespace MeetingAssistant.Tests;
public sealed class SpeakerSampleSpanSelectorTests
{
[Fact]
public void SelectBestSameSpeakerSpanCombinesLinesAndCapsTheClip()
{
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
[
Segment(0, 4),
Segment(7, 13),
Segment(20, 70)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60));
Assert.Equal(3, span.Count);
Assert.Equal(TimeSpan.Zero, span[0].Start);
Assert.Equal(TimeSpan.FromSeconds(60), span[^1].End);
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
}
[Fact]
public void SelectBestSameSpeakerSpanDoesNotCountProviderPausesAsSpeakerAudio()
{
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
[
Segment(0, 4),
Segment(20, 25)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60));
Assert.Empty(span);
}
[Fact]
public void SelectBestSameSpeakerSpansReturnsDistinctMinimumLengthSamples()
{
var segments = Enumerable.Range(0, 6)
.Select(index => new TranscriptionSegment(
TimeSpan.FromSeconds(index * 10),
TimeSpan.FromSeconds((index + 1) * 10),
"Guest01",
$"segment {index} has enough words"))
.ToList();
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
segments,
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
Assert.Equal(5, spans.Count);
Assert.All(spans, span => Assert.True(SpeakerSampleSpanSelector.SpanDuration(span) >= TimeSpan.FromSeconds(10)));
Assert.All(
spans.Zip(spans.Skip(1)),
pair => Assert.True(pair.First[^1].End <= pair.Second[0].Start));
}
[Fact]
public void SelectBestSameSpeakerSpansSkipsSegmentsThatOverlapACompletedSample()
{
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
[
Segment(0, 10),
Segment(9, 19),
Segment(20, 30)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
Assert.Equal(2, spans.Count);
Assert.Equal(TimeSpan.Zero, spans[0][0].Start);
Assert.Equal(TimeSpan.FromSeconds(20), spans[1][0].Start);
}
[Fact]
public void SelectBestSameSpeakerSpansCombinesConsecutiveLinesAcrossProviderPauses()
{
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
[
Segment(0, 4),
Segment(7, 13)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
var span = Assert.Single(spans);
Assert.Equal(TimeSpan.Zero, span[0].Start);
Assert.Equal(TimeSpan.FromSeconds(13), span[^1].End);
}
[Fact]
public void SelectBestSameSpeakerSpansDoesNotCountProviderPausesAsSpeakerAudio()
{
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
[
Segment(0, 4),
Segment(20, 25)
],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
Assert.Empty(spans);
}
[Fact]
public void SelectBestSameSpeakerSpansCapsFinalizedSamplesAtConfiguredMaximumDuration()
{
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
[Segment(0, 70)],
"Guest01",
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(60),
maxSpans: 5);
var span = Assert.Single(spans);
Assert.Equal(TimeSpan.FromSeconds(60), SpeakerSampleSpanSelector.SpanDuration(span));
}
private static TranscriptionSegment Segment(double start, double end)
{
return new TranscriptionSegment(
TimeSpan.FromSeconds(start),
TimeSpan.FromSeconds(end),
"Guest01",
"enough words for this segment");
}
}
@@ -0,0 +1,106 @@
using MeetingAssistant.Speakers;
using Microsoft.EntityFrameworkCore;
namespace MeetingAssistant.Tests;
public sealed class SpeakerVoiceVectorPersistenceTests
{
[Fact]
public async Task IdentityStoresVersionedVoiceVectorAndDeletesItWithIdentity()
{
var databasePath = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"speaker-identities.db");
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath}")
.Options;
await using (var context = new SpeakerIdentityDbContext(options))
{
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
context.SpeakerIdentities.Add(new SpeakerIdentity
{
CanonicalName = "Chris",
CreatedAt = DateTimeOffset.UtcNow,
UpdatedAt = DateTimeOffset.UtcNow,
VoiceVectors =
[
new SpeakerVoiceVector
{
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = new byte[256 * sizeof(float)],
Fingerprint = "vector-a",
CreatedAt = DateTimeOffset.UtcNow
}
]
});
await context.SaveChangesAsync();
}
await using (var context = new SpeakerIdentityDbContext(options))
{
var identity = await context.SpeakerIdentities
.Include(candidate => candidate.VoiceVectors)
.SingleAsync();
var vector = Assert.Single(identity.VoiceVectors);
Assert.Equal("resemblyzer-0.1.4-pretrained", vector.ModelId);
Assert.Equal(256, vector.Dimensions);
Assert.Equal(256 * sizeof(float), vector.VectorBytes.Length);
context.SpeakerIdentities.Remove(identity);
await context.SaveChangesAsync();
}
await using (var context = new SpeakerIdentityDbContext(options))
{
Assert.Empty(await context.SpeakerVoiceVectors.ToListAsync());
}
}
[Fact]
public async Task IdentityRejectsDuplicateVoiceVectorFingerprint()
{
var databasePath = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"speaker-identities.db");
Directory.CreateDirectory(Path.GetDirectoryName(databasePath)!);
var options = new DbContextOptionsBuilder<SpeakerIdentityDbContext>()
.UseSqlite($"Data Source={databasePath}")
.Options;
await using var context = new SpeakerIdentityDbContext(options);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, CancellationToken.None);
var identity = new SpeakerIdentity
{
CanonicalName = "Chris",
CreatedAt = DateTimeOffset.UtcNow,
UpdatedAt = DateTimeOffset.UtcNow
};
context.SpeakerIdentities.Add(identity);
await context.SaveChangesAsync();
context.SpeakerVoiceVectors.AddRange(
CreateVector(identity.Id, "same-vector"),
CreateVector(identity.Id, "same-vector"));
await Assert.ThrowsAsync<DbUpdateException>(() => context.SaveChangesAsync());
}
private static SpeakerVoiceVector CreateVector(int identityId, string fingerprint)
{
return new SpeakerVoiceVector
{
SpeakerIdentityId = identityId,
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = new byte[256 * sizeof(float)],
Fingerprint = fingerprint,
CreatedAt = DateTimeOffset.UtcNow
};
}
}
@@ -0,0 +1,209 @@
using System.Threading.Channels;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Notifications;
using MeetingAssistant.Summary;
using Microsoft.Extensions.Logging.Abstractions;
using MeetingAssistant.Workflow;
namespace MeetingAssistant.Tests;
public sealed class SummaryAgentRequestTests
{
[Theory]
[InlineData("denied")]
[InlineData("expired")]
[InlineData("shutdown")]
public async Task WorkflowDenialExpirationAndShutdownRemovePromptWithoutStartingAgent(string outcome)
{
var prompts = new ControlledPrompts();
var editor = new RecordingEditor();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance,
outcome == "expired" ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromMinutes(10));
using var requests = new WorkflowChangeRequestService(approvals, editor, NullLogger<WorkflowChangeRequestService>.Instance);
await requests.StartAsync(CancellationToken.None);
try
{
Assert.StartsWith("Requested", requests.Request("Add association rule", "Given Alpha, then associate Alpha.", "meeting.md"));
var prompt = await prompts.NextAsync();
if (outcome == "denied") prompt.Response.SetResult(false);
if (outcome == "shutdown") await requests.StopAsync(CancellationToken.None);
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
prompt.Response.TrySetResult(true);
}
finally { await requests.StopAsync(CancellationToken.None); }
Assert.Empty(editor.Prompts);
Assert.StartsWith("Refused", requests.Request("Too late", "Do something", "meeting.md"));
}
[Fact]
public async Task InvalidRequestsAreRefusedBeforeNotification()
{
var fixture = await CreateAsync();
var original = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
var prompts = new ControlledPrompts();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
using var requests = new WorkflowChangeRequestService(approvals, new RecordingEditor(), NullLogger<WorkflowChangeRequestService>.Instance);
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals, workflowRequests: requests);
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("../Alpha", "Relevant"));
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", new string('a', 101)));
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", "two\nlines"));
Assert.StartsWith("Refused", await tools.RequestProjectAssociation("Alpha", ""));
Assert.StartsWith("Refused", tools.RequestWorkflowChange(new string('a', 101), "Detailed scenario"));
Assert.StartsWith("Refused", tools.RequestWorkflowChange("Intent", " "));
Assert.StartsWith("Refused", tools.RequestWorkflowChange("two\nlines", "Detailed scenario"));
Assert.False(prompts.HasMore);
Assert.Equal(original, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
}
[Fact]
public async Task WorkflowRequestReturnsImmediatelyAndApprovalStartsItsDetailedPromptOnce()
{
var fixture = await CreateAsync();
var prompts = new ControlledPrompts();
var editor = new RecordingEditor();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
using var requests = new WorkflowChangeRequestService(approvals, editor, NullLogger<WorkflowChangeRequestService>.Instance);
await requests.StartAsync(CancellationToken.None);
try
{
const string detail = "Given a meeting with Alpha in its title, when metadata arrives, then add project Alpha.";
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, workflowRequests: requests);
var result = tools.RequestWorkflowChange("Associate Alpha meetings automatically", detail);
Assert.StartsWith("Requested", result);
var prompt = await prompts.NextAsync();
Assert.Contains("Workflow change requested: Associate Alpha meetings automatically", prompt.Prompt.Title);
Assert.Empty(editor.Prompts);
await tools.WriteSummary("The summary can finish while approval is pending.", "Summary finished");
Assert.True(tools.SummaryWasWritten);
prompt.Response.SetResult(true);
Assert.Equal(detail, await editor.Started.Task.WaitAsync(TimeSpan.FromSeconds(2)));
Assert.False(prompt.Response.TrySetResult(true));
Assert.Single(editor.Prompts);
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
}
finally { await requests.StopAsync(CancellationToken.None); }
}
[Theory]
[InlineData("denied")]
[InlineData("expired")]
[InlineData("canceled")]
[InlineData("removed")]
public async Task UnapprovedOrStaleAssociationNeverChangesNoteOrProjectAccess(string outcome)
{
var fixture = await CreateAsync();
var original = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
var prompts = new ControlledPrompts();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance,
outcome == "expired" ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromMinutes(10));
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals);
using var cancellation = new CancellationTokenSource();
var request = tools.RequestProjectAssociation("Alpha", "Relevant project", cancellation.Token);
var prompt = await prompts.NextAsync();
if (outcome == "denied") prompt.Response.SetResult(false);
if (outcome == "canceled") await cancellation.CancelAsync();
if (outcome == "removed")
{
Directory.Delete(Path.Combine(fixture.Options.Vault.ProjectsFolder, "Alpha"), recursive: true);
prompt.Response.SetResult(true);
}
if (outcome == "canceled")
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => request);
else
Assert.DoesNotContain("Approved", await request.WaitAsync(TimeSpan.FromSeconds(2)));
await prompt.Removed.Task.WaitAsync(TimeSpan.FromSeconds(2));
prompt.Response.TrySetResult(true);
Assert.Equal(original, await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath));
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
}
[Fact]
public async Task AssociationWaitsForApprovalThenPreservesLatestNoteAndUnlocksProjectWithInstructions()
{
var fixture = await CreateAsync();
var prompts = new ControlledPrompts();
var approvals = new AgentApprovalService(prompts, NullLogger<AgentApprovalService>.Instance);
var tools = new MeetingSummaryTools(fixture.Artifacts, fixture.Options, approvals: approvals);
var request = tools.RequestProjectAssociation("alpha", "Transcript discusses the Alpha launch");
var prompt = await prompts.NextAsync();
Assert.Contains("Alpha Platform", prompt.Prompt.Message);
Assert.Contains("Transcript discusses", prompt.Prompt.Message);
Assert.Equal(TimeSpan.FromMinutes(10), prompt.Prompt.Timeout);
Assert.False(request.IsCompleted);
Assert.Equal("", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
Assert.StartsWith("Refused:", await tools.WriteProjectFile("Alpha", "notes.md", "unauthorized"));
var latestNote = "---\ntitle: Edited by user\nprojects: Existing\ncustom: keep-me\n---\n\nUser's latest notes.";
await File.WriteAllTextAsync(fixture.Artifacts.MeetingNotePath, latestNote);
prompt.Response.SetResult(true);
var result = await request.WaitAsync(TimeSpan.FromSeconds(2));
Assert.StartsWith("Approved", result);
Assert.Contains("Alpha instructions", result);
Assert.Contains("Alpha Platform", result);
var note = await File.ReadAllTextAsync(fixture.Artifacts.MeetingNotePath);
Assert.Contains("Edited by user", note);
Assert.Contains("custom: keep-me", note);
Assert.Contains("User's latest notes.", note);
Assert.Contains("Existing", note);
Assert.Contains("Alpha", note);
Assert.Empty(Directory.EnumerateFiles(Path.GetDirectoryName(fixture.Artifacts.MeetingNotePath)!, "*.tmp"));
Assert.Equal("Alpha instructions", await tools.ReadProjectFile("Alpha", "AGENTS.md"));
Assert.Equal("Alpha/notes.md", await tools.WriteProjectFile("Alpha", "notes.md", "approved"));
Assert.Contains("notes.md:1 approved", await tools.Search("approved", ["Alpha"]));
Assert.StartsWith("Approved", await tools.RequestProjectAssociation("Alpha", "Already associated"));
Assert.False(prompts.HasMore);
}
private static async Task<(MeetingSessionArtifacts Artifacts, MeetingAssistantOptions Options)> CreateAsync()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var project = Path.Combine(root, "Projects", "Alpha");
Directory.CreateDirectory(project);
var artifacts = new MeetingSessionArtifacts(Path.Combine(root, "meeting.md"), Path.Combine(root, "transcript.md"),
Path.Combine(root, "context.md"), Path.Combine(root, "summary.md"));
await File.WriteAllTextAsync(artifacts.MeetingNotePath, "---\ntitle: Planning\nprojects: []\n---\nUser notes.");
await File.WriteAllTextAsync(Path.Combine(project, "PROJECT.md"),
"---\nname: Alpha Platform\ndescription: Meeting automation\n---");
await File.WriteAllTextAsync(Path.Combine(project, "AGENTS.md"), "Alpha instructions");
return (artifacts, new MeetingAssistantOptions { Vault = { ProjectsFolder = Path.Combine(root, "Projects") } });
}
private sealed class RecordingEditor : IWorkflowRulesEditorWindowService
{
public List<string> Prompts { get; } = [];
public TaskCompletionSource<string> Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public void Show() { }
public void ShowWithPrompt(string initialPrompt)
{
Prompts.Add(initialPrompt);
Started.TrySetResult(initialPrompt);
}
}
private sealed class ControlledPrompts : IAgentApprovalPromptService
{
private readonly Channel<PendingPrompt> requests = Channel.CreateUnbounded<PendingPrompt>();
public bool HasMore => requests.Reader.TryPeek(out _);
public Task<PendingPrompt> NextAsync() => requests.Reader.ReadAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2));
public async Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken)
{
var request = new PendingPrompt(prompt, cancellationToken);
requests.Writer.TryWrite(request);
try { return await request.Response.Task.WaitAsync(cancellationToken); }
finally { request.Removed.TrySetResult(); }
}
}
private sealed record PendingPrompt(AgentApprovalPrompt Prompt, CancellationToken Cancellation)
{
public TaskCompletionSource<bool> Response { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource Removed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
}
}
+110 -33
View File
@@ -13,46 +13,112 @@ public sealed class TaskbarIconTests
{
var menu = MeetingTaskbarMenuBuilder.Build(
Status(),
[Profile("default", "Ctrl+Alt+M"), Profile("english", "Ctrl+Alt+L")]);
[Profile("default", "Ctrl+Alt+M"), Profile("english", "Ctrl+Alt+L")],
[new MicrophoneDevice("integrated", "integrated microphone")],
"integrated");
Assert.Equal(RecordingProcessState.Idle, menu.State);
Assert.Contains(menu.Items, item =>
item.Action == MeetingTaskbarAction.EditRules &&
item.Text == "Settings and logs");
Assert.Contains(menu.Items, item =>
item.Action == MeetingTaskbarAction.StartRecording &&
item.ProfileName == "default" &&
item.Text == "Start meeting recording (default)\tCtrl+Alt+M");
Assert.Contains(menu.Items, item =>
item.Action == MeetingTaskbarAction.StartRecording &&
item.ProfileName == "english" &&
item.Text == "Start meeting recording (english)\tCtrl+Alt+L");
Assert.DoesNotContain(menu.Items, item => item.Action == MeetingTaskbarAction.StopRecording);
Assert.DoesNotContain(menu.Items, item => item.Action == MeetingTaskbarAction.AbortRecording);
AssertMenuLayout(
menu,
("Open agent", MeetingTaskbarAction.EditRules, false),
("Microphone", MeetingTaskbarAction.OpenSubmenu, true),
("Start meeting recording (default)\tCtrl+Alt+M", MeetingTaskbarAction.StartRecording, false),
("Start meeting recording (english)\tCtrl+Alt+L", MeetingTaskbarAction.StartRecording, false),
("Exit", MeetingTaskbarAction.Exit, true));
Assert.Equal(
["default", "english"],
menu.Items
.Where(item => item.Action == MeetingTaskbarAction.StartRecording)
.Select(item => item.ProfileName));
}
[Fact]
public void RecordingMenuOffersStopAbortAndOtherProfileSwitches()
public void MenuOffersMicrophoneSubmenuWithEffectiveDeviceChecked()
{
var menu = MeetingTaskbarMenuBuilder.Build(
Status(),
[Profile("default")],
[
new MicrophoneDevice("integrated", "integrated microphone"),
new MicrophoneDevice("other", "other microphone")
],
"integrated");
var microphoneMenu = Assert.Single(menu.Items, item => item.Text == "Microphone");
Assert.Equal(MeetingTaskbarAction.OpenSubmenu, microphoneMenu.Action);
Assert.NotNull(microphoneMenu.Items);
Assert.Contains(microphoneMenu.Items, item =>
item.Action == MeetingTaskbarAction.SelectMicrophone &&
item.MicrophoneDeviceId == "integrated" &&
item.Text == "integrated microphone" &&
item.IsChecked);
Assert.Contains(microphoneMenu.Items, item =>
item.Action == MeetingTaskbarAction.SelectMicrophone &&
item.MicrophoneDeviceId == "other" &&
item.Text == "other microphone" &&
!item.IsChecked);
}
[Fact]
public void RecordingMenuPrioritizesFinishMeetingInDedicatedSection()
{
var menu = MeetingTaskbarMenuBuilder.Build(
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
[Profile("default", "Ctrl+Alt+M"), Profile("english", "Ctrl+Alt+L"), Profile("french", "Ctrl+Alt+F")]);
[Profile("default", "Ctrl+Alt+M"), Profile("english", "Ctrl+Alt+L")],
[new MicrophoneDevice("integrated", "integrated microphone")],
"integrated");
Assert.Equal(RecordingProcessState.Recording, menu.State);
Assert.Contains(menu.Items, item => item.Action == MeetingTaskbarAction.StopRecording);
Assert.Contains(menu.Items, item => item.Action == MeetingTaskbarAction.AbortRecording);
Assert.Contains(menu.Items, item =>
item.Action == MeetingTaskbarAction.SwitchProfile &&
item.ProfileName == "english" &&
item.Text == "Switch to english\tCtrl+Alt+L");
Assert.Contains(menu.Items, item =>
item.Action == MeetingTaskbarAction.SwitchProfile &&
item.ProfileName == "french" &&
item.Text == "Switch to french\tCtrl+Alt+F");
Assert.DoesNotContain(menu.Items, item =>
item.Action == MeetingTaskbarAction.SwitchProfile &&
item.ProfileName == "default");
Assert.DoesNotContain(menu.Items, item => item.Action == MeetingTaskbarAction.StartRecording);
AssertMenuLayout(
menu,
("Open agent", MeetingTaskbarAction.EditRules, false),
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
("Microphone", MeetingTaskbarAction.OpenSubmenu, false),
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
("Switch to english\tCtrl+Alt+L", MeetingTaskbarAction.SwitchProfile, false),
("Exit", MeetingTaskbarAction.Exit, true));
Assert.Equal(
"english",
Assert.Single(menu.Items, item => item.Action == MeetingTaskbarAction.SwitchProfile).ProfileName);
}
[Fact]
public void RecordingMenuKeepsFinishMeetingIsolatedWithoutMicrophones()
{
var menu = MeetingTaskbarMenuBuilder.Build(
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
[Profile("default")]);
AssertMenuLayout(
menu,
("Open agent", MeetingTaskbarAction.EditRules, false),
("Finish meeting", MeetingTaskbarAction.StopRecording, true),
("Pause transcription", MeetingTaskbarAction.PauseTranscription, true),
("Cancel meeting recording and discard", MeetingTaskbarAction.AbortRecording, false),
("Exit", MeetingTaskbarAction.Exit, true));
}
[Fact]
public void TranscriptionPauseActionTracksActivePauseStateWithoutHidingFinish()
{
var runningMenu = MeetingTaskbarMenuBuilder.Build(
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default"),
[Profile("default")]);
var pausedMenu = MeetingTaskbarMenuBuilder.Build(
Status(isRecording: true, state: RecordingProcessState.Recording, profile: "default", isPaused: true),
[Profile("default")]);
Assert.Contains(runningMenu.Items, item =>
item.Action == MeetingTaskbarAction.PauseTranscription &&
item.Text == "Pause transcription");
Assert.Contains(pausedMenu.Items, item =>
item.Action == MeetingTaskbarAction.UnpauseTranscription &&
item.Text == "Unpause transcription");
Assert.Contains(pausedMenu.Items, item =>
item.Action == MeetingTaskbarAction.StopRecording &&
item.Text == "Finish meeting");
}
[Fact]
@@ -108,7 +174,7 @@ public sealed class TaskbarIconTests
{
Assert.Equal(
requiresConfirmation,
MeetingTaskbarExitPolicy.RequiresConfirmation(Status(state: state)));
MeetingTaskbarExitPolicy.RequiresConfirmation(state));
}
[Fact]
@@ -145,10 +211,20 @@ public sealed class TaskbarIconTests
});
}
private static void AssertMenuLayout(
MeetingTaskbarMenu menu,
params (string Text, MeetingTaskbarAction Action, bool StartsSection)[] expected)
{
Assert.Equal(
expected,
menu.Items.Select(item => (item.Text, item.Action, item.StartsSection)));
}
private static RecordingStatus Status(
bool isRecording = false,
RecordingProcessState state = RecordingProcessState.Idle,
string? profile = null)
string? profile = null,
bool isPaused = false)
{
return new RecordingStatus(
isRecording,
@@ -157,7 +233,8 @@ public sealed class TaskbarIconTests
state == RecordingProcessState.Idle ? null : "context.md",
state == RecordingProcessState.Idle ? null : "summary.md",
state,
profile);
profile,
isPaused);
}
[SupportedOSPlatform("windows")]
@@ -0,0 +1,44 @@
using MeetingAssistant.Workflow;
namespace MeetingAssistant.Tests;
internal sealed class TransformingAttendeeWorkflowEngine : IMeetingWorkflowEngine
{
private readonly string source;
private readonly string replacement;
public TransformingAttendeeWorkflowEngine(string source, string replacement)
{
this.source = source;
this.replacement = replacement;
}
public List<string> AttendeeRequests { get; } = [];
public Task RunAsync(
MeetingWorkflowEvent workflowEvent,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
public Task<string> TransformTranscriptLineAsync(
MeetingWorkflowEvent workflowEvent,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return Task.FromResult(workflowEvent.TranscriptLineText ?? "");
}
public Task<string> TransformAttendeeAsync(
MeetingWorkflowEvent workflowEvent,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
AttendeeRequests.Add(workflowEvent.AttendeeName ?? "");
return Task.FromResult(string.Equals(workflowEvent.AttendeeName, source, StringComparison.OrdinalIgnoreCase)
? replacement
: workflowEvent.AttendeeName ?? "");
}
}
@@ -0,0 +1,249 @@
using MeetingAssistant;
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using System.Text.Json;
namespace MeetingAssistant.Tests;
public sealed class VenvResemblyzerVoiceEncoderTests
{
[Fact]
public async Task WarmUpAsyncProvisionsVersionedCpuOnlyEnvironmentWithoutDocker()
{
using var fixture = new EncoderFixture();
await fixture.Encoder.WarmUpAsync(CancellationToken.None);
Assert.DoesNotContain(
fixture.Runner.Commands,
command => command.FileName.Contains("docker", StringComparison.OrdinalIgnoreCase));
Assert.Contains(
fixture.Runner.Commands,
command => command.FileName == "python-test"
&& command.Arguments.Take(2).SequenceEqual(["-m", "venv"]));
Assert.Contains(
fixture.Runner.Commands,
command => command.Arguments.Contains("torch==2.14.0+cpu")
&& command.Arguments.Contains("https://download.pytorch.org/whl/cpu"));
Assert.Contains(
fixture.Runner.Commands,
command => command.Arguments.Contains("webrtcvad-wheels==2.0.14"));
Assert.Contains(
fixture.Runner.Commands,
command => command.Arguments.Contains("Resemblyzer==0.1.4")
&& command.Arguments.Contains("--no-deps"));
}
[Fact]
public async Task WarmUpAsyncDoesNotMarkFailedEnvironmentReady()
{
using var fixture = new EncoderFixture(
warmupExitCode: 21,
warmupError: "incompatible environment");
await Assert.ThrowsAsync<InvalidOperationException>(
() => fixture.Encoder.WarmUpAsync(CancellationToken.None));
Assert.Empty(Directory.EnumerateFiles(
fixture.RuntimeFolder,
".ready",
SearchOption.AllDirectories));
}
[Fact]
public async Task EncodeAsyncRunsBatchWithManagedEnvironmentPython()
{
var first = new float[256];
first[0] = 2;
var second = new float[256];
second[1] = 3;
using var fixture = new EncoderFixture(VectorOutput([first, second]));
var vectors = await fixture.Encoder.EncodeAsync([[1, 2, 3], [4, 5, 6]], CancellationToken.None);
Assert.Equal(2, vectors.Count);
Assert.Equal(1f, vectors[0][0], 5);
Assert.Equal(1f, vectors[1][1], 5);
var encodingCommand = Assert.Single(
fixture.Runner.Commands,
command => command.Arguments.Any(
argument => argument.EndsWith("encode.py", StringComparison.Ordinal)));
Assert.EndsWith(
Path.Combine("Scripts", "python.exe"),
encodingCommand.FileName,
StringComparison.OrdinalIgnoreCase);
Assert.Equal(2, encodingCommand.Arguments.Count);
Assert.True(Path.IsPathFullyQualified(encodingCommand.Arguments[1]));
}
[Fact]
public async Task EncodeAsyncRejectsWrongVectorDimension()
{
var vector = new float[255];
vector[0] = 1;
using var fixture = new EncoderFixture(VectorOutput([vector]));
var error = await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
Assert.Contains("255 dimensions", error.Message);
}
[Fact]
public async Task EncodeAsyncRejectsMalformedVectorJson()
{
using var fixture = new EncoderFixture(
"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[not-json]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
}
[Fact]
public async Task EncodeAsyncRejectsNonFiniteVector()
{
var values = string.Join(',', new[] { "NaN" }.Concat(Enumerable.Repeat("0", 255)));
using var fixture = new EncoderFixture(
$"__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__\n[[{values}]]\n__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__");
await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
}
[Fact]
public async Task EncodeAsyncRejectsWrongResultCount()
{
var vector = new float[256];
vector[0] = 1;
using var fixture = new EncoderFixture(VectorOutput([vector, vector]));
var error = await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
Assert.Contains("2 vectors for 1 WAV samples", error.Message);
}
[Fact]
public async Task EncodeAsyncRejectsZeroMagnitudeVector()
{
using var fixture = new EncoderFixture(VectorOutput([new float[256]]));
var error = await Assert.ThrowsAsync<InvalidDataException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
Assert.Contains("zero magnitude", error.Message);
}
[Fact]
public async Task EncodeAsyncReportsFailedLocalCommand()
{
using var fixture = new EncoderFixture("", 17, "runtime failed");
var error = await Assert.ThrowsAsync<InvalidOperationException>(
() => fixture.Encoder.EncodeAsync([[1]], CancellationToken.None));
Assert.Contains("exit code 17", error.Message);
Assert.Contains("runtime failed", error.Message);
}
private static string VectorOutput(IReadOnlyList<float[]> vectors)
{
return $"""
__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__
{JsonSerializer.Serialize(vectors)}
__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__
""";
}
private sealed class EncoderFixture : IDisposable
{
public EncoderFixture(
string encodingOutput = "",
int encodingExitCode = 0,
string encodingError = "",
int warmupExitCode = 0,
string warmupError = "")
{
RuntimeFolder = Path.Combine(
Path.GetTempPath(),
"meeting-assistant-tests",
Guid.NewGuid().ToString("N"),
"resemblyzer");
Runner = new CapturingCommandRunner(
encodingOutput,
encodingExitCode,
encodingError,
warmupExitCode,
warmupError);
var options = new MeetingAssistantOptions();
options.SpeakerIdentification.Resemblyzer.PythonCommand = "python-test";
options.SpeakerIdentification.Resemblyzer.RuntimeFolder = RuntimeFolder;
Encoder = new VenvResemblyzerVoiceEncoder(
Runner,
Options.Create(options),
NullLogger<VenvResemblyzerVoiceEncoder>.Instance);
}
public CapturingCommandRunner Runner { get; }
public VenvResemblyzerVoiceEncoder Encoder { get; }
public string RuntimeFolder { get; }
public void Dispose()
{
if (Directory.Exists(RuntimeFolder))
{
Directory.Delete(RuntimeFolder, recursive: true);
}
}
}
private sealed class CapturingCommandRunner : ICommandRunner
{
private readonly string encodingOutput;
private readonly int encodingExitCode;
private readonly string encodingError;
private readonly int warmupExitCode;
private readonly string warmupError;
public CapturingCommandRunner(
string encodingOutput = "",
int encodingExitCode = 0,
string encodingError = "",
int warmupExitCode = 0,
string warmupError = "")
{
this.encodingOutput = encodingOutput;
this.encodingExitCode = encodingExitCode;
this.encodingError = encodingError;
this.warmupExitCode = warmupExitCode;
this.warmupError = warmupError;
}
public List<CapturedCommand> Commands { get; } = [];
public Task<CommandResult> RunAsync(
string fileName,
IReadOnlyList<string> arguments,
CancellationToken cancellationToken,
IReadOnlyDictionary<string, string>? environment = null)
{
Commands.Add(new CapturedCommand(fileName, arguments.ToList()));
var isEncoding = arguments.Any(argument => argument.EndsWith("encode.py", StringComparison.Ordinal));
if (isEncoding)
{
return Task.FromResult(new CommandResult(encodingExitCode, encodingOutput, encodingError));
}
var isWarmup = arguments.Any(argument => argument.Contains("VoiceEncoder", StringComparison.Ordinal));
return Task.FromResult(isWarmup
? new CommandResult(warmupExitCode, string.Empty, warmupError)
: new CommandResult(0, string.Empty, string.Empty));
}
}
private sealed record CapturedCommand(string FileName, IReadOnlyList<string> Arguments);
}
@@ -20,6 +20,7 @@ public sealed class WorkflowRulesEditorTests
Key = "summary-key",
KeyEnv = "SUMMARY_KEY",
Model = "summary-model",
UseStreaming = true,
EnableThinking = true,
ReasoningEffort = ReasoningEffortOption.High,
ReconnectionAttempts = 7,
@@ -33,6 +34,7 @@ public sealed class WorkflowRulesEditorTests
var editor = new WorkflowRulesEditorOptions
{
Model = "editor-model",
UseStreaming = false,
EnableThinking = false,
MaxOutputTokens = 50
};
@@ -43,6 +45,7 @@ public sealed class WorkflowRulesEditorTests
Assert.Equal("summary-key", effective.Key);
Assert.Equal("SUMMARY_KEY", effective.KeyEnv);
Assert.Equal("editor-model", effective.Model);
Assert.False(effective.UseStreaming);
Assert.False(effective.EnableThinking);
Assert.Equal(ReasoningEffortOption.High, effective.ReasoningEffort);
Assert.Equal(7, effective.ReconnectionAttempts);
@@ -442,7 +445,8 @@ public sealed class WorkflowRulesEditorTests
Assert.Equal("summary body", await File.ReadAllTextAsync(Path.Combine(summariesRoot, "daily-summary.md")));
Assert.Equal("one\ntwo\nthree", await File.ReadAllTextAsync(Path.Combine(transcriptsRoot, "daily-transcript.md")));
var note = await File.ReadAllTextAsync(Path.Combine(notesRoot, "daily.md"));
var note = (await File.ReadAllTextAsync(Path.Combine(notesRoot, "daily.md")))
.Replace("\r\n", "\n", StringComparison.Ordinal);
Assert.Contains("title: Fixed", note);
Assert.Contains("projects:\n- Alpha", note);
Assert.Contains("Existing body.", note);
@@ -636,6 +640,28 @@ public sealed class WorkflowRulesEditorTests
Assert.Equal("rules: []", await File.ReadAllTextAsync(fixture.RulesPath));
}
[Fact]
public async Task RulesEditorToolsRefuseSideEffectingAttendeeAddedRules()
{
var fixture = await CreateRulesEditorFixtureAsync();
var result = await fixture.Tools.WriteRules("""
rules:
- name: attendee-side-effect
on:
- attendee_added:
contains: Contoso
steps:
- uses: add_context
value: '@Model.Attendee.Name'
""", replace_file: true);
Assert.StartsWith("Refused: workflow rules are invalid.", result);
Assert.Contains("attendee-side-effect", result);
Assert.Contains("attendee.name", result);
Assert.Equal("rules: []", await File.ReadAllTextAsync(fixture.RulesPath));
}
[Fact]
public async Task RulesEditorToolsAcceptAndParseMaskedProfanityRedactionRule()
{
@@ -721,12 +747,45 @@ public sealed class WorkflowRulesEditorTests
Assert.Contains("read_logs", instructions);
Assert.Contains("read_spec_file", instructions);
Assert.Contains("list_recent_summaries", instructions);
}
[Fact]
public async Task InstructionBuilderIncludesProjectSyncGuidanceForCorrectedMeetingNotes()
{
var builder = new WorkflowRulesEditorInstructionBuilder(
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
var instructions = await builder.BuildAsync(new MeetingAssistantOptions(), CancellationToken.None);
Assert.Contains("When correcting a meeting note that has project references", instructions);
Assert.Contains("read that project's AGENTS.md", instructions);
Assert.Contains("new project reference", instructions);
Assert.Contains("matching correction", instructions);
}
[Fact]
public async Task InstructionBuilderTreatsAssistantContextAsMeetingMemoryDuringRepairs()
{
var builder = new WorkflowRulesEditorInstructionBuilder(
NullLogger<WorkflowRulesEditorInstructionBuilder>.Instance);
var options = new MeetingAssistantOptions
{
WorkflowRulesEditor = new WorkflowRulesEditorOptions
{
InitialPrompt = "Custom interactive agent instructions."
}
};
var instructions = await builder.BuildAsync(options, CancellationToken.None);
Assert.Contains("Custom interactive agent instructions.", instructions);
Assert.Contains("meeting-specific memory", instructions);
Assert.Contains("fix or investigate a meeting or summary", instructions);
Assert.Contains("read the matching assistant context", instructions);
Assert.Contains("problems, missing information, assumptions, prior fixes, and conclusions", instructions);
Assert.Contains("append a concise record of your fixes and conclusions", instructions);
}
[Fact]
public async Task RulesEditorToolsCrudAndSearchSpeakerIdentities()
{
@@ -750,6 +809,32 @@ public sealed class WorkflowRulesEditorTests
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
}
[Fact]
public async Task RulesEditorToolsExposeVoiceVectorCountsSeparatelyFromWavSamples()
{
await using var fixture = await WorkflowRulesEditorIdentityFixture.CreateAsync();
var identity = await fixture.AddIdentityAsync("Sabrina", sample: [1, 2, 3]);
fixture.Context.SpeakerVoiceVectors.Add(new SpeakerVoiceVector
{
SpeakerIdentityId = identity.Id,
ModelId = "resemblyzer-0.1.4-pretrained",
Dimensions = 256,
VectorBytes = new byte[256 * sizeof(float)],
Fingerprint = "vector-1",
CreatedAt = DateTimeOffset.UtcNow
});
await fixture.Context.SaveChangesAsync();
var tools = fixture.CreateTools();
var searchResult = await tools.SearchIdentities("Sabrina");
var readResult = await tools.ReadIdentity(identity.Id);
Assert.Contains("\"sampleCount\": 1", searchResult);
Assert.Contains("\"voiceVectorCount\": 1", searchResult);
Assert.Contains("\"voiceVectorCount\": 1", readResult);
Assert.DoesNotContain("vector-1", readResult);
}
[Fact]
public async Task RulesEditorToolsRefusesSamplelessSpeakerIdentityCreation()
{
@@ -758,7 +843,7 @@ public sealed class WorkflowRulesEditorTests
var result = await tools.CreateIdentity("Sabrina", ["Sabi"], ["Guest-01"]);
Assert.StartsWith("Refused: speaker identities require at least one audio sample.", result);
Assert.StartsWith("Refused: speaker identities require audio evidence", result);
Assert.Empty(await fixture.Context.SpeakerIdentities.ToListAsync());
}
@@ -846,15 +931,37 @@ public sealed class WorkflowRulesEditorTests
Assert.Equal("", viewModel.Draft);
Assert.Equal(
[new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, "Rename ABS Daily")],
viewModel.Messages.ToArray());
viewModel.Messages.Select(item => item.Message).OfType<WorkflowRulesEditorChatMessage>().ToArray());
pipeline.Release.SetResult();
await sendTask.WaitAsync(TimeSpan.FromSeconds(2));
Assert.False(viewModel.IsThinking);
Assert.Equal(2, viewModel.Messages.Count);
Assert.Equal(WorkflowRulesEditorChatRole.Agent, viewModel.Messages[1].Role);
Assert.Equal("Updated rules.", viewModel.Messages[1].Content);
AssertCompletedActivity(viewModel, ["Thinking..."], "Updated rules.");
}
[Fact]
public async Task ApprovedWorkflowPromptStartsImmediatelyInItsOwnConversation()
{
var existing = new WorkflowRulesEditorChatViewModel(new BlockingRulesEditorPipeline("Other work"))
{
Draft = "My unfinished draft"
};
var pipeline = new BlockingRulesEditorPipeline("Workflow updated.");
var requested = new WorkflowRulesEditorChatViewModel(pipeline);
const string prompt = "Given Alpha in the meeting title, when metadata arrives, then associate project Alpha.";
var task = requested.SendPromptAsync(prompt);
await pipeline.Started.Task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.True(requested.IsThinking);
Assert.Empty(pipeline.LastConversation);
Assert.Equal(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, prompt), requested.Messages[0].Message);
Assert.Equal("My unfinished draft", existing.Draft);
Assert.Empty(existing.Messages);
pipeline.Release.SetResult();
await task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.False(requested.IsThinking);
}
[Fact]
@@ -870,20 +977,19 @@ public sealed class WorkflowRulesEditorTests
Assert.False(viewModel.IsThinking);
Assert.Equal("", viewModel.Draft);
Assert.Equal(
[
new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, "check logs"),
new WorkflowRulesEditorChatMessage(
WorkflowRulesEditorChatRole.Agent,
"Settings and logs failed: The request timed out.")
],
viewModel.Messages.ToArray());
Assert.Equal(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, "check logs"), viewModel.Messages[0].Message);
AssertCompletedActivity(
viewModel,
["Thinking..."],
"Meeting Summary Agent failed: The request timed out.");
}
[Fact]
public async Task ViewModelShowsReconnectStatusReportedByPipeline()
{
var pipeline = new StatusReportingRulesEditorPipeline("Updated rules.");
var pipeline = new ReportingRulesEditorPipeline(
"Updated rules.",
WorkflowRulesEditorActivityUpdate.Status("Reconnecting..."));
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
{
Draft = "check logs"
@@ -900,7 +1006,146 @@ public sealed class WorkflowRulesEditorTests
Assert.False(viewModel.IsThinking);
Assert.Equal("Thinking...", viewModel.ActivityMessage);
Assert.Equal("Updated rules.", viewModel.Messages[1].Content);
AssertCompletedActivity(viewModel, ["Thinking...", "Reconnecting..."], "Updated rules.");
}
[Fact]
public async Task ViewModelShowsToolCallsAbovePersistentThinkingLine()
{
var pipeline = new ReportingRulesEditorPipeline(
"Updated rules.",
WorkflowRulesEditorActivityUpdate.ToolCall("read_logs"));
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
{
Draft = "check logs"
};
var sendTask = viewModel.SendAsync();
await pipeline.Reported.Task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.True(viewModel.IsThinking);
Assert.Equal("Thinking...", viewModel.ActivityMessage);
Assert.Equal(["Called tool: read_logs"], viewModel.ActivityMessages.ToArray());
pipeline.Release.SetResult();
await sendTask.WaitAsync(TimeSpan.FromSeconds(2));
Assert.False(viewModel.IsThinking);
Assert.Empty(viewModel.ActivityMessages);
AssertCompletedActivity(viewModel, ["Thinking...", "Called tool: read_logs"], "Updated rules.");
}
[Fact]
public async Task ViewModelShowsVisibleThinkingOutputInActivityExpander()
{
var pipeline = new ReportingRulesEditorPipeline(
"Updated rules.",
WorkflowRulesEditorActivityUpdate.Thinking("Checked the recent logs."),
WorkflowRulesEditorActivityUpdate.ToolCall("read_logs"),
WorkflowRulesEditorActivityUpdate.Thinking("Matched the rule to the user request."));
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
{
Draft = "check logs"
};
var sendTask = viewModel.SendAsync();
await pipeline.Reported.Task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.True(viewModel.IsThinking);
Assert.Equal("Thinking...", viewModel.ActivityMessage);
Assert.Equal(
[
"Checked the recent logs.",
"Called tool: read_logs",
"Matched the rule to the user request."
],
viewModel.ActivityMessages.ToArray());
pipeline.Release.SetResult();
await sendTask.WaitAsync(TimeSpan.FromSeconds(2));
Assert.False(viewModel.IsThinking);
Assert.Empty(viewModel.ActivityMessages);
AssertCompletedActivity(
viewModel,
[
"Checked the recent logs.",
"Called tool: read_logs",
"Matched the rule to the user request."
],
"Updated rules.");
}
[Fact]
public async Task ViewModelMarshalsBackgroundActivityUpdatesToCapturedContext()
{
var previousContext = SynchronizationContext.Current;
var context = new RecordingSynchronizationContext();
var pipeline = new BackgroundReportingRulesEditorPipeline(
"Updated rules.",
WorkflowRulesEditorActivityUpdate.Thinking("Checked the recent logs."));
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
{
Draft = "check logs"
};
SynchronizationContext.SetSynchronizationContext(context);
var sendTask = viewModel.SendAsync();
SynchronizationContext.SetSynchronizationContext(previousContext);
await pipeline.Reported.Task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.True(viewModel.IsThinking);
Assert.Equal(["Checked the recent logs."], viewModel.ActivityMessages.ToArray());
Assert.Equal(1, context.SendCount);
pipeline.Release.SetResult();
await sendTask.WaitAsync(TimeSpan.FromSeconds(2));
AssertCompletedActivity(viewModel, ["Checked the recent logs."], "Updated rules.");
}
[Fact]
public async Task ViewModelKeepsActivityItemsOutOfModelConversation()
{
var pipeline = new BlockingRulesEditorPipeline("Updated rules.");
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
{
Draft = "first"
};
var firstSend = viewModel.SendAsync();
await pipeline.Started.Task.WaitAsync(TimeSpan.FromSeconds(2));
pipeline.Release.SetResult();
await firstSend.WaitAsync(TimeSpan.FromSeconds(2));
pipeline.Reset("Second response.");
viewModel.Draft = "second";
var secondSend = viewModel.SendAsync();
await pipeline.Started.Task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.Equal(
[
new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, "first"),
new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.Agent, "Updated rules.")
],
pipeline.LastConversation);
pipeline.Release.SetResult();
await secondSend.WaitAsync(TimeSpan.FromSeconds(2));
}
private static void AssertCompletedActivity(
WorkflowRulesEditorChatViewModel viewModel,
IReadOnlyList<string> expectedActivityLines,
string expectedAgentMessage)
{
Assert.Equal(3, viewModel.Messages.Count);
Assert.Equal(WorkflowRulesEditorConversationItemKind.Activity, viewModel.Messages[1].Kind);
Assert.StartsWith("Worked for ", viewModel.Messages[1].Content);
Assert.Equal(expectedActivityLines, viewModel.Messages[1].ActivityLines);
Assert.Equal(WorkflowRulesEditorConversationItemKind.Message, viewModel.Messages[2].Kind);
Assert.Equal(WorkflowRulesEditorChatRole.Agent, viewModel.Messages[2].Message?.Role);
Assert.Equal(expectedAgentMessage, viewModel.Messages[2].Message?.Content);
}
[Theory]
@@ -1185,31 +1430,36 @@ public sealed class WorkflowRulesEditorTests
private sealed class BlockingRulesEditorPipeline : IWorkflowRulesEditorChatPipeline
{
private readonly string response;
private string response;
public BlockingRulesEditorPipeline(string response)
{
this.response = response;
}
public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource Started { get; private set; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource Release { get; private set; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public IReadOnlyList<WorkflowRulesEditorChatMessage> LastConversation { get; private set; } = [];
public void Reset(string nextResponse)
{
response = nextResponse;
Started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
Release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
}
public async Task<WorkflowRulesEditorChatResult> SendAsync(
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
string userMessage,
CancellationToken cancellationToken,
Action<string>? statusChanged = null)
Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null)
{
LastConversation = conversation;
Started.SetResult();
await Release.Task.WaitAsync(cancellationToken);
return new WorkflowRulesEditorChatResult(
response,
conversation
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, userMessage))
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.Agent, response))
.ToList());
return new WorkflowRulesEditorChatResult(response);
}
}
@@ -1226,19 +1476,23 @@ public sealed class WorkflowRulesEditorTests
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
string userMessage,
CancellationToken cancellationToken,
Action<string>? statusChanged = null)
Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null)
{
throw exception;
}
}
private sealed class StatusReportingRulesEditorPipeline : IWorkflowRulesEditorChatPipeline
private sealed class ReportingRulesEditorPipeline : IWorkflowRulesEditorChatPipeline
{
private readonly string response;
private readonly IReadOnlyList<WorkflowRulesEditorActivityUpdate> updates;
public StatusReportingRulesEditorPipeline(string response)
public ReportingRulesEditorPipeline(
string response,
params WorkflowRulesEditorActivityUpdate[] updates)
{
this.response = response;
this.updates = updates;
}
public TaskCompletionSource Reported { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
@@ -1249,17 +1503,57 @@ public sealed class WorkflowRulesEditorTests
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
string userMessage,
CancellationToken cancellationToken,
Action<string>? statusChanged = null)
Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null)
{
statusChanged?.Invoke("Reconnecting...");
foreach (var update in updates)
{
activityChanged?.Invoke(update);
}
Reported.SetResult();
await Release.Task.WaitAsync(cancellationToken);
return new WorkflowRulesEditorChatResult(
response,
conversation
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, userMessage))
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.Agent, response))
.ToList());
return new WorkflowRulesEditorChatResult(response);
}
}
private sealed class BackgroundReportingRulesEditorPipeline : IWorkflowRulesEditorChatPipeline
{
private readonly string response;
private readonly WorkflowRulesEditorActivityUpdate update;
public BackgroundReportingRulesEditorPipeline(
string response,
WorkflowRulesEditorActivityUpdate update)
{
this.response = response;
this.update = update;
}
public TaskCompletionSource Reported { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public async Task<WorkflowRulesEditorChatResult> SendAsync(
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
string userMessage,
CancellationToken cancellationToken,
Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null)
{
await Task.Run(() => activityChanged?.Invoke(update), cancellationToken);
Reported.SetResult();
await Release.Task.WaitAsync(cancellationToken);
return new WorkflowRulesEditorChatResult(response);
}
}
private sealed class RecordingSynchronizationContext : SynchronizationContext
{
public int SendCount { get; private set; }
public override void Send(SendOrPostCallback callback, object? state)
{
SendCount++;
callback(state);
}
}
@@ -1,4 +1,5 @@
using MeetingAssistant.Recording;
using MeetingAssistant.MeetingNotes;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Calendar;
@@ -84,7 +85,8 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
foreach (var meeting in cachedMeetings.OrderBy(meeting => meeting.Start))
{
var promptKey = GetPromptKey(meeting);
if (promptedMeetingKeys.Contains(promptKey) ||
if (meeting.IsCanceled ||
promptedMeetingKeys.Contains(promptKey) ||
!IsInPromptWindow(meeting, now, promptOptions))
{
continue;
@@ -96,8 +98,10 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
meeting.Subject,
meeting.Start);
await promptService.ShowPromptAsync(
new MeetingStartPromptRequest(meeting),
HandlePromptResponseAsync,
new MeetingStartPromptRequest(
meeting,
recordingController.CurrentStatus.IsRecording && meeting.Metadata is not null),
(response, token) => HandlePromptResponseAsync(meeting, response, token),
cancellationToken);
}
}
@@ -133,9 +137,22 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
}
private async Task HandlePromptResponseAsync(
CalendarMeeting meeting,
MeetingStartPromptResponse response,
CancellationToken cancellationToken)
{
if (response == MeetingStartPromptResponse.AttachMetadataToCurrentMeeting)
{
if (meeting.Metadata is not null)
{
await recordingController.AttachMetadataToCurrentMeetingAsync(
meeting.Metadata,
cancellationToken);
}
return;
}
if (response != MeetingStartPromptResponse.Record)
{
return;
@@ -146,7 +163,7 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
await recordingController.StopAsync(cancellationToken);
}
await recordingController.StartAsync(cancellationToken);
await recordingController.StartFromPromptAsync(meeting.Metadata, cancellationToken);
}
private void ResetPromptedMeetingsIfDayChanged(DateOnly today)
@@ -183,6 +200,7 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
var syncDelay = nextSyncAt - now;
var nextMeetingStart = cachedMeetings
.Where(meeting => !meeting.IsCanceled)
.Where(meeting => !promptedMeetingKeys.Contains(GetPromptKey(meeting)))
.Where(meeting => meeting.End > now)
.Select(meeting => meeting.Start <= now ? now : meeting.Start)
@@ -228,7 +246,9 @@ public sealed record CalendarMeeting(
string Id,
string Subject,
DateTimeOffset Start,
DateTimeOffset End);
DateTimeOffset End,
MeetingMetadata? Metadata = null,
bool IsCanceled = false);
public interface IMeetingStartPromptService
{
@@ -238,12 +258,15 @@ public interface IMeetingStartPromptService
CancellationToken cancellationToken);
}
public sealed record MeetingStartPromptRequest(CalendarMeeting Meeting);
public sealed record MeetingStartPromptRequest(
CalendarMeeting Meeting,
bool CanAttachToCurrentMeeting = false);
public enum MeetingStartPromptResponse
{
Record,
Skip
Skip,
AttachMetadataToCurrentMeeting
}
public interface IMeetingPromptRecordingController
@@ -252,6 +275,14 @@ public interface IMeetingPromptRecordingController
Task<RecordingStatus> StartAsync(CancellationToken cancellationToken);
Task<RecordingStatus> StartFromPromptAsync(
MeetingMetadata? metadata,
CancellationToken cancellationToken);
Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
MeetingMetadata metadata,
CancellationToken cancellationToken);
Task<RecordingStatus> StopAsync(CancellationToken cancellationToken);
}
@@ -271,6 +302,20 @@ public sealed class MeetingPromptRecordingController : IMeetingPromptRecordingCo
return coordinator.StartAsync(cancellationToken);
}
public Task<RecordingStatus> StartFromPromptAsync(
MeetingMetadata? metadata,
CancellationToken cancellationToken)
{
return coordinator.StartFromPromptAsync(metadata, cancellationToken);
}
public Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
MeetingMetadata metadata,
CancellationToken cancellationToken)
{
return coordinator.AttachMetadataToCurrentMeetingAsync(metadata, cancellationToken);
}
public Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
{
return coordinator.StopAsync(cancellationToken);
@@ -60,18 +60,21 @@ public sealed class OutlookClassicCalendarMeetingProvider : ICalendarMeetingProv
var end = Convert.ToDateTime(OutlookClassicCom.GetProperty(item, "End"));
if (end <= dayStart ||
start < dayStart ||
!IsTeamsAppointment(item))
OutlookClassicCom.IsCanceledAppointment(item) ||
!OutlookClassicAppointmentMetadata.IsTeamsAppointment(item))
{
continue;
}
var rawSubject = Convert.ToString(OutlookClassicCom.GetProperty(item, "Subject"))?.Trim();
var subject = string.IsNullOrWhiteSpace(rawSubject) ? "Teams meeting" : rawSubject;
var subject = OutlookClassicAppointmentMetadata.ReadSubject(item);
var localEnd = OutlookClassicCom.ToLocalOffset(end);
meetings.Add(new CalendarMeeting(
GetAppointmentId(item, start, end, subject),
subject,
OutlookClassicCom.ToLocalOffset(start),
OutlookClassicCom.ToLocalOffset(end)));
localEnd,
OutlookClassicAppointmentMetadata.CreateMetadata(item, end),
IsCanceled: false));
}
finally
{
@@ -98,13 +101,5 @@ public sealed class OutlookClassicCalendarMeetingProvider : ICalendarMeetingProv
? $"{start:O}|{end:O}|{subject}"
: $"{id}|{start:O}";
}
private static bool IsTeamsAppointment(object appointment)
{
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? "";
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body);
}
}
#endif
@@ -118,21 +118,10 @@ public sealed class WindowsMeetingStartPromptService : IMeetingStartPromptServic
string promptId,
MeetingStartPromptRequest request)
{
var yesButton = new ToastButton()
.SetContent("Yes")
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", "record")
.SetBackgroundActivation();
var yesButton = BuildResponseButton("Yes", promptId, "record");
var noButton = BuildResponseButton("No", promptId, "skip");
var noButton = new ToastButton()
.SetContent("No")
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", "skip")
.SetBackgroundActivation();
return new ToastContentBuilder()
var notification = new ToastContentBuilder()
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.SetToastScenario(ToastScenario.Reminder)
@@ -141,6 +130,30 @@ public sealed class WindowsMeetingStartPromptService : IMeetingStartPromptServic
.AddText($"{request.Meeting.Subject} starts at {request.Meeting.Start.LocalDateTime:t}.")
.AddButton(yesButton)
.AddButton(noButton);
if (request.CanAttachToCurrentMeeting)
{
notification.AddButton(
BuildResponseButton(
"Add metadata to current meeting",
promptId,
"attach-metadata"));
}
return notification;
}
private static ToastButton BuildResponseButton(
string content,
string promptId,
string response)
{
return new ToastButton()
.SetContent(content)
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", response)
.SetBackgroundActivation();
}
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
@@ -166,10 +179,14 @@ public sealed class WindowsMeetingStartPromptService : IMeetingStartPromptServic
return;
}
var response = TryGetArgument(arguments, "response", out var responseValue) &&
string.Equals(responseValue, "record", StringComparison.OrdinalIgnoreCase)
? MeetingStartPromptResponse.Record
: MeetingStartPromptResponse.Skip;
var response = TryGetArgument(arguments, "response", out var responseValue)
? responseValue.ToLowerInvariant() switch
{
"record" => MeetingStartPromptResponse.Record,
"attach-metadata" => MeetingStartPromptResponse.AttachMetadataToCurrentMeeting,
_ => MeetingStartPromptResponse.Skip
}
: MeetingStartPromptResponse.Skip;
_ = Task.Run(async () =>
{
try
@@ -32,7 +32,14 @@ Calendar-Feature is refined, Golem feature needs more work.
PROJECT.md should use this structure:
Maintain its YAML `name` as the project's display name and `description` as a grounded description of its purpose in at most 256 characters. These fields help summarizers identify missing project associations. Preserve other frontmatter and body content when updating them.
```markdown
---
name: Project display name
description: Concise project purpose and distinguishing context.
---
# Executive Summary
<Elevator Pitch>
@@ -1,4 +1,5 @@
using Microsoft.Extensions.Configuration;
using MeetingAssistant.Speakers;
namespace MeetingAssistant.LaunchProfiles;
@@ -48,6 +49,7 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
var options = BindDefaultOptions();
if (profileName.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase))
{
ValidateSpeakerSampleDurations(options, DefaultProfileName);
return new LaunchProfile(DefaultProfileName, options);
}
@@ -59,6 +61,7 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
profileSection.Bind(options);
ApplyArrayOverrides(profileSection, options);
ValidateSpeakerSampleDurations(options, profileName);
return new LaunchProfile(profileName, options);
}
@@ -149,6 +152,18 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
: name.Trim();
}
private static void ValidateSpeakerSampleDurations(
MeetingAssistantOptions options,
string profileName)
{
SpeakerSampleDurationConfiguration.ValidateOrThrow(
options.SpeakerIdentification,
$"Launch profile '{profileName}' has invalid speaker sample durations");
ResemblyzerSpeakerRecognitionConfiguration.ValidateOrThrow(
options.SpeakerIdentification.Resemblyzer,
$"Launch profile '{profileName}' has invalid Resemblyzer speaker-recognition settings");
}
private static void ApplyArrayOverrides(
IConfigurationSection profileSection,
MeetingAssistantOptions options)
+10 -9
View File
@@ -5,7 +5,7 @@
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<PreserveCompilationContext>true</PreserveCompilationContext>
<MicrosoftSpeechVersion>1.50.0</MicrosoftSpeechVersion>
<MicrosoftSpeechVersion>1.52.0</MicrosoftSpeechVersion>
</PropertyGroup>
<PropertyGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
@@ -20,18 +20,19 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.11.0" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.21.0" />
<PackageReference Include="DiffPlex" Version="1.9.0" />
<PackageReference Include="Microsoft.CognitiveServices.Speech" Version="$(MicrosoftSpeechVersion)" />
<PackageReference Include="Microsoft.CognitiveServices.Speech.Extension.MAS" Version="$(MicrosoftSpeechVersion)" ExcludeAssets="build" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.9" />
<PackageReference Include="NAudio" Version="2.3.0" />
<PackageReference Include="NCalcSync" Version="5.12.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.12" />
<PackageReference Include="NAudio" Version="3.1.0" />
<PackageReference Include="NCalcSync" Version="7.2.0" />
<PackageReference Include="RazorLight" Version="2.3.1" />
<PackageReference Include="System.Drawing.Common" Version="10.0.9" />
<PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.5" />
<PackageReference Include="System.Drawing.Common" Version="10.0.12" />
<PackageReference Include="Whisper.net" Version="1.9.1" />
<PackageReference Include="Whisper.net.Runtime" Version="1.9.1" />
<PackageReference Include="YamlDotNet" Version="18.0.0" />
<PackageReference Include="YamlDotNet" Version="18.1.0" />
</ItemGroup>
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
@@ -49,8 +50,7 @@
</ItemGroup>
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
<None Include="$(NuGetPackageRoot)microsoft.cognitiveservices.speech.extension.mas\$(MicrosoftSpeechVersion)\contentFiles\any\any\models\aec_v1.fpie" Link="runtimes\win-x64\native\MASmodels\aec_v1.fpie" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
<None Include="$(NuGetPackageRoot)microsoft.cognitiveservices.speech.extension.mas\$(MicrosoftSpeechVersion)\contentFiles\any\any\models\pns_avg4.fpie" Link="runtimes\win-x64\native\MASmodels\pns_avg4.fpie" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
<None Include="$(NuGetPackageRoot)microsoft.cognitiveservices.speech.extension.mas\$(MicrosoftSpeechVersion)\contentFiles\any\any\models\*.fpie" Link="runtimes\win-x64\native\MASmodels\%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
</ItemGroup>
<ItemGroup>
@@ -64,6 +64,7 @@
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) != 'windows'">
<Compile Remove="Hotkeys\GlobalHotkeyService.cs" />
<Compile Remove="Recording\NaudioCaptureSource.cs" />
<Compile Remove="Recording\WindowsMicrophoneDeviceProvider.cs" />
<Compile Remove="MeetingNotes\OutlookClassicMeetingMetadataProvider.Windows.cs" />
<Compile Remove="Screenshots\ActiveWindowScreenshotCapture.Windows.cs" />
<Compile Remove="Taskbar\UnoTaskbarIconService.Windows.cs" />
+62 -9
View File
@@ -59,6 +59,7 @@ public sealed class ScreenshotOcrOptions
Identify who appears to be talking, who appears to be presenting, and what is being presented when visible.
If people or participant tiles are visible, state whether it is clear from the screenshot exactly who is in the meeting or whether the visible people are only a partial participant result.
Include any attendee names that are clearly visible or strongly deduced from the screenshot in the meeting-assistant metadata. The attendee list may be partial.
If the screenshot contains slides, capture the visible text in clean markdown.
If the screenshot contains a diagram, convert it to Mermaid when possible and preserve labels accurately.
If it contains another kind of shared screen or scene, describe the scene and the relevant visible information.
@@ -67,9 +68,9 @@ public sealed class ScreenshotOcrOptions
Do not return crop coordinates for a person, webcam tile, whole app chrome, or content you cannot isolate confidently.
End your response with exactly one fenced json block containing meeting-assistant metadata, for example:
```json
{ "crop": { "x": 0, "y": 0, "width": 1920, "height": 1080 } }
{ "crop": { "x": 0, "y": 0, "width": 1920, "height": 1080 }, "attendees": ["Ada Lovelace"] }
```
Use `{ "crop": null }` when no confident presentation/shared-screen crop exists.
Use `{ "crop": null, "attendees": [] }` when no confident presentation/shared-screen crop exists and no attendees are visible or deducible.
Do not invent information that is not visible in the screenshot.
""";
@@ -125,6 +126,8 @@ public sealed class RecordingOptions
public int Channels { get; set; } = 1;
public string? MicrophoneDeviceId { get; set; }
public double MicrophoneMixGain { get; set; } = 1;
public double SystemAudioMixGain { get; set; } = 1;
@@ -162,6 +165,8 @@ public sealed class RecordingInactivitySafeguardOptions
public TimeSpan AutoStopAfter { get; set; } = TimeSpan.FromMinutes(30);
public TimeSpan MaximumPauseDuration { get; set; } = TimeSpan.FromHours(4);
public TimeSpan InferredEndPadding { get; set; } = TimeSpan.FromMinutes(1);
public TimeSpan CheckInterval { get; set; } = TimeSpan.FromSeconds(15);
@@ -178,10 +183,8 @@ public sealed class WhisperLocalOptions
public PyannoteDiarizationOptions Diarization { get; set; } = new();
}
public sealed class PyannoteDiarizationOptions
public class PyannoteRuntimeOptions
{
public bool Enabled { get; set; } = true;
public string DockerCommand { get; set; } = "docker";
public string BaseImage { get; set; } = "python:3.11-slim";
@@ -205,6 +208,11 @@ public sealed class PyannoteDiarizationOptions
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(30);
}
public sealed class PyannoteDiarizationOptions : PyannoteRuntimeOptions
{
public bool Enabled { get; set; } = true;
}
public sealed class AzureSpeechOptions
{
public string Endpoint { get; set; } = "";
@@ -337,9 +345,9 @@ public sealed class SpeakerIdentificationOptions
public int MaxSnippetsPerSpeaker { get; set; } = 3;
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(30);
public TimeSpan MinimumSampleSpeechDuration { get; set; } = TimeSpan.FromSeconds(10);
public TimeSpan MaximumSampleSegmentGap { get; set; } = TimeSpan.FromSeconds(1);
public TimeSpan MaximumSampleDuration { get; set; } = TimeSpan.FromSeconds(60);
public double SilenceBetweenSnippetsSeconds { get; set; } = 1;
@@ -352,6 +360,47 @@ public sealed class SpeakerIdentificationOptions
public AzureSpeechOptions AzureSpeech { get; set; } = new();
public SpeakerIdentityPyannoteValidationOptions PyannoteValidation { get; set; } = new();
public ResemblyzerSpeakerRecognitionOptions Resemblyzer { get; set; } = new();
}
public sealed class ResemblyzerSpeakerRecognitionOptions
{
public bool Enabled { get; set; }
public int RequiredVectorsPerSpeaker { get; set; } = 5;
public int MaxVectorsPerIdentity { get; set; } = 1000;
public int OutlierPruningMinimumVectors { get; set; } = 20;
public double OutlierPruningNeighborSimilarity { get; set; } = 0.75;
public int OutlierPruningMinimumNeighbors { get; set; } = 3;
public double OutlierPruningMinimumClusterRatio { get; set; } = 0.60;
public double MinimumClusterCohesion { get; set; } = 0.75;
public double MinimumIdentitySimilarity { get; set; } = 0.75;
public double MinimumSimilarityMargin { get; set; } = 0.05;
public string ModelId { get; set; } = "resemblyzer-0.1.4-pretrained";
public string PackageVersion { get; set; } = "0.1.4";
public string PythonCommand { get; set; } = "python";
public string TorchVersion { get; set; } = "2.14.0+cpu";
public string TorchIndexUrl { get; set; } = "https://download.pytorch.org/whl/cpu";
public string WebRtcVadVersion { get; set; } = "2.0.14";
public string RuntimeFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Resemblyzer";
public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromMinutes(15);
}
public sealed class SpeakerIdentityPyannoteValidationOptions
@@ -362,9 +411,8 @@ public sealed class SpeakerIdentityPyannoteValidationOptions
public double MinimumMatchingKnownSnippetRatio { get; set; } = 0.50;
public PyannoteDiarizationOptions Diarization { get; set; } = new()
public PyannoteRuntimeOptions Diarization { get; set; } = new()
{
Enabled = true,
CommandTimeout = TimeSpan.FromHours(1),
AlignmentMode = PyannoteAlignmentMode.PyannoteTurns
};
@@ -380,6 +428,8 @@ public sealed class AgentOptions
public string Model { get; set; } = "chatgpt/gpt-5.5";
public bool UseStreaming { get; set; } = true;
public bool EnableThinking { get; set; } = true;
public ReasoningEffortOption ReasoningEffort { get; set; } = ReasoningEffortOption.Medium;
@@ -411,6 +461,8 @@ public sealed class WorkflowRulesEditorOptions
public string? Model { get; set; }
public bool? UseStreaming { get; set; }
public bool? EnableThinking { get; set; }
public ReasoningEffortOption? ReasoningEffort { get; set; }
@@ -439,6 +491,7 @@ public sealed class WorkflowRulesEditorOptions
Key = string.IsNullOrWhiteSpace(Key) ? defaults.Key : Key,
KeyEnv = string.IsNullOrWhiteSpace(KeyEnv) ? defaults.KeyEnv : KeyEnv!,
Model = string.IsNullOrWhiteSpace(Model) ? defaults.Model : Model!,
UseStreaming = UseStreaming ?? defaults.UseStreaming,
EnableThinking = EnableThinking ?? defaults.EnableThinking,
ReasoningEffort = ReasoningEffort ?? defaults.ReasoningEffort,
ReconnectionAttempts = ReconnectionAttempts ?? defaults.ReconnectionAttempts,
@@ -1,4 +1,6 @@
using Microsoft.Extensions.Options;
using System.Globalization;
using System.Text;
using YamlDotNet.Core;
using YamlDotNet.Serialization;
@@ -113,7 +115,24 @@ public sealed class MarkdownMeetingNoteStore : IMeetingNoteStore
private static string EscapeQuoted(string value)
{
return $"\"{value.Replace("\"", "\\\"", StringComparison.Ordinal)}\"";
var escaped = new StringBuilder(value.Length + 2);
escaped.Append('"');
foreach (var character in value)
{
escaped.Append(character switch
{
'\\' => "\\\\",
'"' => "\\\"",
'\r' => "\\r",
'\n' => "\\n",
'\t' => "\\t",
< ' ' => "\\x" + ((int)character).ToString("X2", CultureInfo.InvariantCulture),
_ => character.ToString()
});
}
escaped.Append('"');
return escaped.ToString();
}
private static string EscapeNullableDateTime(DateTimeOffset? value)
@@ -135,7 +154,7 @@ public sealed class MarkdownMeetingNoteStore : IMeetingNoteStore
return "\"\"";
}
if (value.Contains(':', StringComparison.Ordinal) || value.StartsWith("[", StringComparison.Ordinal))
if (RequiresQuotedScalar(value))
{
return EscapeQuoted(value);
}
@@ -143,6 +162,49 @@ public sealed class MarkdownMeetingNoteStore : IMeetingNoteStore
return value;
}
private static bool RequiresQuotedScalar(string value)
{
if (char.IsWhiteSpace(value[0]) || char.IsWhiteSpace(value[^1]))
{
return true;
}
if (value.Contains(':', StringComparison.Ordinal) ||
value.Contains('\'', StringComparison.Ordinal) ||
value.Contains(" #", StringComparison.Ordinal) ||
value.Any(char.IsControl))
{
return true;
}
if (IsYamlIndicator(value[0]))
{
return true;
}
if (IsYamlCoreSchemaKeyword(value))
{
return true;
}
return DateTimeOffset.TryParse(value, out _) ||
double.TryParse(value, CultureInfo.InvariantCulture, out _);
}
private static bool IsYamlIndicator(char character)
{
return character is '-' or '?' or ':' or ',' or '[' or ']' or '{' or '}' or
'#' or '&' or '*' or '!' or '|' or '>' or '\'' or '"' or '%' or '@' or '`';
}
private static bool IsYamlCoreSchemaKeyword(string value)
{
return string.Equals(value, "true", StringComparison.OrdinalIgnoreCase) ||
string.Equals(value, "false", StringComparison.OrdinalIgnoreCase) ||
string.Equals(value, "null", StringComparison.OrdinalIgnoreCase) ||
value == "~";
}
private sealed class MeetingNoteYaml
{
[YamlMember(Alias = "title")]
@@ -8,4 +8,13 @@ internal static class MeetingAttendeeNames
var emailStart = trimmed.IndexOf('<', StringComparison.Ordinal);
return emailStart > 0 ? trimmed[..emailStart].Trim() : trimmed;
}
public static List<string> NormalizeDistinct(IEnumerable<string> attendees)
{
return attendees
.Select(attendee => attendee.Trim())
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
}
}
@@ -14,4 +14,24 @@ public static class MeetingNoteActionLinks
var url = $"{baseUrl}/meetings/summary/retry?summaryPath={Uri.EscapeDataString(summaryFileName)}";
return $"[{label}]({url})";
}
public static string CreateScreenshotOcrRetryLink(
string publicBaseUrl,
MeetingSessionArtifacts artifacts,
string screenshotPath,
string screenshotId,
string label = "Retry screenshot OCR")
{
var baseUrl = string.IsNullOrWhiteSpace(publicBaseUrl)
? "http://localhost:5090"
: publicBaseUrl.TrimEnd('/');
var url = $"{baseUrl}/meetings/screenshot-ocr/retry" +
$"?meetingNotePath={Uri.EscapeDataString(artifacts.MeetingNotePath)}" +
$"&transcriptPath={Uri.EscapeDataString(artifacts.TranscriptPath)}" +
$"&assistantContextPath={Uri.EscapeDataString(artifacts.AssistantContextPath)}" +
$"&summaryPath={Uri.EscapeDataString(artifacts.SummaryPath)}" +
$"&screenshotPath={Uri.EscapeDataString(screenshotPath)}" +
$"&screenshotId={Uri.EscapeDataString(screenshotId)}";
return $"[{label}]({url})";
}
}
@@ -0,0 +1,176 @@
#if WINDOWS
using MeetingAssistant.Calendar;
namespace MeetingAssistant.MeetingNotes;
internal static class OutlookClassicAppointmentMetadata
{
public static MeetingMetadata CreateMetadata(object appointment, DateTime end)
{
var title = ReadSubject(appointment);
return new MeetingMetadata(
title,
ReadAttendees(appointment),
ExtractAgenda(Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? ""),
OutlookClassicCom.ToLocalOffset(end));
}
public static string ReadSubject(object appointment)
{
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject"))?.Trim();
return string.IsNullOrWhiteSpace(subject) ? "Teams meeting" : subject;
}
public static bool IsTeamsAppointment(object appointment)
{
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? "";
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body);
}
public static string ExtractAgenda(string body)
{
if (string.IsNullOrWhiteSpace(body))
{
return "";
}
var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n');
var agendaLines = new List<string>();
foreach (var line in lines)
{
if (IsTeamsSeparator(line) || IsTeamsJoinLine(line))
{
break;
}
agendaLines.Add(line);
}
return string.Join(Environment.NewLine, agendaLines).Trim();
}
public static IReadOnlyList<string> ReadAttendees(object appointment)
{
var attendees = new List<string>();
var organizer = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Organizer"));
if (!string.IsNullOrWhiteSpace(organizer))
{
attendees.Add(organizer.Trim());
}
object? recipients = null;
try
{
recipients = OutlookClassicCom.GetProperty(appointment, "Recipients");
var count = Convert.ToInt32(OutlookClassicCom.GetProperty(recipients!, "Count"));
for (var index = 1; index <= count; index++)
{
object? recipient = null;
try
{
recipient = OutlookClassicCom.Invoke(recipients!, "Item", index);
var formatted = FormatRecipient(recipient!);
if (!string.IsNullOrWhiteSpace(formatted))
{
attendees.Add(formatted);
}
}
finally
{
OutlookClassicCom.ReleaseComObject(recipient);
}
}
}
finally
{
OutlookClassicCom.ReleaseComObject(recipients);
}
return NormalizeAttendees(attendees);
}
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
{
return attendees
.Select(NormalizeAttendee)
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase)
.Select(group => group
.OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal))
.ThenBy(attendee => attendee.Length)
.First())
.ToList();
}
private static bool IsTeamsSeparator(string line)
{
var trimmed = line.Trim();
return trimmed.Length >= 8 &&
trimmed.All(character => character is '_' or '-' or '*' or ' ');
}
private static bool IsTeamsJoinLine(string line)
{
return TeamsMeetingMarkerDetector.ContainsTeamsMarker(line);
}
private static string NormalizeAttendee(string attendee)
{
var normalized = attendee.Trim();
var mailtoPrefix = "mailto:";
normalized = normalized.Replace(mailtoPrefix, "", StringComparison.OrdinalIgnoreCase);
while (normalized.Contains(" ", StringComparison.Ordinal))
{
normalized = normalized.Replace(" ", " ", StringComparison.Ordinal);
}
var emailStart = normalized.IndexOf('<', StringComparison.Ordinal);
var emailEnd = normalized.LastIndexOf('>');
if (emailStart > 0 && emailEnd > emailStart)
{
var name = normalized[..emailStart].Trim();
var email = normalized[(emailStart + 1)..emailEnd].Trim();
if (name.Equals(email, StringComparison.OrdinalIgnoreCase))
{
return email;
}
return $"{name} <{email}>";
}
return normalized;
}
private static string GetAttendeeDeduplicationKey(string attendee)
{
var emailStart = attendee.IndexOf('<', StringComparison.Ordinal);
var emailEnd = attendee.LastIndexOf('>');
if (emailStart > 0 && emailEnd > emailStart)
{
return attendee[(emailStart + 1)..emailEnd].Trim();
}
return attendee.Trim();
}
private static string FormatRecipient(object recipient)
{
var name = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Name"))?.Trim() ?? "";
var email = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Address"))?.Trim() ?? "";
if (string.IsNullOrWhiteSpace(name))
{
return email;
}
if (string.IsNullOrWhiteSpace(email) ||
email.Contains('/'))
{
return name;
}
return $"{name} <{email}>";
}
}
#endif
@@ -108,6 +108,37 @@ internal static class OutlookClassicCom
}
}
public static bool IsCanceledAppointment(object appointment)
{
var meetingStatus = TryGetProperty(appointment, "MeetingStatus");
if (meetingStatus is not null)
{
try
{
var status = Convert.ToInt32(meetingStatus);
if (status is 5 or 7)
{
return true;
}
}
catch (Exception exception) when (exception is FormatException or InvalidCastException or OverflowException)
{
// Fall back to subject prefixes when Outlook exposes a non-numeric status value.
}
}
var subject = Convert.ToString(TryGetProperty(appointment, "Subject")) ?? "";
return SubjectIndicatesCancellation(subject);
}
internal static bool SubjectIndicatesCancellation(string subject)
{
var normalized = subject.TrimStart();
return normalized.StartsWith("Canceled:", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Cancelled:", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Abgesagt:", StringComparison.OrdinalIgnoreCase);
}
public static DateTimeOffset ToLocalOffset(DateTime value)
{
return new DateTimeOffset(DateTime.SpecifyKind(value, DateTimeKind.Local));
@@ -121,7 +121,9 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv
}
var end = Convert.ToDateTime(OutlookClassicCom.GetProperty(item, "End"));
if (end < windowStart || !IsTeamsAppointment(item))
if (end < windowStart ||
OutlookClassicCom.IsCanceledAppointment(item) ||
!OutlookClassicAppointmentMetadata.IsTeamsAppointment(item))
{
continue;
}
@@ -150,15 +152,7 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv
return null;
}
var appointment = selected.Appointment;
var title = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
var attendees = ReadAttendees(appointment);
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
return new MeetingMetadata(
title.Trim(),
attendees,
ExtractAgenda(body),
OutlookClassicCom.ToLocalOffset(selected.End));
return OutlookClassicAppointmentMetadata.CreateMetadata(selected.Appointment, selected.End);
}
finally
{
@@ -169,158 +163,6 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv
}
}
internal static string ExtractAgenda(string body)
{
if (string.IsNullOrWhiteSpace(body))
{
return "";
}
var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n');
var agendaLines = new List<string>();
foreach (var line in lines)
{
if (IsTeamsSeparator(line) || IsTeamsJoinLine(line))
{
break;
}
agendaLines.Add(line);
}
return string.Join(Environment.NewLine, agendaLines).Trim();
}
private static bool IsTeamsSeparator(string line)
{
var trimmed = line.Trim();
return trimmed.Length >= 8 &&
trimmed.All(character => character is '_' or '-' or '*' or ' ');
}
private static bool IsTeamsJoinLine(string line)
{
return TeamsMeetingMarkerDetector.ContainsTeamsMarker(line);
}
private static bool IsTeamsAppointment(object appointment)
{
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? "";
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body);
}
private static IReadOnlyList<string> ReadAttendees(object appointment)
{
var attendees = new List<string>();
var organizer = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Organizer"));
if (!string.IsNullOrWhiteSpace(organizer))
{
attendees.Add(organizer.Trim());
}
object? recipients = null;
try
{
recipients = OutlookClassicCom.GetProperty(appointment, "Recipients");
var count = Convert.ToInt32(OutlookClassicCom.GetProperty(recipients!, "Count"));
for (var index = 1; index <= count; index++)
{
object? recipient = null;
try
{
recipient = OutlookClassicCom.Invoke(recipients!, "Item", index);
var formatted = FormatRecipient(recipient!);
if (!string.IsNullOrWhiteSpace(formatted))
{
attendees.Add(formatted);
}
}
finally
{
OutlookClassicCom.ReleaseComObject(recipient);
}
}
}
finally
{
OutlookClassicCom.ReleaseComObject(recipients);
}
return NormalizeAttendees(attendees);
}
internal static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
{
return attendees
.Select(NormalizeAttendee)
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase)
.Select(group => group
.OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal))
.ThenBy(attendee => attendee.Length)
.First())
.ToList();
}
private static string NormalizeAttendee(string attendee)
{
var normalized = attendee.Trim();
var mailtoPrefix = "mailto:";
normalized = normalized.Replace(mailtoPrefix, "", StringComparison.OrdinalIgnoreCase);
while (normalized.Contains(" ", StringComparison.Ordinal))
{
normalized = normalized.Replace(" ", " ", StringComparison.Ordinal);
}
var emailStart = normalized.IndexOf('<', StringComparison.Ordinal);
var emailEnd = normalized.LastIndexOf('>');
if (emailStart > 0 && emailEnd > emailStart)
{
var name = normalized[..emailStart].Trim();
var email = normalized[(emailStart + 1)..emailEnd].Trim();
if (name.Equals(email, StringComparison.OrdinalIgnoreCase))
{
return email;
}
return $"{name} <{email}>";
}
return normalized;
}
private static string GetAttendeeDeduplicationKey(string attendee)
{
var emailStart = attendee.IndexOf('<', StringComparison.Ordinal);
var emailEnd = attendee.LastIndexOf('>');
if (emailStart > 0 && emailEnd > emailStart)
{
return attendee[(emailStart + 1)..emailEnd].Trim();
}
return attendee.Trim();
}
private static string FormatRecipient(object recipient)
{
var name = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Name"))?.Trim() ?? "";
var email = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Address"))?.Trim() ?? "";
if (string.IsNullOrWhiteSpace(name))
{
return email;
}
if (string.IsNullOrWhiteSpace(email) ||
email.Contains('/'))
{
return name;
}
return $"{name} <{email}>";
}
private sealed record OutlookAppointmentCandidate(
object Appointment,
DateTime Start,
@@ -0,0 +1,56 @@
namespace MeetingAssistant.Notifications;
public sealed record AgentApprovalPrompt(string Title, string Message, TimeSpan Timeout);
public interface IAgentApprovalPromptService
{
Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken);
}
public sealed class NoopAgentApprovalPromptService : IAgentApprovalPromptService
{
public Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken) => Task.FromResult(false);
}
public sealed class AgentApprovalService
{
private readonly IAgentApprovalPromptService prompts;
private readonly ILogger<AgentApprovalService> logger;
private readonly TimeSpan timeout;
public AgentApprovalService(IAgentApprovalPromptService prompts, ILogger<AgentApprovalService> logger)
: this(prompts, logger, TimeSpan.FromMinutes(10)) { }
internal AgentApprovalService(IAgentApprovalPromptService prompts, ILogger<AgentApprovalService> logger, TimeSpan timeout)
{
this.prompts = prompts;
this.logger = logger;
this.timeout = timeout;
}
public async Task<bool> ConfirmAsync(string title, string message, CancellationToken cancellationToken)
{
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
deadline.CancelAfter(timeout);
try
{
var accepted = await prompts.ConfirmAsync(new AgentApprovalPrompt(title, message, timeout), deadline.Token)
.WaitAsync(deadline.Token);
cancellationToken.ThrowIfCancellationRequested();
return accepted && !deadline.IsCancellationRequested;
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
return false;
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogWarning(exception, "Could not obtain user approval for {Title}", title);
return false;
}
finally
{
await deadline.CancelAsync();
}
}
}
@@ -0,0 +1,85 @@
#if WINDOWS
using System.Collections.Concurrent;
using CommunityToolkit.WinUI.Notifications;
using MeetingAssistant.Recording;
namespace MeetingAssistant.Notifications;
public sealed class WindowsAgentApprovalPromptService(ILogger<WindowsAgentApprovalPromptService> logger)
: IAgentApprovalPromptService, IDisposable
{
private const string Source = "summary-agent-approval";
private readonly ConcurrentDictionary<string, TaskCompletionSource<bool>> pending = new();
private readonly object gate = new();
private bool registered;
private bool disposed;
public async Task<bool> ConfirmAsync(AgentApprovalPrompt prompt, CancellationToken cancellationToken)
{
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763)) return false;
cancellationToken.ThrowIfCancellationRequested();
var id = Guid.NewGuid().ToString("N");
var response = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
try
{
lock (gate)
{
if (disposed) return false;
if (!registered)
{
ToastNotificationManagerCompat.OnActivated += OnActivated;
registered = true;
}
pending[id] = response;
new ToastContentBuilder()
.AddArgument("source", Source).AddArgument("requestId", id)
.SetToastScenario(ToastScenario.Reminder)
.SetToastDuration(ToastDuration.Long)
.AddText(prompt.Title)
.AddText(prompt.Message)
.AddButton(new ToastButton().SetContent("Yes")
.AddArgument("source", Source).AddArgument("requestId", id)
.AddArgument("response", "yes").SetBackgroundActivation())
.AddButton(new ToastButton().SetContent("No")
.AddArgument("source", Source).AddArgument("requestId", id)
.AddArgument("response", "no").SetBackgroundActivation())
.Show(toast =>
{
toast.Tag = id;
toast.Group = Source;
toast.ExpirationTime = DateTimeOffset.Now.Add(prompt.Timeout);
});
}
return await response.Task.WaitAsync(cancellationToken);
}
finally
{
pending.TryRemove(id, out _);
try { ToastNotificationManagerCompat.History.Remove(id, Source); }
catch (Exception exception) { logger.LogDebug(exception, "Could not remove approval notification {RequestId}", id); }
}
}
private void OnActivated(ToastNotificationActivatedEventArgsCompat args)
{
var arguments = NotificationActivationArguments.Parse(args.Argument);
if (arguments.GetValueOrDefault("source") != Source ||
!arguments.TryGetValue("requestId", out var id) ||
!pending.TryRemove(id, out var response)) return;
response.TrySetResult(arguments.GetValueOrDefault("response") == "yes");
}
public void Dispose()
{
lock (gate)
{
disposed = true;
if (registered) ToastNotificationManagerCompat.OnActivated -= OnActivated;
registered = false;
foreach (var response in pending.Values) response.TrySetResult(false);
pending.Clear();
}
}
}
#endif
+184 -4
View File
@@ -4,6 +4,7 @@ using MeetingAssistant.Hotkeys;
using MeetingAssistant.LaunchProfiles;
using MeetingAssistant.Logging;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Notifications;
using MeetingAssistant.Recording;
using MeetingAssistant.Screenshots;
using MeetingAssistant.Speakers;
@@ -16,9 +17,21 @@ using Microsoft.Extensions.Options;
var builder = WebApplication.CreateBuilder(args);
builder.Logging.AddProvider(new MeetingAssistantFileLoggerProvider());
builder.Services.Configure<MeetingAssistantOptions>(builder.Configuration.GetSection("MeetingAssistant"));
builder.Services.AddSingleton<IValidateOptions<MeetingAssistantOptions>, MeetingAssistantSpeakerSampleOptionsValidator>();
builder.Services.AddOptions<MeetingAssistantOptions>()
.Bind(builder.Configuration.GetSection("MeetingAssistant"))
.ValidateOnStart();
builder.Services.AddSingleton<ILaunchProfileOptionsProvider, ConfigurationLaunchProfileOptionsProvider>();
#if WINDOWS
builder.Services.AddSingleton<MicrophoneDeviceSelection>();
builder.Services.AddSingleton<WindowsMicrophoneDeviceProvider>();
builder.Services.AddSingleton<IMicrophoneDeviceProvider>(services =>
services.GetRequiredService<WindowsMicrophoneDeviceProvider>());
builder.Services.AddSingleton<IMicrophoneCaptureSourceFactory>(services =>
services.GetRequiredService<WindowsMicrophoneDeviceProvider>());
builder.Services.AddSingleton<IMicrophoneSwitchPromptService, WindowsMicrophoneSwitchPromptService>();
builder.Services.AddSingleton<IAgentApprovalPromptService, WindowsAgentApprovalPromptService>();
builder.Services.AddSingleton<MicrophoneSwitchMonitor>();
builder.Services.AddSingleton<MicrophoneAudioSource>();
builder.Services.AddSingleton<SystemAudioSource>();
builder.Services.AddSingleton<IAcousticEchoCancellerFactory, AdaptiveFilterAcousticEchoCancellerFactory>();
@@ -30,6 +43,8 @@ builder.Services.AddSingleton<IMeetingAudioSource>(services => new CompositeMeet
builder.Services.AddSingleton<IMeetingMetadataProvider, OutlookClassicMeetingMetadataProvider>();
#else
builder.Services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
builder.Services.AddSingleton<IMicrophoneSwitchPromptService, NoopMicrophoneSwitchPromptService>();
builder.Services.AddSingleton<IAgentApprovalPromptService, NoopAgentApprovalPromptService>();
builder.Services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
#endif
builder.Services.AddSingleton<ITranscriptStore, VaultTranscriptStore>();
@@ -70,7 +85,10 @@ builder.Services.AddSingleton<IActiveWindowScreenshotCapture, ActiveWindowScreen
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, UnavailableActiveWindowScreenshotCapture>();
#endif
builder.Services.AddSingleton<IScreenshotOcrClient, LiteLlmScreenshotOcrClient>();
builder.Services.AddSingleton<IMeetingScreenshotService, MeetingScreenshotService>();
builder.Services.AddSingleton<MeetingScreenshotService>();
builder.Services.AddSingleton<IMeetingScreenshotService>(sp => sp.GetRequiredService<MeetingScreenshotService>());
builder.Services.AddSingleton<IScreenshotOcrRetryRunner>(sp => sp.GetRequiredService<MeetingScreenshotService>());
builder.Services.AddSingleton<IMeetingNoteImageOcrService>(sp => sp.GetRequiredService<MeetingScreenshotService>());
builder.Services.AddDbContextFactory<SpeakerIdentityDbContext>((services, dbOptions) =>
{
var appOptions = services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value;
@@ -82,8 +100,35 @@ builder.Services.AddSingleton<ISpeakerSnippetExtractor, WavSpeakerSnippetExtract
builder.Services.AddSingleton<ISpeakerIdentityDiarizationClient, AzureSpeechSpeakerIdentityDiarizationClient>();
builder.Services.AddSingleton<ISpeakerIdentityMatchValidator, PyannoteSpeakerIdentityMatchValidator>();
builder.Services.AddSingleton<ISpeakerIdentityMatcher, AzureSpeechSpeakerIdentityMatcher>();
builder.Services.AddSingleton<ISpeakerIdentificationService, SpeakerIdentityService>();
builder.Services.AddSingleton<ISpeakerIdentityMergeService, SpeakerIdentityMergeService>();
builder.Services.AddSingleton<SpeakerIdentityService>();
builder.Services.AddSingleton<IResemblyzerVoiceEncoder, VenvResemblyzerVoiceEncoder>();
builder.Services.AddSingleton(services => new ResemblyzerVoiceClusterMatcher(
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
services.GetRequiredService<ILogger<ResemblyzerVoiceClusterMatcher>>()));
builder.Services.AddSingleton(services => new ResemblyzerVoiceVectorOutlierPruner(
services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value.SpeakerIdentification.Resemblyzer,
services.GetRequiredService<ILogger<ResemblyzerVoiceVectorOutlierPruner>>()));
builder.Services.AddSingleton<ResemblyzerSpeakerIdentificationService>();
builder.Services.AddSingleton(services =>
{
var resemblyzer = services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
.Value.SpeakerIdentification.Resemblyzer;
return resemblyzer.Enabled
? SpeakerSampleCollectionPolicy.IndependentVectors(resemblyzer.MaxVectorsPerIdentity)
: SpeakerSampleCollectionPolicy.ExistingBackend;
});
builder.Services.AddSingleton<ISpeakerIdentificationService>(services =>
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
.Value.SpeakerIdentification.Resemblyzer.Enabled
? services.GetRequiredService<ResemblyzerSpeakerIdentificationService>()
: services.GetRequiredService<SpeakerIdentityService>());
builder.Services.AddSingleton<SpeakerIdentityMergeService>();
builder.Services.AddSingleton<ResemblyzerSpeakerIdentityMergeService>();
builder.Services.AddSingleton<ISpeakerIdentityMergeService>(services =>
services.GetRequiredService<IOptions<MeetingAssistantOptions>>()
.Value.SpeakerIdentification.Resemblyzer.Enabled
? services.GetRequiredService<ResemblyzerSpeakerIdentityMergeService>()
: services.GetRequiredService<SpeakerIdentityMergeService>());
builder.Services.AddSingleton<ISpeakerIdentityAttendeeCanonicalizer, SpeakerIdentityAttendeeCanonicalizer>();
builder.Services.AddSingleton<IMeetingSummaryArtifactResolver, MeetingSummaryArtifactResolver>();
builder.Services.AddSingleton<IMeetingSummaryFailureWriter, MeetingSummaryFailureWriter>();
@@ -93,7 +138,11 @@ builder.Services.AddSingleton<IMeetingSummaryRetryRunner, MeetingSummaryRetryRun
builder.Services.AddSingleton<IMeetingWorkflowRulesProvider, FileMeetingWorkflowRulesProvider>();
builder.Services.AddSingleton<IMeetingWorkflowEngine, MeetingWorkflowEngine>();
builder.Services.AddSingleton<IWorkflowRulesEditorInstructionBuilder, WorkflowRulesEditorInstructionBuilder>();
#if WINDOWS
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, WorkflowRulesEditorSamplePlaybackQueue>();
#else
builder.Services.AddSingleton<IWorkflowRulesEditorSamplePlaybackQueue, UnavailableWorkflowRulesEditorSamplePlaybackQueue>();
#endif
builder.Services.AddSingleton<IWorkflowRulesEditorChatPipeline, WorkflowRulesEditorChatPipeline>();
builder.Services.AddTransient<WorkflowRulesEditorChatViewModel>();
#if WINDOWS
@@ -103,6 +152,9 @@ builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, WpfWorkflowRule
#else
builder.Services.AddSingleton<IWorkflowRulesEditorWindowService, NoopWorkflowRulesEditorWindowService>();
#endif
builder.Services.AddSingleton<AgentApprovalService>();
builder.Services.AddSingleton<WorkflowChangeRequestService>();
builder.Services.AddHostedService(services => services.GetRequiredService<WorkflowChangeRequestService>());
builder.Services.AddSingleton<AsrDiagnosticService>();
builder.Services.AddSingleton<ICommandRunner, ProcessCommandRunner>();
builder.Services.AddSingleton<IFunAsrBackendReadinessProbe, FunAsrWebSocketBackendReadinessProbe>();
@@ -125,6 +177,7 @@ builder.Services.AddHostedService<OfflineTranscriptionBacklogHostedService>();
builder.Services.AddHostedService<SpeakerIdentityDatabaseInitializer>();
builder.Services.AddHostedService<SpeechRecognitionPipelineHostedService>();
builder.Services.AddHostedService<PyannoteDiarizationWarmupHostedService>();
builder.Services.AddHostedService<ResemblyzerWarmupHostedService>();
#if WINDOWS
builder.Services.AddHostedService<GlobalHotkeyService>();
builder.Services.AddHostedService<UnoTaskbarIconService>();
@@ -410,6 +463,81 @@ app.MapGet("/profiles/{launchProfile}/meetings/summary/retry", async (
launchProfiles,
cancellationToken));
app.MapPost("/meetings/screenshot-ocr/retry", async (
ScreenshotOcrRetryRequest request,
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
IOptions<MeetingAssistantOptions> options,
CancellationToken cancellationToken) =>
await RetryScreenshotOcrAsync(
null,
request,
screenshotOcrRetryRunner,
options.Value,
null,
cancellationToken));
app.MapPost("/profiles/{launchProfile}/meetings/screenshot-ocr/retry", async (
string launchProfile,
ScreenshotOcrRetryRequest request,
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
IOptions<MeetingAssistantOptions> options,
ILaunchProfileOptionsProvider launchProfiles,
CancellationToken cancellationToken) =>
await RetryScreenshotOcrAsync(
launchProfile,
request,
screenshotOcrRetryRunner,
options.Value,
launchProfiles,
cancellationToken));
app.MapGet("/meetings/screenshot-ocr/retry", async (
string meetingNotePath,
string transcriptPath,
string assistantContextPath,
string summaryPath,
string screenshotPath,
string screenshotId,
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
IOptions<MeetingAssistantOptions> options,
CancellationToken cancellationToken) =>
await RetryScreenshotOcrAsync(
null,
new ScreenshotOcrRetryRequest(
meetingNotePath,
transcriptPath,
assistantContextPath,
summaryPath,
screenshotPath,
screenshotId),
screenshotOcrRetryRunner,
options.Value,
null,
cancellationToken));
app.MapGet("/profiles/{launchProfile}/meetings/screenshot-ocr/retry", async (
string launchProfile,
string meetingNotePath,
string transcriptPath,
string assistantContextPath,
string summaryPath,
string screenshotPath,
string screenshotId,
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
IOptions<MeetingAssistantOptions> options,
ILaunchProfileOptionsProvider launchProfiles,
CancellationToken cancellationToken) =>
await RetryScreenshotOcrAsync(
launchProfile,
new ScreenshotOcrRetryRequest(
meetingNotePath,
transcriptPath,
assistantContextPath,
summaryPath,
screenshotPath,
screenshotId),
screenshotOcrRetryRunner,
options.Value,
launchProfiles,
cancellationToken));
static async Task<IResult> RetrySummaryAsync(
string? launchProfile,
string? summaryPath,
@@ -438,6 +566,50 @@ static async Task<IResult> RetrySummaryAsync(
});
}
static async Task<IResult> RetryScreenshotOcrAsync(
string? launchProfile,
ScreenshotOcrRetryRequest request,
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
MeetingAssistantOptions defaultOptions,
ILaunchProfileOptionsProvider? launchProfiles,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(request.MeetingNotePath) ||
string.IsNullOrWhiteSpace(request.TranscriptPath) ||
string.IsNullOrWhiteSpace(request.AssistantContextPath) ||
string.IsNullOrWhiteSpace(request.SummaryPath) ||
string.IsNullOrWhiteSpace(request.ScreenshotPath) ||
string.IsNullOrWhiteSpace(request.ScreenshotId))
{
return Results.BadRequest(new { error = "Meeting artifact paths, screenshot path, and screenshot id are required." });
}
var profileOptions = ResolveProfileOptions(launchProfile, defaultOptions, launchProfiles);
var artifacts = new MeetingSessionArtifacts(
request.MeetingNotePath,
request.TranscriptPath,
request.AssistantContextPath,
request.SummaryPath);
var trigger = await screenshotOcrRetryRunner.TriggerOcrRetryAsync(
artifacts,
request.ScreenshotPath,
request.ScreenshotId,
profileOptions,
cancellationToken);
if (trigger is null)
{
return Results.NotFound(new { error = "No retryable screenshot OCR block was found for the requested screenshot." });
}
return Results.Accepted(value: new
{
assistantContextPath = trigger.AssistantContextPath,
screenshotPath = trigger.ScreenshotPath,
screenshotId = trigger.ScreenshotId,
state = "ocr retrying"
});
}
app.MapPost("/asr/transcribe-file", async (
AsrDiagnosticRequest request,
AsrDiagnosticService diagnostics,
@@ -544,6 +716,14 @@ public sealed record AsrDiagnosticRequest(string Path, int? NumSpeakers = null);
public sealed record SummaryRetryRequest(string SummaryPath);
public sealed record ScreenshotOcrRetryRequest(
string MeetingNotePath,
string TranscriptPath,
string AssistantContextPath,
string SummaryPath,
string ScreenshotPath,
string ScreenshotId);
public sealed record SpeakerIdentityMergeDiagnosticRequest(double? RecentDays = null);
public sealed record WorkflowConfigurationReloadResponse(string? RulesPath);
@@ -66,32 +66,22 @@ public sealed class FileOfflineTranscriptionBacklog : IOfflineTranscriptionBackl
return items;
}
public async Task CompleteAsync(string id, CancellationToken cancellationToken)
public Task CompleteAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken)
{
var folder = GetBacklogFolder(options);
var path = GetItemPath(folder, id);
var path = GetItemPath(folder, item.Id);
if (!File.Exists(path))
{
return;
}
OfflineTranscriptionBacklogItem? item = null;
try
{
item = JsonSerializer.Deserialize<OfflineTranscriptionBacklogItem>(
await File.ReadAllTextAsync(path, cancellationToken),
JsonOptions);
}
catch (JsonException exception)
{
logger.LogWarning(exception, "Could not read completed offline transcription backlog item {BacklogItemPath}", path);
return Task.CompletedTask;
}
File.Delete(path);
if (item is not null && File.Exists(item.AudioPath))
if (File.Exists(item.AudioPath))
{
DeleteCompletedAudio(item.AudioPath);
}
return Task.CompletedTask;
}
private static string GetBacklogFolder(MeetingAssistantOptions options)
@@ -6,6 +6,8 @@ public interface IMeetingInactivityPromptService
MeetingInactivityPromptRequest request,
Func<MeetingInactivityPromptResponse, CancellationToken, Task> handleResponseAsync,
CancellationToken cancellationToken);
Task DismissAllAsync(CancellationToken cancellationToken);
}
public sealed record MeetingInactivityPromptRequest(
@@ -18,9 +20,32 @@ public enum MeetingInactivityPromptResponse
{
Dismissed,
Continue,
Pause,
Stop
}
internal sealed record MeetingInactivityPromptAction(
string Content,
string ResponseArgument,
MeetingInactivityPromptResponse Response);
internal static class MeetingInactivityPromptActions
{
public static IReadOnlyList<MeetingInactivityPromptAction> All { get; } =
[
new("Yes", "stop", MeetingInactivityPromptResponse.Stop),
new("No", "continue", MeetingInactivityPromptResponse.Continue),
new("Pause transcription", "pause", MeetingInactivityPromptResponse.Pause)
];
public static MeetingInactivityPromptResponse ParseResponse(string? value)
{
return All.FirstOrDefault(action =>
action.ResponseArgument.Equals(value, StringComparison.OrdinalIgnoreCase))
?.Response ?? MeetingInactivityPromptResponse.Continue;
}
}
public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromptService
{
public Task ShowStopPromptAsync(
@@ -30,4 +55,9 @@ public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromp
{
return Task.CompletedTask;
}
public Task DismissAllAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
@@ -0,0 +1,8 @@
namespace MeetingAssistant.Recording;
public interface IMicrophoneCaptureSourceFactory
{
// The caller owns the returned source and disposes it when it implements IDisposable,
// including when capture is canceled before enumeration starts.
IMeetingAudioSource CreateCapture(MeetingAssistantOptions options);
}
@@ -0,0 +1,8 @@
namespace MeetingAssistant.Recording;
public interface IMicrophoneDeviceProvider
{
IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones();
MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options);
}
@@ -0,0 +1,12 @@
namespace MeetingAssistant.Recording;
public interface IMicrophoneSwitchPromptService
{
Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken);
}
public sealed class NoopMicrophoneSwitchPromptService : IMicrophoneSwitchPromptService
{
public Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
=> Task.FromResult(false);
}
@@ -33,6 +33,17 @@ public interface ITranscriptStore
string line,
CancellationToken cancellationToken);
async Task AppendLinesAsync(
TranscriptSession session,
IReadOnlyList<string> lines,
CancellationToken cancellationToken)
{
foreach (var line in lines)
{
await AppendLineAsync(session, line, cancellationToken);
}
}
Task ReplaceLineAsync(
TranscriptSession session,
TranscriptLineReference lineReference,
@@ -28,11 +28,13 @@ public sealed class MeetingRecordingCoordinator
private readonly ILaunchProfileOptionsProvider? launchProfiles;
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
private readonly IMeetingScreenshotService screenshotService;
private readonly IMeetingNoteImageOcrService meetingNoteImageOcrService;
private readonly IMeetingRunArtifactCleaner artifactCleaner;
private readonly IMeetingInactivityPromptService inactivityPromptService;
private readonly IMeetingInactivityClock inactivityClock;
private readonly IOfflineTranscriptionBacklog offlineTranscriptionBacklog;
private readonly MeetingAssistantOptions options;
private readonly SpeakerSampleCollectionPolicy speakerSampleCollectionPolicy;
private readonly ILogger<MeetingRecordingCoordinator> logger;
private readonly SemaphoreSlim gate = new(1, 1);
private RecordingRun? currentRun;
@@ -58,10 +60,12 @@ public sealed class MeetingRecordingCoordinator
ILaunchProfileOptionsProvider? launchProfiles = null,
IMeetingWorkflowEngine? meetingWorkflowEngine = null,
IMeetingScreenshotService? screenshotService = null,
IMeetingNoteImageOcrService? meetingNoteImageOcrService = null,
IMeetingRunArtifactCleaner? artifactCleaner = null,
IMeetingInactivityPromptService? inactivityPromptService = null,
IMeetingInactivityClock? inactivityClock = null,
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null)
IOfflineTranscriptionBacklog? offlineTranscriptionBacklog = null,
SpeakerSampleCollectionPolicy? speakerSampleCollectionPolicy = null)
{
this.audioSource = audioSource;
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
@@ -78,11 +82,14 @@ public sealed class MeetingRecordingCoordinator
this.launchProfiles = launchProfiles;
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
this.screenshotService = screenshotService ?? NoopMeetingScreenshotService.Instance;
this.meetingNoteImageOcrService = meetingNoteImageOcrService ?? NoopMeetingNoteImageOcrService.Instance;
this.artifactCleaner = artifactCleaner ?? new MeetingRunArtifactCleaner();
this.inactivityPromptService = inactivityPromptService ?? new NoopMeetingInactivityPromptService();
this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock();
this.offlineTranscriptionBacklog = offlineTranscriptionBacklog ?? NoopOfflineTranscriptionBacklog.Instance;
this.options = options.Value;
this.speakerSampleCollectionPolicy = speakerSampleCollectionPolicy
?? SpeakerSampleCollectionPolicy.ExistingBackend;
this.logger = logger;
}
@@ -93,7 +100,8 @@ public sealed class MeetingRecordingCoordinator
currentArtifacts?.AssistantContextPath,
currentArtifacts?.SummaryPath,
GetProcessState(currentRun),
currentRun?.LaunchProfileName);
currentRun?.LaunchProfileName,
IsRecording && currentRun?.IsTranscriptionPaused == true);
public MeetingSessionArtifacts? CurrentArtifacts => currentArtifacts;
@@ -142,12 +150,66 @@ public sealed class MeetingRecordingCoordinator
public async Task<RecordingStatus> StartAsync(CancellationToken cancellationToken)
{
return await StartAsync(null, cancellationToken);
return await StartAsync(null, null, suppressMetadataLookup: false, cancellationToken);
}
public async Task<RecordingStatus> StartFromPromptAsync(
MeetingMetadata? metadata,
CancellationToken cancellationToken)
{
return await StartAsync(null, metadata, suppressMetadataLookup: true, cancellationToken);
}
public async Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
MeetingMetadata metadata,
CancellationToken cancellationToken)
{
await gate.WaitAsync(cancellationToken);
try
{
var run = currentRun;
if (run is null || run.IsCaptureStopping)
{
return CurrentStatus;
}
var meetingNote = await ApplyAndPersistMeetingMetadataAsync(
run.Artifacts,
await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken),
metadata,
run.Options,
cancellationToken);
currentMeetingNote = meetingNote;
await transcriptStore.UpdateMetadataAsync(
run.Session,
run.Artifacts,
meetingNote,
cancellationToken);
run.MarkPromptMetadataAttached();
logger.LogInformation(
"Attached prompted calendar metadata to active meeting {MeetingNotePath}",
run.MeetingNotePath);
return CurrentStatus;
}
finally
{
gate.Release();
}
}
public async Task<RecordingStatus> StartAsync(
string? launchProfileName,
CancellationToken cancellationToken)
{
return await StartAsync(launchProfileName, null, suppressMetadataLookup: false, cancellationToken);
}
private async Task<RecordingStatus> StartAsync(
string? launchProfileName,
MeetingMetadata? suppliedMetadata,
bool suppressMetadataLookup,
CancellationToken cancellationToken)
{
await gate.WaitAsync(cancellationToken);
try
@@ -164,14 +226,16 @@ public sealed class MeetingRecordingCoordinator
var recordedAudio = await recordedAudioStore.CreateSessionAsync(runOptions, cancellationToken);
var assistantContextPath = GetAssistantContextPath(startedAt, runOptions);
var summaryPath = GetSummaryPath(startedAt, runOptions);
var meetingNote = MeetingNoteTemplate.Create(
title: $"Meeting {startedAt:yyyy-MM-dd HH:mm}",
startTime: startedAt,
attendees: [],
transcriptPath: currentSession.TranscriptPath,
assistantContextPath: assistantContextPath,
summaryPath: summaryPath);
currentMeetingNote = await meetingNoteStore.SaveAsync(
MeetingNoteTemplate.Create(
title: $"Meeting {startedAt:yyyy-MM-dd HH:mm}",
startTime: startedAt,
attendees: [],
transcriptPath: currentSession.TranscriptPath,
assistantContextPath: assistantContextPath,
summaryPath: summaryPath),
meetingNote,
runOptions,
cancellationToken);
currentArtifacts = new MeetingSessionArtifacts(
@@ -179,12 +243,27 @@ public sealed class MeetingRecordingCoordinator
currentSession.TranscriptPath,
assistantContextPath,
summaryPath);
await meetingArtifactStore.CreateAssistantContextAsync(currentArtifacts, currentMeetingNote, "", null, cancellationToken);
await meetingArtifactStore.CreateAssistantContextAsync(
currentArtifacts,
currentMeetingNote,
"",
null,
cancellationToken);
await meetingWorkflowEngine.RunAsync(
MeetingWorkflowEvent.Created(currentArtifacts),
runOptions,
cancellationToken);
currentMeetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
if (suppliedMetadata is not null)
{
currentMeetingNote = await ApplyAndPersistMeetingMetadataAsync(
currentArtifacts,
currentMeetingNote,
suppliedMetadata,
runOptions,
cancellationToken);
}
await meetingArtifactStore.UpdateAssistantContextMeetingAsync(
currentArtifacts,
currentMeetingNote,
@@ -215,10 +294,12 @@ public sealed class MeetingRecordingCoordinator
startedAt,
launchProfile.Name,
runOptions.SpeakerIdentification.LiveSampleBufferDuration,
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker,
speakerSampleCollectionPolicy.ResolveRetainedSampleLimit(
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker),
speakerSampleCollectionPolicy.RequireNonOverlappingSamples,
logger);
run.Task = Task.Run(() => RecordAsync(run), CancellationToken.None);
if (runOptions.Recording.InactivitySafeguard.Enabled)
if (ShouldRunInactivitySafeguard(runOptions.Recording.InactivitySafeguard))
{
_ = Task.Run(
() => RunInactivitySafeguardAsync(run),
@@ -226,9 +307,20 @@ public sealed class MeetingRecordingCoordinator
}
currentRun = run;
_ = Task.Run(
() => ApplyMeetingMetadataWhenAvailableAsync(startedAt, run),
CancellationToken.None);
if (suppliedMetadata is null && !suppressMetadataLookup)
{
_ = Task.Run(
() => ApplyMeetingMetadataWhenAvailableAsync(startedAt, run),
CancellationToken.None);
}
else
{
await TransitionMeetingAsync(
run,
AssistantContextState.Transcribing,
cancellationToken);
}
logger.LogInformation("Meeting recording started");
return CurrentStatus;
@@ -249,11 +341,14 @@ public sealed class MeetingRecordingCoordinator
public async Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
{
return await StopAsync(null, cancellationToken);
return await StopAsync(null, null, null, null, cancellationToken);
}
private async Task<RecordingStatus> StopAsync(
DateTimeOffset? inferredEndTime,
RecordingRun? expectedRun,
long? expectedActivityVersion,
DateTimeOffset? expectedPauseStartedAt,
CancellationToken cancellationToken)
{
RecordingRun run;
@@ -262,13 +357,30 @@ public sealed class MeetingRecordingCoordinator
await gate.WaitAsync(cancellationToken);
try
{
if (currentRun is null || currentRun.IsCaptureStopping)
if (currentRun is null ||
currentRun.IsCaptureStopping ||
expectedRun is not null && !ReferenceEquals(currentRun, expectedRun))
{
return CurrentStatus;
}
run = currentRun;
if (inferredEndTime is not null)
var inactivityStopClaimed = expectedActivityVersion is not null;
if (expectedActivityVersion is not null &&
(inferredEndTime is null ||
!run.TryClaimInactivityStop(expectedActivityVersion.Value, inferredEndTime.Value)))
{
return CurrentStatus;
}
var maximumPauseStopClaimed = expectedPauseStartedAt is not null;
if (expectedPauseStartedAt is not null &&
!run.TryClaimMaximumPauseStop(expectedPauseStartedAt.Value))
{
return CurrentStatus;
}
if (!inactivityStopClaimed && !maximumPauseStopClaimed && inferredEndTime is not null)
{
run.SetInferredEndTime(inferredEndTime.Value);
}
@@ -278,7 +390,7 @@ public sealed class MeetingRecordingCoordinator
{
run.Abort();
}
else
else if (!inactivityStopClaimed && !maximumPauseStopClaimed)
{
run.StopCapture();
}
@@ -304,6 +416,12 @@ public sealed class MeetingRecordingCoordinator
}
catch (TimeoutException)
{
if (run.ContextState is AssistantContextState.Summarizing or AssistantContextState.Finished or AssistantContextState.Error)
{
logger.LogInformation("Recording has reached summarization; continuing remaining processing in the background after the stop wait expired");
return CurrentStatus;
}
if (IsAzureSpeechRun(run))
{
logger.LogWarning("Timed out while draining Azure Speech transcription after recording stop; queueing offline transcription backlog item");
@@ -324,6 +442,65 @@ public sealed class MeetingRecordingCoordinator
return CurrentStatus;
}
public async Task<RecordingStatus> SetTranscriptionPausedAsync(
bool isPaused,
CancellationToken cancellationToken)
{
return await SetTranscriptionPausedAsync(isPaused, null, null, cancellationToken);
}
private async Task<RecordingStatus> SetTranscriptionPausedAsync(
bool isPaused,
RecordingRun? expectedRun,
long? expectedActivityVersion,
CancellationToken cancellationToken)
{
await gate.WaitAsync(cancellationToken);
try
{
var run = currentRun;
if (run is null ||
run.IsCaptureStopping ||
expectedRun is not null && !ReferenceEquals(run, expectedRun))
{
return CurrentStatus;
}
var changed = false;
await run.PipelineGate.WaitAsync(cancellationToken);
try
{
var changedAt = inactivityClock.Now;
changed = run.TrySetTranscriptionPaused(
isPaused,
changedAt,
expectedActivityVersion);
}
finally
{
run.PipelineGate.Release();
}
if (changed && isPaused)
{
await DismissInactivityPromptsAsync(cancellationToken);
}
if (changed)
{
logger.LogInformation(
"Meeting transcription {PauseState}",
isPaused ? "paused" : "unpaused");
}
return CurrentStatus;
}
finally
{
gate.Release();
}
}
public async Task<RecordingStatus> AbortAsync(CancellationToken cancellationToken)
{
RecordingRun run;
@@ -445,7 +622,7 @@ public sealed class MeetingRecordingCoordinator
TimeSpan.Zero,
TimeSpan.Zero,
"System",
$"Transcription profile changed to {targetProfile.Name}.");
$"Transcription profile changed to {targetProfile.Name}. Speaker recognition and identities reset.");
run.AddLiveSegment(marker);
await AppendTranscriptSegmentAsync(run, marker, cancellationToken);
@@ -576,8 +753,7 @@ public sealed class MeetingRecordingCoordinator
chunk.Channels);
}
await run.RecordedAudio.AppendAsync(chunk, run.CaptureCancellation);
await run.WriteAudioAsync(chunk, run.CaptureCancellation);
await run.RouteCapturedAudioAsync(chunk, run.CaptureCancellation);
}
logger.LogInformation(
@@ -616,7 +792,6 @@ public sealed class MeetingRecordingCoordinator
run.ResetSpeakerIdentification();
}
run.RecordTranscriptActivity(segment, inactivityClock.Now);
var sample = run.TryAddSpeakerSample(segment);
if (sample is not null)
{
@@ -633,6 +808,56 @@ public sealed class MeetingRecordingCoordinator
var relabeledSegment = run.Relabel(segment);
run.AddLiveSegment(relabeledSegment);
await AppendTranscriptSegmentAsync(run, relabeledSegment, cancellationToken);
run.RecordTranscriptActivity(segment, inactivityClock.Now);
if (!string.IsNullOrWhiteSpace(segment.Text))
{
// Switching profiles holds the coordinator gate while draining this reader.
// Notification cleanup must not make transcript draining wait for that gate.
_ = DismissInactivityPromptsForRunAsync(run, run.CaptureCancellation);
}
}
}
private async Task DismissInactivityPromptsForRunAsync(
RecordingRun run,
CancellationToken cancellationToken)
{
try
{
await gate.WaitAsync(cancellationToken);
try
{
if (!ReferenceEquals(currentRun, run) || run.IsCaptureStopping)
{
return;
}
await DismissInactivityPromptsAsync(cancellationToken);
}
finally
{
gate.Release();
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Pending cleanup is no longer needed after this run stops capturing.
}
}
private async Task DismissInactivityPromptsAsync(CancellationToken cancellationToken)
{
try
{
await inactivityPromptService.DismissAllAsync(cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to dismiss transcript inactivity prompts");
}
}
@@ -642,14 +867,23 @@ public sealed class MeetingRecordingCoordinator
CancellationToken cancellationToken)
{
var formatted = TranscriptLineFormatter.Format(segment);
var lineReference = segment.ReplacesMarkerId is { } replacesMarkerId &&
run.TryTakeTranscriptMarker(replacesMarkerId, out var markerReference)
? await ReplaceTranscriptMarkerAsync(run, markerReference, formatted, cancellationToken)
: segment.Kind == TranscriptionSegmentKind.Marker &&
segment.MarkerId is { } existingMarkerId &&
run.TryTakeTranscriptMarker(existingMarkerId, out var existingMarkerReference)
? await ReplaceTranscriptMarkerAsync(run, existingMarkerReference, formatted, cancellationToken)
: await transcriptStore.AppendLineAsync(run.Session, formatted.Line, cancellationToken);
TranscriptLineReference lineReference;
if (segment.ReplacesMarkerId is { } replacesMarkerId &&
run.TryTakeTranscriptMarker(replacesMarkerId, out var markerReference))
{
lineReference = await ReplaceTranscriptMarkerAsync(run, markerReference, formatted, cancellationToken);
}
else if (segment.Kind == TranscriptionSegmentKind.Marker &&
segment.MarkerId is { } existingMarkerId &&
run.TryTakeTranscriptMarker(existingMarkerId, out var existingMarkerReference))
{
lineReference = await ReplaceTranscriptMarkerAsync(run, existingMarkerReference, formatted, cancellationToken);
}
else
{
lineReference = await transcriptStore.AppendLineAsync(run.Session, formatted.Line, cancellationToken);
}
if (segment.Kind == TranscriptionSegmentKind.Marker && segment.MarkerId is { } markerId)
{
run.SetTranscriptMarker(markerId, lineReference);
@@ -751,7 +985,10 @@ public sealed class MeetingRecordingCoordinator
private async Task RunInactivitySafeguardAsync(RecordingRun run)
{
var safeguardOptions = run.Options.Recording.InactivitySafeguard;
var promptThresholds = GetInactivityPromptThresholds(safeguardOptions);
var promptThresholds = safeguardOptions.Enabled
? GetInactivityPromptThresholds(safeguardOptions)
: [];
var checkInterval = GetInactivitySafeguardCheckInterval(safeguardOptions);
var promptedThresholds = new HashSet<TimeSpan>();
var lastActivityVersion = run.GetTranscriptActivitySnapshot().ActivityVersion;
@@ -759,7 +996,7 @@ public sealed class MeetingRecordingCoordinator
"Recording inactivity safeguard started with prompt thresholds {PromptThresholds}, auto-stop {AutoStopAfter}, check interval {CheckInterval}",
string.Join(", ", promptThresholds),
safeguardOptions.AutoStopAfter,
safeguardOptions.CheckInterval);
checkInterval);
try
{
@@ -775,9 +1012,38 @@ public sealed class MeetingRecordingCoordinator
snapshot.LastTranscriptActivityAt);
}
var pauseStartedAt = snapshot.TranscriptionPausedAt;
if (pauseStartedAt is not null)
{
var maximumPauseDuration = safeguardOptions.MaximumPauseDuration;
if (maximumPauseDuration > TimeSpan.Zero &&
inactivityClock.Now - pauseStartedAt.Value >= maximumPauseDuration)
{
logger.LogWarning(
"Recording inactivity safeguard stopping meeting after transcription remained paused for {PauseDuration}",
maximumPauseDuration);
var stopStatus = await StopAsync(
null,
run,
null,
pauseStartedAt,
CancellationToken.None);
if (!stopStatus.IsRecording)
{
return;
}
}
await inactivityClock.DelayAsync(
checkInterval,
run.CaptureCancellation);
continue;
}
var now = inactivityClock.Now;
var inactivityDuration = now - snapshot.LastTranscriptActivityAt;
if (safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
if (safeguardOptions.Enabled &&
safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
inactivityDuration >= safeguardOptions.AutoStopAfter)
{
var inferredEndTime = run.GetInactivitySafeguardEndTime(safeguardOptions.InferredEndPadding);
@@ -785,8 +1051,16 @@ public sealed class MeetingRecordingCoordinator
"Recording inactivity safeguard auto-stopping meeting after {InactivityDuration} without transcript text; inferred end time {InferredEndTime}",
inactivityDuration,
inferredEndTime);
await StopAsync(inferredEndTime, CancellationToken.None);
return;
var stopStatus = await StopAsync(
inferredEndTime,
run,
snapshot.ActivityVersion,
null,
CancellationToken.None);
if (!stopStatus.IsRecording)
{
return;
}
}
foreach (var threshold in promptThresholds)
@@ -803,6 +1077,7 @@ public sealed class MeetingRecordingCoordinator
inactivityDuration,
threshold,
inferredEndTime);
var promptActivityVersion = snapshot.ActivityVersion;
await inactivityPromptService.ShowStopPromptAsync(
new MeetingInactivityPromptRequest(
inactivityDuration,
@@ -818,16 +1093,36 @@ public sealed class MeetingRecordingCoordinator
if (response == MeetingInactivityPromptResponse.Stop)
{
await StopAsync(inferredEndTime, CancellationToken.None);
await StopAsync(
inferredEndTime,
run,
promptActivityVersion,
null,
CancellationToken.None);
}
else if (response == MeetingInactivityPromptResponse.Pause)
{
await SetTranscriptionPausedAsync(
true,
run,
promptActivityVersion,
CancellationToken.None);
}
},
run.CaptureCancellation);
var currentActivity = run.GetTranscriptActivitySnapshot();
if (currentActivity.IsTranscriptionPaused ||
currentActivity.ActivityVersion != promptActivityVersion)
{
await DismissInactivityPromptsAsync(run.CaptureCancellation);
}
break;
}
await inactivityClock.DelayAsync(
GetInactivitySafeguardCheckInterval(safeguardOptions),
checkInterval,
run.CaptureCancellation);
}
}
@@ -895,11 +1190,11 @@ public sealed class MeetingRecordingCoordinator
try
{
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken);
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote);
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote, samples);
if (checkpoint is null)
{
logger.LogInformation(
"Skipping live speaker identity matching because no new unmapped sample speakers or attendee changes were found");
"Skipping live speaker identity matching because no new eligible speaker evidence or attendee changes were found");
return;
}
@@ -955,7 +1250,7 @@ public sealed class MeetingRecordingCoordinator
return;
}
if (run.IsAborted)
if (run.IsAborted || run.HasAttachedPromptMetadata)
{
return;
}
@@ -963,25 +1258,21 @@ public sealed class MeetingRecordingCoordinator
await gate.WaitAsync(CancellationToken.None);
try
{
if (run.IsAborted)
if (run.IsAborted || run.HasAttachedPromptMetadata)
{
return;
}
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, CancellationToken.None);
await ApplyMeetingMetadataAsync(meetingNote, metadata, run.Options, CancellationToken.None);
meetingNote = await meetingNoteStore.SaveAsync(meetingNote, run.Options, CancellationToken.None);
var meetingNote = await ApplyAndPersistMeetingMetadataAsync(
run.Artifacts,
await meetingNoteStore.ReadAsync(run.MeetingNotePath, CancellationToken.None),
metadata,
run.Options,
CancellationToken.None);
if (currentMeetingNote?.Path == meetingNote.Path)
{
currentMeetingNote = meetingNote;
}
await meetingArtifactStore.UpdateAssistantContextMetadataAsync(
run.Artifacts,
meetingNote,
metadata.Agenda,
metadata.ScheduledEnd,
CancellationToken.None);
logger.LogInformation(
"Applied Outlook meeting metadata to {MeetingNotePath}",
run.MeetingNotePath);
@@ -1021,6 +1312,7 @@ public sealed class MeetingRecordingCoordinator
}
private async Task ApplyMeetingMetadataAsync(
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote,
MeetingMetadata metadata,
MeetingAssistantOptions options,
@@ -1043,10 +1335,65 @@ public sealed class MeetingRecordingCoordinator
return;
}
meetingNote.Frontmatter.Attendees = await CanonicalizeAttendeesAsync(metadata.Attendees, cancellationToken);
var canonicalized = await CanonicalizeAttendeesAsync(metadata.Attendees, cancellationToken);
meetingNote.Frontmatter.Attendees = await TransformAttendeesAsync(
artifacts,
canonicalized,
options,
cancellationToken);
}
}
private async Task<MeetingNote> ApplyAndPersistMeetingMetadataAsync(
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote,
MeetingMetadata metadata,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
await ApplyMeetingMetadataAsync(
artifacts,
meetingNote,
metadata,
options,
cancellationToken);
var savedMeetingNote = await meetingNoteStore.SaveAsync(
meetingNote,
options,
cancellationToken);
await meetingArtifactStore.UpdateAssistantContextMetadataAsync(
artifacts,
savedMeetingNote,
metadata.Agenda,
metadata.ScheduledEnd,
cancellationToken);
return savedMeetingNote;
}
private async Task<List<string>> TransformAttendeesAsync(
MeetingSessionArtifacts artifacts,
IReadOnlyList<string> attendees,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var transformed = new List<string>();
foreach (var attendee in attendees)
{
var value = await TransformAttendeeAsync(artifacts, attendee, options, cancellationToken);
var storageValue = string.Equals(value, attendee, StringComparison.Ordinal)
? attendee.Trim()
: NormalizeAttendeeName(value);
var normalized = NormalizeAttendeeName(storageValue);
if (!string.IsNullOrWhiteSpace(normalized) &&
!transformed.Contains(normalized, StringComparer.OrdinalIgnoreCase))
{
transformed.Add(storageValue);
}
}
return transformed;
}
private async Task<List<string>> CanonicalizeAttendeesAsync(
IReadOnlyList<string> attendees,
CancellationToken cancellationToken)
@@ -1361,8 +1708,17 @@ public sealed class MeetingRecordingCoordinator
continue;
}
var transformedDisplayName = await TransformAttendeeAsync(
artifacts,
match.DisplayName.Trim(),
runOptions,
cancellationToken);
var displayName = string.Equals(transformedDisplayName, match.DisplayName.Trim(), StringComparison.Ordinal)
? match.DisplayName.Trim()
: NormalizeAttendeeName(transformedDisplayName);
var acceptedNames = match.AcceptedNames
.Append(match.DisplayName)
.Append(displayName)
.Select(NormalizeAttendeeName)
.Where(name => !string.IsNullOrWhiteSpace(name))
.Distinct(StringComparer.OrdinalIgnoreCase)
@@ -1381,7 +1737,6 @@ public sealed class MeetingRecordingCoordinator
continue;
}
var displayName = match.DisplayName.Trim();
meetingNote.Frontmatter.Attendees.Add(displayName);
existingNames.Add(NormalizeAttendeeName(displayName));
changed = true;
@@ -1441,6 +1796,18 @@ public sealed class MeetingRecordingCoordinator
return MeetingAttendeeNames.NormalizeDisplayName(attendee);
}
private async Task<string> TransformAttendeeAsync(
MeetingSessionArtifacts artifacts,
string attendee,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return await meetingWorkflowEngine.TransformAttendeeAsync(
MeetingWorkflowEvent.AttendeeAdded(artifacts, attendee),
options,
cancellationToken);
}
private static bool RemoveDuplicateAcceptedAliases(
List<string> attendees,
string displayName,
@@ -1505,6 +1872,10 @@ public sealed class MeetingRecordingCoordinator
try
{
await meetingNoteImageOcrService.ProcessMeetingNoteImageEmbedsAsync(
run.Artifacts,
run.Options,
cancellationToken);
await screenshotService.WaitForPendingOcrAsync(
run.Artifacts,
run.Options.Screenshots.Ocr.GetEffectiveTimeout(),
@@ -1651,6 +2022,11 @@ public sealed class MeetingRecordingCoordinator
.ToList();
}
private static bool ShouldRunInactivitySafeguard(RecordingInactivitySafeguardOptions options)
{
return options.Enabled || options.MaximumPauseDuration > TimeSpan.Zero;
}
private static TimeSpan GetInactivitySafeguardCheckInterval(
RecordingInactivitySafeguardOptions options)
{
@@ -1707,6 +2083,8 @@ public sealed class MeetingRecordingCoordinator
private TimeSpan? lastTranscriptSegmentEnd;
private long transcriptActivityVersion;
private DateTimeOffset? inferredEndTime;
private bool isTranscriptionPaused;
private DateTimeOffset? transcriptionPausedAt;
public RecordingRun(
CancellationTokenSource captureCancellation,
@@ -1722,6 +2100,7 @@ public sealed class MeetingRecordingCoordinator
string launchProfileName,
TimeSpan liveSampleBufferDuration,
int maxSpeakerSamples,
bool requireNonOverlappingSpeakerSamples,
ILogger logger)
{
CaptureCancellationSource = captureCancellation;
@@ -1742,8 +2121,9 @@ public sealed class MeetingRecordingCoordinator
liveSampleBufferDuration,
maxSpeakerSamples,
options.SpeakerIdentification.MinimumSampleSpeechDuration,
options.SpeakerIdentification.MaximumSampleSegmentGap,
logger);
options.SpeakerIdentification.MaximumSampleDuration,
logger,
requireNonOverlappingSpeakerSamples);
}
public CancellationTokenSource CaptureCancellationSource { get; }
@@ -1788,6 +2168,19 @@ public sealed class MeetingRecordingCoordinator
public bool HasSwitchedProfile { get; private set; }
public bool HasAttachedPromptMetadata { get; private set; }
public bool IsTranscriptionPaused
{
get
{
lock (transcriptActivityGate)
{
return isTranscriptionPaused;
}
}
}
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
public DateTimeOffset? InferredEndTime
@@ -1806,6 +2199,42 @@ public sealed class MeetingRecordingCoordinator
CaptureCancellationSource.Cancel();
}
public void MarkPromptMetadataAttached()
{
HasAttachedPromptMetadata = true;
}
public bool TrySetTranscriptionPaused(
bool isPaused,
DateTimeOffset changedAt,
long? expectedActivityVersion)
{
lock (transcriptActivityGate)
{
if (expectedActivityVersion is not null &&
transcriptActivityVersion != expectedActivityVersion.Value)
{
return false;
}
if (isTranscriptionPaused == isPaused)
{
return false;
}
isTranscriptionPaused = isPaused;
transcriptionPausedAt = isPaused ? changedAt : null;
if (!isPaused)
{
lastTranscriptActivityAt = changedAt;
}
transcriptActivityVersion++;
return true;
}
}
public void Abort()
{
IsAborted = true;
@@ -1834,13 +2263,17 @@ public sealed class MeetingRecordingCoordinator
return string.Equals(LaunchProfileName, launchProfileName, StringComparison.OrdinalIgnoreCase);
}
public async ValueTask WriteAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
public async ValueTask RouteCapturedAudioAsync(AudioChunk chunk, CancellationToken cancellationToken)
{
await PipelineGate.WaitAsync(cancellationToken);
try
{
AppendAudio(chunk);
await Pipeline.WriteAsync(chunk, cancellationToken);
var routedChunk = IsTranscriptionPaused
? chunk with { Pcm = new byte[chunk.Pcm.Length] }
: chunk;
await RecordedAudio.AppendAsync(routedChunk, cancellationToken);
AppendAudio(routedChunk);
await Pipeline.WriteAsync(routedChunk, cancellationToken);
}
finally
{
@@ -2024,7 +2457,9 @@ public sealed class MeetingRecordingCoordinator
return new TranscriptActivitySnapshot(
lastTranscriptActivityAt,
lastTranscriptSegmentEnd,
transcriptActivityVersion);
transcriptActivityVersion,
isTranscriptionPaused,
transcriptionPausedAt);
}
}
@@ -2044,6 +2479,35 @@ public sealed class MeetingRecordingCoordinator
}
}
public bool TryClaimInactivityStop(long expectedActivityVersion, DateTimeOffset endTime)
{
lock (transcriptActivityGate)
{
if (isTranscriptionPaused || transcriptActivityVersion != expectedActivityVersion)
{
return false;
}
inferredEndTime = endTime;
StopCapture();
return true;
}
}
public bool TryClaimMaximumPauseStop(DateTimeOffset expectedPauseStartedAt)
{
lock (transcriptActivityGate)
{
if (!isTranscriptionPaused || transcriptionPausedAt != expectedPauseStartedAt)
{
return false;
}
StopCapture();
return true;
}
}
public void AppendAudio(AudioChunk chunk)
{
speakerSampleCollector.AppendAudio(chunk);
@@ -2067,9 +2531,11 @@ public sealed class MeetingRecordingCoordinator
return speakerSampleCollector.Snapshot();
}
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(MeetingNote meetingNote)
public LiveIdentificationCheckpoint? CreateLiveIdentificationCheckpoint(
MeetingNote meetingNote,
IReadOnlyList<SpeakerAudioSample> samples)
{
var speakers = GetUnmappedSampleSpeakers();
var speakers = GetUnmappedSampleSpeakers(samples);
if (speakers.Count == 0)
{
return null;
@@ -2077,7 +2543,11 @@ public sealed class MeetingRecordingCoordinator
var checkpoint = new LiveIdentificationCheckpoint(
speakers,
BuildAttendeeSignature(meetingNote));
BuildAttendeeSignature(meetingNote),
Options.SpeakerIdentification.Resemblyzer.Enabled
? speakers.Select(speaker => samples.Count(sample =>
string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))).ToArray()
: []);
lock (liveIdentificationGate)
{
return lastLiveIdentificationCheckpoint?.Matches(checkpoint) == true
@@ -2135,9 +2605,9 @@ public sealed class MeetingRecordingCoordinator
StringComparer.OrdinalIgnoreCase);
}
private IReadOnlyList<string> GetUnmappedSampleSpeakers()
private IReadOnlyList<string> GetUnmappedSampleSpeakers(IReadOnlyList<SpeakerAudioSample> samples)
{
return GetSpeakerSamplesSnapshot()
return samples
.Select(sample => sample.Speaker)
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
.Where(speaker => !string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase))
@@ -2173,7 +2643,9 @@ public sealed class MeetingRecordingCoordinator
public sealed record TranscriptActivitySnapshot(
DateTimeOffset LastTranscriptActivityAt,
TimeSpan? LastTranscriptSegmentEnd,
long ActivityVersion);
long ActivityVersion,
bool IsTranscriptionPaused,
DateTimeOffset? TranscriptionPausedAt);
private static int StateRank(AssistantContextState state)
{
@@ -2191,11 +2663,13 @@ public sealed class MeetingRecordingCoordinator
public sealed record LiveIdentificationCheckpoint(
IReadOnlyList<string> Speakers,
string AttendeeSignature)
string AttendeeSignature,
IReadOnlyList<int> SampleCounts)
{
public bool Matches(LiveIdentificationCheckpoint other)
{
return string.Equals(AttendeeSignature, other.AttendeeSignature, StringComparison.Ordinal) &&
SampleCounts.SequenceEqual(other.SampleCounts) &&
Speakers.Count == other.Speakers.Count &&
Speakers.SequenceEqual(other.Speakers, StringComparer.OrdinalIgnoreCase);
}
@@ -2228,7 +2702,8 @@ public sealed record RecordingStatus(
string? AssistantContextPath,
string? SummaryPath,
RecordingProcessState State = RecordingProcessState.Idle,
string? LaunchProfile = null);
string? LaunchProfile = null,
bool IsPaused = false);
public enum RecordingProcessState
{
@@ -0,0 +1,234 @@
using System.Runtime.CompilerServices;
namespace MeetingAssistant.Recording;
public sealed class MicrophoneAudioSource : IMeetingAudioSource
{
private static readonly TimeSpan DefaultRecoveryDelay = TimeSpan.FromSeconds(1);
private readonly IMicrophoneCaptureSourceFactory captureSources;
private readonly ILogger<MicrophoneAudioSource> logger;
private readonly TimeSpan recoveryDelay;
private readonly MicrophoneSwitchMonitor? switchMonitor;
private readonly MicrophoneDeviceSelection? selection;
public MicrophoneAudioSource(
IMicrophoneCaptureSourceFactory captureSources,
ILogger<MicrophoneAudioSource> logger,
MicrophoneSwitchMonitor? switchMonitor = null,
MicrophoneDeviceSelection? selection = null)
: this(captureSources, logger, DefaultRecoveryDelay, switchMonitor, selection)
{
}
internal MicrophoneAudioSource(
IMicrophoneCaptureSourceFactory captureSources,
ILogger<MicrophoneAudioSource> logger,
TimeSpan recoveryDelay,
MicrophoneSwitchMonitor? switchMonitor = null,
MicrophoneDeviceSelection? selection = null)
{
this.captureSources = captureSources;
this.logger = logger;
this.recoveryDelay = recoveryDelay;
this.switchMonitor = switchMonitor;
this.selection = selection;
}
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
{
return CaptureAsync(new MeetingAssistantOptions(), cancellationToken);
}
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
MeetingAssistantOptions options,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
using var session = new CaptureSession(selection, cancellationToken);
var monitoring = switchMonitor?.RunAsync(session.SwitchTo, session.Token) ?? Task.CompletedTask;
try
{
await foreach (var chunk in CaptureWithRecoveryAsync(options, session, session.Token))
{
yield return chunk;
}
}
finally
{
session.Stop();
await monitoring;
}
}
private async IAsyncEnumerable<AudioChunk> CaptureWithRecoveryAsync(
MeetingAssistantOptions options,
CaptureSession session,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
var failedAttempts = 0;
while (!cancellationToken.IsCancellationRequested)
{
var captureToken = session.BeginCapture();
IMeetingAudioSource? captureSource = null;
IAsyncEnumerator<AudioChunk>? capture = null;
Exception? failure = null;
try
{
captureSource = captureSources.CreateCapture(options);
capture = captureSource
.CaptureAsync(options, captureToken)
.GetAsyncEnumerator(captureToken);
}
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
{
}
catch (Exception exception)
{
failure = exception;
}
using var captureOwner = captureSource as IDisposable;
if (cancellationToken.IsCancellationRequested)
{
yield break;
}
if (capture is not null)
{
try
{
while (!captureToken.IsCancellationRequested)
{
var hasNext = false;
try
{
hasNext = await capture.MoveNextAsync();
}
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
{
}
catch (Exception exception)
{
failure = exception;
}
if (captureToken.IsCancellationRequested || failure is not null || !hasNext)
{
break;
}
if (failedAttempts > 0)
{
logger.LogInformation(
"Microphone capture recovered after {FailedAttemptCount} failed attempt(s)",
failedAttempts);
failedAttempts = 0;
}
yield return capture.Current;
}
}
finally
{
try
{
await capture.DisposeAsync();
}
catch (OperationCanceledException) when (captureToken.IsCancellationRequested)
{
}
catch (Exception exception)
{
failure ??= exception;
}
}
}
var switchRequested = captureToken.IsCancellationRequested;
session.EndCapture();
if (cancellationToken.IsCancellationRequested)
{
yield break;
}
if (switchRequested)
{
logger.LogInformation("Replacing microphone capture after an accepted device-switch notification");
continue;
}
failedAttempts++;
logger.LogWarning(
failure,
"Microphone capture stopped unexpectedly; re-resolving an available microphone in {RecoveryDelay}",
recoveryDelay);
if (!await WaitForRecoveryAsync(cancellationToken))
{
yield break;
}
}
}
private async Task<bool> WaitForRecoveryAsync(CancellationToken cancellationToken)
{
try
{
await Task.Delay(recoveryDelay, cancellationToken);
return true;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return false;
}
}
private sealed class CaptureSession(MicrophoneDeviceSelection? selection, CancellationToken cancellationToken) : IDisposable
{
private readonly object gate = new();
private readonly CancellationTokenSource lifetime = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
private CancellationTokenSource? capture;
public CancellationToken Token => lifetime.Token;
public CancellationToken BeginCapture()
{
lock (gate)
{
capture = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token);
return capture.Token;
}
}
public void EndCapture()
{
lock (gate)
{
capture?.Dispose();
capture = null;
}
}
public void SwitchTo(MicrophoneDevice device, CancellationToken promptCancellation)
{
lock (gate)
{
if (lifetime.IsCancellationRequested || promptCancellation.IsCancellationRequested || selection is null) return;
selection.Select(device.Id);
capture?.Cancel();
}
}
public void Stop()
{
lock (gate) lifetime.Cancel();
}
public void Dispose()
{
Stop();
EndCapture();
lifetime.Dispose();
}
}
}
@@ -0,0 +1,9 @@
namespace MeetingAssistant.Recording;
public sealed record MicrophoneDevice(
string Id,
string Name);
public sealed record MicrophoneDeviceSnapshot(
IReadOnlyList<MicrophoneDevice> Available,
MicrophoneDevice? Current);
@@ -0,0 +1,56 @@
namespace MeetingAssistant.Recording;
public sealed class MicrophoneDeviceSelection
{
private readonly object sync = new();
private string? selectedDeviceId;
public string? SelectedDeviceId
{
get
{
lock (sync)
{
return selectedDeviceId;
}
}
}
public void Select(string deviceId)
{
lock (sync)
{
selectedDeviceId = string.IsNullOrWhiteSpace(deviceId)
? null
: deviceId.Trim();
}
}
public MicrophoneDevice? Resolve(
string? configuredDeviceId,
MicrophoneDevice? defaultDevice,
IReadOnlyList<MicrophoneDevice> availableDevices)
{
var selected = SelectedDeviceId;
return FindById(availableDevices, selected) ??
FindById(availableDevices, configuredDeviceId) ??
FindById(availableDevices, defaultDevice?.Id) ??
availableDevices.FirstOrDefault() ??
defaultDevice;
}
private static MicrophoneDevice? FindById(
IReadOnlyList<MicrophoneDevice> devices,
string? id)
{
if (string.IsNullOrWhiteSpace(id))
{
return null;
}
return devices.FirstOrDefault(device => string.Equals(
device.Id,
id.Trim(),
StringComparison.OrdinalIgnoreCase));
}
}
@@ -0,0 +1,114 @@
namespace MeetingAssistant.Recording;
public sealed class MicrophoneSwitchMonitor
{
private readonly IMicrophoneDeviceProvider devices;
private readonly IMicrophoneSwitchPromptService prompts;
private readonly ILogger<MicrophoneSwitchMonitor> logger;
private readonly TimeSpan pollInterval;
private readonly TimeSpan promptLifetime;
public MicrophoneSwitchMonitor(
IMicrophoneDeviceProvider devices,
IMicrophoneSwitchPromptService prompts,
ILogger<MicrophoneSwitchMonitor> logger)
: this(devices, prompts, logger, TimeSpan.FromSeconds(1), TimeSpan.FromMinutes(1))
{
}
internal MicrophoneSwitchMonitor(
IMicrophoneDeviceProvider devices,
IMicrophoneSwitchPromptService prompts,
ILogger<MicrophoneSwitchMonitor> logger,
TimeSpan pollInterval,
TimeSpan promptLifetime)
{
this.devices = devices;
this.prompts = prompts;
this.logger = logger;
this.pollInterval = pollInterval;
this.promptLifetime = promptLifetime;
}
public async Task RunAsync(Action<MicrophoneDevice, CancellationToken> switchMicrophone, CancellationToken cancellationToken)
{
var known = (ReadDevices() ?? []).Select(device => device.Id).ToHashSet(StringComparer.OrdinalIgnoreCase);
var pending = new Dictionary<string, PendingPrompt>(StringComparer.OrdinalIgnoreCase);
try
{
while (true)
{
await Task.Delay(pollInterval, cancellationToken);
var available = ReadDevices();
if (available is null) continue;
var availableIds = available.Select(device => device.Id).ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var (id, prompt) in pending.ToArray())
{
if (!availableIds.Contains(id)) await prompt.Cancellation.CancelAsync();
if (prompt.Task.IsCompleted)
{
prompt.Cancellation.Dispose();
pending.Remove(id);
}
}
foreach (var device in available)
{
if (known.Add(device.Id))
{
var promptCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
promptCancellation.CancelAfter(promptLifetime);
pending.Add(device.Id, new PendingPrompt(promptCancellation,
OfferAsync(device, switchMicrophone, promptCancellation.Token)));
}
}
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
finally
{
foreach (var prompt in pending.Values) await prompt.Cancellation.CancelAsync();
await Task.WhenAll(pending.Values.Select(prompt => prompt.Task));
foreach (var prompt in pending.Values) prompt.Cancellation.Dispose();
}
}
private async Task OfferAsync(MicrophoneDevice device, Action<MicrophoneDevice, CancellationToken> switchMicrophone,
CancellationToken cancellationToken)
{
try
{
if (await prompts.ConfirmSwitchAsync(device, cancellationToken)
&& !cancellationToken.IsCancellationRequested
&& ReadDevices()?.Any(current => current.Id.Equals(device.Id, StringComparison.OrdinalIgnoreCase)) == true
&& !cancellationToken.IsCancellationRequested)
{
switchMicrophone(device, cancellationToken);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not offer microphone {MicrophoneName}", device.Name);
}
}
private IReadOnlyList<MicrophoneDevice>? ReadDevices()
{
try
{
return devices.GetAvailableMicrophones();
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not check for newly available microphones");
return null;
}
}
private sealed record PendingPrompt(CancellationTokenSource Cancellation, Task Task);
}
@@ -4,40 +4,34 @@ using NAudio.Wave;
namespace MeetingAssistant.Recording;
public sealed class MicrophoneAudioSource : IMeetingAudioSource
internal sealed class NaudioCaptureAudioSource : IMeetingAudioSource, IDisposable
{
private readonly ILogger<MicrophoneAudioSource> logger;
private readonly IWaveIn capture;
private readonly string sourceName;
private readonly ILogger logger;
private int disposed;
public MicrophoneAudioSource(ILogger<MicrophoneAudioSource> logger)
public NaudioCaptureAudioSource(
IWaveIn capture,
string sourceName,
ILogger logger)
{
this.capture = capture;
this.sourceName = sourceName;
this.logger = logger;
}
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
{
return CaptureAsync(new MeetingAssistantOptions(), cancellationToken);
return CaptureWith(cancellationToken);
}
public IAsyncEnumerable<AudioChunk> CaptureAsync(
MeetingAssistantOptions options,
CancellationToken cancellationToken)
public void Dispose()
{
return CaptureAsync(new WasapiCapture(), options, cancellationToken);
if (Interlocked.Exchange(ref disposed, 1) == 0) capture.Dispose();
}
private IAsyncEnumerable<AudioChunk> CaptureAsync(
IWaveIn capture,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
capture.WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels);
return CaptureWith(capture, "microphone", logger, cancellationToken);
}
internal static async IAsyncEnumerable<AudioChunk> CaptureWith(
IWaveIn capture,
string sourceName,
ILogger logger,
private async IAsyncEnumerable<AudioChunk> CaptureWith(
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
var channel = Channel.CreateUnbounded<AudioChunk>();
@@ -92,7 +86,7 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
}
finally
{
capture.Dispose();
Dispose();
}
}
}
@@ -120,6 +114,6 @@ public sealed class SystemAudioSource : IMeetingAudioSource
WaveFormat = new WaveFormat(options.Recording.SampleRate, 16, options.Recording.Channels)
};
return MicrophoneAudioSource.CaptureWith(capture, "system", logger, cancellationToken);
return new NaudioCaptureAudioSource(capture, "system", logger).CaptureAsync(cancellationToken);
}
}
@@ -6,7 +6,7 @@ public interface IOfflineTranscriptionBacklog
Task<IReadOnlyList<OfflineTranscriptionBacklogItem>> ListAsync(CancellationToken cancellationToken);
Task CompleteAsync(string id, CancellationToken cancellationToken);
Task CompleteAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken);
}
public sealed record OfflineTranscriptionBacklogItem(
@@ -38,7 +38,7 @@ public sealed class NoopOfflineTranscriptionBacklog : IOfflineTranscriptionBackl
return Task.FromResult<IReadOnlyList<OfflineTranscriptionBacklogItem>>([]);
}
public Task CompleteAsync(string id, CancellationToken cancellationToken)
public Task CompleteAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
@@ -4,14 +4,11 @@ using MeetingAssistant.Summary;
using MeetingAssistant.Transcription;
using MeetingAssistant.Workflow;
using Microsoft.Extensions.Options;
using NAudio.Wave;
namespace MeetingAssistant.Recording;
public sealed class OfflineTranscriptionBacklogProcessor
{
private const int MaxChunkDurationMilliseconds = 1000;
private readonly IOfflineTranscriptionBacklog backlog;
private readonly ISpeechRecognitionPipelineFactory pipelineFactory;
private readonly ITranscriptStore transcriptStore;
@@ -72,7 +69,7 @@ public sealed class OfflineTranscriptionBacklogProcessor
try
{
await ProcessAsync(item, cancellationToken);
await backlog.CompleteAsync(item.Id, cancellationToken);
await backlog.CompleteAsync(item, cancellationToken);
logger.LogInformation(
"Completed offline transcription backlog item {BacklogItemId} for recording {RecordingPath}",
item.Id,
@@ -120,8 +117,25 @@ public sealed class OfflineTranscriptionBacklogProcessor
pipelineOptions,
cancellationToken);
if (!finishedSegments.Any(segment =>
segment.Kind == TranscriptionSegmentKind.Speech && !string.IsNullOrWhiteSpace(segment.Text)))
{
throw new InvalidOperationException(
"Offline transcription replay returned no speech text; preserving the existing transcript and recovery files.");
}
var lines = await TransformTranscriptLinesAsync(artifacts, runOptions, finishedSegments, cancellationToken);
await transcriptStore.ReplaceLinesAsync(session, lines, cancellationToken);
await transcriptStore.AppendLinesAsync(session,
[
"",
"## Offline transcription recovery",
"",
"> The following transcript was recovered by replaying the entire recording. "
+ "It may duplicate earlier transcript passages. "
+ "Treat repeated passages as the same discussion when summarizing.",
"",
.. lines
], cancellationToken);
meetingNote.Frontmatter.EndTime = item.InferredEndTime ?? DateTimeOffset.Now;
var completedMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, runOptions, cancellationToken);
@@ -215,29 +229,9 @@ public sealed class OfflineTranscriptionBacklogProcessor
ISpeechRecognitionPipeline pipeline,
CancellationToken cancellationToken)
{
using var reader = new WaveFileReader(audioPath);
var bytesPerSecond = reader.WaveFormat.AverageBytesPerSecond;
var chunkSize = Math.Max(
reader.WaveFormat.BlockAlign,
bytesPerSecond * MaxChunkDurationMilliseconds / 1000);
chunkSize -= chunkSize % reader.WaveFormat.BlockAlign;
var buffer = new byte[chunkSize];
while (true)
await foreach (var chunk in PcmWavAudioChunkReader.ReadChunksAsync(audioPath, cancellationToken: cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
var read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken);
if (read == 0)
{
break;
}
await pipeline.WriteAsync(
new AudioChunk(
buffer[..read],
reader.WaveFormat.SampleRate,
reader.WaveFormat.Channels),
cancellationToken);
await pipeline.WriteAsync(chunk, cancellationToken);
}
}
}
@@ -0,0 +1,48 @@
using System.Runtime.CompilerServices;
using NAudio.Wave;
namespace MeetingAssistant.Recording;
public static class PcmWavAudioChunkReader
{
private const int MaxChunkDurationMilliseconds = 1000;
public static async IAsyncEnumerable<AudioChunk> ReadChunksAsync(
string path,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var reader = new WaveFileReader(path);
try
{
var format = reader.WaveFormat;
if (format.Encoding != WaveFormatEncoding.Pcm || format.BitsPerSample != 16)
{
throw new InvalidDataException(
$"Only 16-bit PCM WAV files are supported. Found {format.Encoding} with {format.BitsPerSample} bits per sample.");
}
var chunkSize = Math.Max(
format.BlockAlign,
format.AverageBytesPerSecond * MaxChunkDurationMilliseconds / 1000);
chunkSize -= chunkSize % format.BlockAlign;
var buffer = new byte[chunkSize];
int read;
while ((read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)) > 0)
{
yield return new AudioChunk(buffer[..read], format.SampleRate, format.Channels);
}
}
finally
{
try
{
reader.Dispose();
}
catch (NotSupportedException)
{
// NAudio can throw while disposing reader streams from async iterators after all chunks were read.
}
}
}
}
@@ -9,37 +9,31 @@ internal sealed class SpeakerAudioSampleCollector
private readonly object gate = new();
private readonly RollingAudioBuffer audioBuffer;
private readonly Dictionary<string, List<SpeakerAudioSample>> samplesBySpeaker = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, TimeSpan> lastAcceptedEndBySpeaker = new(StringComparer.OrdinalIgnoreCase);
private readonly int maxSamplesPerSpeaker;
private readonly TimeSpan minimumUninterruptedSpeechDuration;
private readonly TimeSpan maximumSegmentGap;
private readonly TimeSpan minimumSampleSpeechDuration;
private readonly TimeSpan maximumSampleDuration;
private readonly bool requireNonOverlappingSamples;
private readonly ILogger? logger;
private PendingSpeakerSpan? pendingSpan;
public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker)
: this(
bufferDuration,
maxSamplesPerSpeaker,
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(1),
logger: null)
{
}
private SpeakerSampleSpan? pendingSpan;
public SpeakerAudioSampleCollector(
TimeSpan bufferDuration,
int maxSamplesPerSpeaker,
TimeSpan minimumUninterruptedSpeechDuration,
TimeSpan maximumSegmentGap,
ILogger? logger = null)
TimeSpan minimumSampleSpeechDuration,
TimeSpan maximumSampleDuration,
ILogger? logger = null,
bool requireNonOverlappingSamples = false)
{
SpeakerSampleDurationConfiguration.ValidateOrThrow(
minimumSampleSpeechDuration,
maximumSampleDuration,
"Speaker sample collector configuration is invalid");
audioBuffer = new RollingAudioBuffer(bufferDuration);
this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker);
this.minimumUninterruptedSpeechDuration = minimumUninterruptedSpeechDuration > TimeSpan.Zero
? minimumUninterruptedSpeechDuration
: TimeSpan.Zero;
this.maximumSegmentGap = maximumSegmentGap >= TimeSpan.Zero
? maximumSegmentGap
: TimeSpan.Zero;
this.minimumSampleSpeechDuration = minimumSampleSpeechDuration;
this.maximumSampleDuration = maximumSampleDuration;
this.requireNonOverlappingSamples = requireNonOverlappingSamples;
this.logger = logger;
}
@@ -53,6 +47,7 @@ internal sealed class SpeakerAudioSampleCollector
lock (gate)
{
samplesBySpeaker.Clear();
lastAcceptedEndBySpeaker.Clear();
pendingSpan = null;
audioBuffer.Reset();
}
@@ -68,34 +63,54 @@ internal sealed class SpeakerAudioSampleCollector
return null;
}
TranscriptionSegment sampleSegment;
PendingSpanReset? reset;
SpeakerSampleSpan? sampleSpan;
SpeakerSampleSpan? previousSpan;
lock (gate)
{
(sampleSegment, reset) = ExtendPendingSpan(segment);
if (requireNonOverlappingSamples &&
lastAcceptedEndBySpeaker.TryGetValue(segment.Speaker, out var lastAcceptedEnd) &&
segment.Start < lastAcceptedEnd)
{
logger?.LogInformation(
"Discarding speaker identity sample for {Speaker} because it overlaps an accepted sample ending at {AcceptedSampleEnd}",
segment.Speaker,
lastAcceptedEnd);
return null;
}
(sampleSpan, previousSpan) = ExtendPendingSpan(segment);
}
if (reset is not null)
if (sampleSpan is null)
{
logger?.LogInformation(
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}, gap {Gap}, maximum gap {MaximumGap}",
reset.PreviousSpeaker,
segment.Speaker,
reset.PreviousEnd,
segment.Start,
reset.Gap,
maximumSegmentGap);
"Discarding speaker identity sample for {Speaker} because the segment duration is not positive",
segment.Speaker);
return null;
}
var score = Score(sampleSegment, minimumUninterruptedSpeechDuration);
var sampleSegment = sampleSpan.ToSegment();
if (previousSpan is not null)
{
logger?.LogInformation(
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}",
previousSpan.Speaker,
segment.Speaker,
previousSpan.End,
segment.Start);
}
var score = Score(sampleSegment, sampleSpan.SpeechDuration, minimumSampleSpeechDuration);
if (!score.Accepted)
{
logger?.LogInformation(
"Discarding speaker identity sample for {Speaker} because {Reason}: duration {Duration}, minimum duration {MinimumDuration}, word count {WordCount}",
"Discarding speaker identity sample for {Speaker} because {Reason}: speaker audio {SpeakerAudioDuration}, clip duration {ClipDuration}, minimum duration {MinimumDuration}, word count {WordCount}",
sampleSegment.Speaker,
score.Reason,
sampleSpan.SpeechDuration,
sampleSegment.End - sampleSegment.Start,
minimumUninterruptedSpeechDuration,
minimumSampleSpeechDuration,
score.WordCount);
return null;
}
@@ -115,6 +130,13 @@ internal sealed class SpeakerAudioSampleCollector
var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score.Value);
lock (gate)
{
if (requireNonOverlappingSamples &&
ReferenceEquals(pendingSpan, sampleSpan))
{
pendingSpan = null;
lastAcceptedEndBySpeaker[sampleSegment.Speaker] = sampleSegment.End;
}
if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples))
{
samples = [];
@@ -163,35 +185,26 @@ internal sealed class SpeakerAudioSampleCollector
!string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase);
}
private (TranscriptionSegment Segment, PendingSpanReset? Reset) ExtendPendingSpan(TranscriptionSegment segment)
private (SpeakerSampleSpan? Span, SpeakerSampleSpan? PreviousSpan) ExtendPendingSpan(TranscriptionSegment segment)
{
if (pendingSpan is null ||
!SpeakerSampleSpanSelector.CanExtend(pendingSpan.Speaker, pendingSpan.End, segment, maximumSegmentGap))
if (pendingSpan is not null && pendingSpan.TryExtend(segment, out var extended))
{
var reset = pendingSpan is null
? null
: new PendingSpanReset(
pendingSpan.Speaker,
pendingSpan.End,
segment.Start - pendingSpan.End);
pendingSpan = new PendingSpeakerSpan(
segment.Speaker,
segment.Start,
segment.End,
[segment.Text]);
return (pendingSpan.ToSegment(), reset);
pendingSpan = extended;
return (pendingSpan, null);
}
pendingSpan = pendingSpan.Extend(segment);
return (pendingSpan.ToSegment(), null);
var previousSpan = pendingSpan;
pendingSpan = SpeakerSampleSpan.Create(segment, maximumSampleDuration);
return (pendingSpan, previousSpan);
}
private static SampleScore Score(
TranscriptionSegment segment,
TimeSpan minimumUninterruptedSpeechDuration)
TimeSpan speechDuration,
TimeSpan minimumSampleSpeechDuration)
{
var durationSeconds = (segment.End - segment.Start).TotalSeconds;
if (durationSeconds < minimumUninterruptedSpeechDuration.TotalSeconds)
var durationSeconds = speechDuration.TotalSeconds;
if (durationSeconds < minimumSampleSpeechDuration.TotalSeconds)
{
return new SampleScore(false, 0, "speech duration is below the configured minimum", WordCount(segment.Text));
}
@@ -202,7 +215,7 @@ internal sealed class SpeakerAudioSampleCollector
return new SampleScore(false, 0, "word count is below the minimum useful sample length", words);
}
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumUninterruptedSpeechDuration.TotalSeconds), 2);
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumSampleSpeechDuration.TotalSeconds), 2);
var wordScore = Math.Min(words / 60.0, 1);
var sentenceBonus = segment.Text.TrimEnd().EndsWith('.') ||
segment.Text.TrimEnd().EndsWith('?') ||
@@ -221,30 +234,4 @@ internal sealed class SpeakerAudioSampleCollector
private sealed record SampleScore(bool Accepted, double Value, string? Reason, int WordCount);
private sealed record PendingSpanReset(string PreviousSpeaker, TimeSpan PreviousEnd, TimeSpan Gap);
private sealed record PendingSpeakerSpan(
string Speaker,
TimeSpan Start,
TimeSpan End,
IReadOnlyList<string> TextParts)
{
public PendingSpeakerSpan Extend(TranscriptionSegment segment)
{
return this with
{
End = segment.End > End ? segment.End : End,
TextParts = TextParts.Append(segment.Text).ToList()
};
}
public TranscriptionSegment ToSegment()
{
return new TranscriptionSegment(
Start,
End,
Speaker,
string.Join(' ', TextParts.Where(part => !string.IsNullOrWhiteSpace(part))));
}
}
}
@@ -49,6 +49,25 @@ public sealed class VaultTranscriptStore : ITranscriptStore
cancellationToken);
}
public Task AppendLinesAsync(
TranscriptSession session,
IReadOnlyList<string> lines,
CancellationToken cancellationToken)
{
if (lines.Count == 0)
{
return Task.CompletedTask;
}
return WithFileGateAsync(
session.TranscriptPath,
() => File.AppendAllTextAsync(
session.TranscriptPath,
string.Concat(lines.Select(line => line + Environment.NewLine)),
cancellationToken),
cancellationToken);
}
public async Task ReplaceLineAsync(
TranscriptSession session,
TranscriptLineReference lineReference,
@@ -11,7 +11,7 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
private const string NotificationGroup = "meeting-inactivity";
private readonly ConcurrentDictionary<string, PendingPrompt> pendingPrompts = new(StringComparer.OrdinalIgnoreCase);
private readonly ILogger<WindowsMeetingInactivityPromptService> logger;
private readonly object registrationGate = new();
private readonly object notificationGate = new();
private bool registered;
public WindowsMeetingInactivityPromptService(ILogger<WindowsMeetingInactivityPromptService> logger)
@@ -37,24 +37,27 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
try
{
EnsureRegistered();
var promptId = Guid.NewGuid().ToString("N");
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
logger.LogInformation(
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
promptId,
request.Threshold);
var notification = BuildNotification(promptId, request);
notification.Show(toast =>
lock (notificationGate)
{
toast.Group = NotificationGroup;
toast.Tag = promptId;
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
});
logger.LogInformation(
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
promptId,
request.Threshold);
EnsureRegistered();
var promptId = Guid.NewGuid().ToString("N");
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
logger.LogInformation(
"Registered native Windows inactivity notification {PromptId} response callback at threshold {Threshold}",
promptId,
request.Threshold);
var notification = BuildNotification(promptId, request);
notification.Show(toast =>
{
toast.Group = NotificationGroup;
toast.Tag = promptId;
toast.ExpirationTime = MeetingToastExpirationPolicy.StopReminderExpiration(DateTimeOffset.Now);
});
logger.LogInformation(
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
promptId,
request.Threshold);
}
}
catch (Exception exception)
{
@@ -76,6 +79,36 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
return Task.CompletedTask;
}
public Task DismissAllAsync(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
lock (notificationGate)
{
if (pendingPrompts.IsEmpty)
{
return Task.CompletedTask;
}
pendingPrompts.Clear();
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763))
{
return Task.CompletedTask;
}
try
{
ToastNotificationManagerCompat.History.RemoveGroup(NotificationGroup);
logger.LogInformation("Dismissed all native Windows inactivity notifications");
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to dismiss native Windows inactivity notifications");
}
return Task.CompletedTask;
}
}
public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
@@ -83,29 +116,28 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
public void Dispose()
{
if (!registered)
lock (notificationGate)
{
return;
}
if (!registered)
{
return;
}
try
{
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
try
{
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
registered = false;
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to detach native Windows toast notification activation handler");
}
}
}
private void EnsureRegistered()
{
if (registered)
{
return;
}
lock (registrationGate)
lock (notificationGate)
{
if (registered)
{
@@ -122,29 +154,25 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
string promptId,
MeetingInactivityPromptRequest request)
{
var yesButton = new ToastButton()
.SetContent("Yes")
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", "stop")
.SetBackgroundActivation();
var noButton = new ToastButton()
.SetContent("No")
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", "continue")
.SetBackgroundActivation();
return new ToastContentBuilder()
var builder = new ToastContentBuilder()
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.SetToastScenario(ToastScenario.Reminder)
.SetToastDuration(ToastDuration.Long)
.AddText("Stop meeting?")
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.")
.AddButton(yesButton)
.AddButton(noButton);
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.");
foreach (var action in MeetingInactivityPromptActions.All)
{
builder.AddButton(new ToastButton()
.SetContent(action.Content)
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", action.ResponseArgument)
.SetBackgroundActivation());
}
return builder;
}
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
@@ -177,10 +205,9 @@ public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPr
return;
}
var response = TryGetArgument(arguments, "response", out var responseValue) &&
string.Equals(responseValue, "stop", StringComparison.OrdinalIgnoreCase)
? MeetingInactivityPromptResponse.Stop
: MeetingInactivityPromptResponse.Continue;
var response = TryGetArgument(arguments, "response", out var responseValue)
? MeetingInactivityPromptActions.ParseResponse(responseValue)
: MeetingInactivityPromptResponse.Continue;
_ = Task.Run(async () =>
{
try
@@ -0,0 +1,96 @@
using NAudio.CoreAudioApi;
using NAudio.Wave;
namespace MeetingAssistant.Recording;
public sealed class WindowsMicrophoneDeviceProvider : IMicrophoneDeviceProvider, IMicrophoneCaptureSourceFactory
{
private readonly MicrophoneDeviceSelection selection;
private readonly ILogger<WindowsMicrophoneDeviceProvider> logger;
public WindowsMicrophoneDeviceProvider(
MicrophoneDeviceSelection selection,
ILogger<WindowsMicrophoneDeviceProvider> logger)
{
this.selection = selection;
this.logger = logger;
}
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
{
try
{
using var enumerator = new MMDeviceEnumerator();
return enumerator
.EnumerateAudioEndPoints(DataFlow.Capture, DeviceState.Active)
.Select(ToMicrophoneDevice)
.OrderBy(device => device.Name, StringComparer.CurrentCultureIgnoreCase)
.ToArray();
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not enumerate microphone capture endpoints");
return [];
}
}
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
{
var devices = GetAvailableMicrophones();
return new MicrophoneDeviceSnapshot(
devices,
selection.Resolve(options.Recording.MicrophoneDeviceId, GetDefaultMicrophone(), devices));
}
public IMeetingAudioSource CreateCapture(MeetingAssistantOptions options)
{
var current = GetMicrophoneSnapshot(options).Current;
IWaveIn capture;
if (current is null)
{
logger.LogInformation("Starting microphone capture from Windows default capture endpoint");
capture = new WasapiCapture();
}
else
{
logger.LogInformation(
"Starting microphone capture from {MicrophoneName} ({MicrophoneDeviceId})",
current.Name,
current.Id);
using var enumerator = new MMDeviceEnumerator();
capture = new WasapiCapture(enumerator.GetDevice(current.Id));
}
try
{
capture.WaveFormat = new WaveFormat(
options.Recording.SampleRate,
16,
options.Recording.Channels);
return new NaudioCaptureAudioSource(capture, "microphone", logger);
}
catch
{
capture.Dispose();
throw;
}
}
private static MicrophoneDevice? GetDefaultMicrophone()
{
try
{
using var enumerator = new MMDeviceEnumerator();
return ToMicrophoneDevice(enumerator.GetDefaultAudioEndpoint(DataFlow.Capture, Role.Console));
}
catch
{
return null;
}
}
private static MicrophoneDevice ToMicrophoneDevice(MMDevice device)
{
return new MicrophoneDevice(device.ID, device.FriendlyName);
}
}
@@ -0,0 +1,96 @@
#if WINDOWS
using System.Collections.Concurrent;
using CommunityToolkit.WinUI.Notifications;
namespace MeetingAssistant.Recording;
public sealed class WindowsMicrophoneSwitchPromptService : IMicrophoneSwitchPromptService, IDisposable
{
private const string Source = "meeting-microphone-switch";
private readonly ConcurrentDictionary<string, TaskCompletionSource<bool>> pending = new();
private readonly ILogger<WindowsMicrophoneSwitchPromptService> logger;
private readonly object gate = new();
private bool registered;
public WindowsMicrophoneSwitchPromptService(ILogger<WindowsMicrophoneSwitchPromptService> logger)
{
this.logger = logger;
}
public async Task<bool> ConfirmSwitchAsync(MicrophoneDevice microphone, CancellationToken cancellationToken)
{
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763)) return false;
cancellationToken.ThrowIfCancellationRequested();
var id = Guid.NewGuid().ToString("N");
var response = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
try
{
lock (gate)
{
if (!registered)
{
ToastNotificationManagerCompat.OnActivated += OnActivated;
registered = true;
}
pending[id] = response;
new ToastContentBuilder()
.AddArgument("source", Source)
.AddArgument("promptId", id)
.SetToastScenario(ToastScenario.Reminder)
.SetToastDuration(ToastDuration.Long)
.AddText("New microphone available")
.AddText($"Switch this meeting to {microphone.Name}?")
.AddButton(new ToastButton().SetContent("Yes")
.AddArgument("source", Source).AddArgument("promptId", id)
.AddArgument("response", "yes").SetBackgroundActivation())
.AddButton(new ToastButton().SetContent("No")
.AddArgument("source", Source).AddArgument("promptId", id)
.AddArgument("response", "no").SetBackgroundActivation())
.Show(toast =>
{
toast.Tag = id;
toast.Group = Source;
toast.ExpirationTime = DateTimeOffset.Now.AddMinutes(1);
});
}
logger.LogInformation("Offered newly available microphone {MicrophoneName}", microphone.Name);
return await response.Task.WaitAsync(cancellationToken);
}
finally
{
pending.TryRemove(id, out _);
try
{
ToastNotificationManagerCompat.History.Remove(id, Source);
}
catch (Exception exception)
{
logger.LogDebug(exception, "Could not remove microphone notification {PromptId}", id);
}
}
}
private void OnActivated(ToastNotificationActivatedEventArgsCompat args)
{
var arguments = NotificationActivationArguments.Parse(args.Argument);
if (arguments.GetValueOrDefault("source") != Source ||
!arguments.TryGetValue("promptId", out var id) ||
!pending.TryRemove(id, out var response)) return;
response.TrySetResult(arguments.GetValueOrDefault("response") == "yes");
}
public void Dispose()
{
lock (gate)
{
if (registered) ToastNotificationManagerCompat.OnActivated -= OnActivated;
registered = false;
foreach (var response in pending.Values) response.TrySetResult(false);
pending.Clear();
}
}
}
#endif
@@ -1,14 +1,13 @@
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.RegularExpressions;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Summary;
using Microsoft.Extensions.AI;
namespace MeetingAssistant.Screenshots;
public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
{
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
private readonly ILogger<LiteLlmScreenshotOcrClient> logger;
private readonly Func<HttpMessageHandler>? httpMessageHandlerFactory;
@@ -39,20 +38,30 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
: options.Agent.Model;
var key = ResolveApiKey(options);
var imageBytes = await File.ReadAllBytesAsync(screenshotPath, cancellationToken);
using var httpClient = CreateHttpClient();
httpClient.BaseAddress = NormalizeEndpoint(new Uri(endpoint));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", key);
var payload = CreatePayload(model, CreatePrompt(prompt, imageBytes), imageBytes);
using var content = new StringContent(payload.ToJsonString(JsonOptions), Encoding.UTF8, "application/json");
using var response = await httpClient.PostAsync("responses", content, cancellationToken);
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken);
if (!response.IsSuccessStatusCode)
{
throw new InvalidOperationException(
$"Screenshot OCR request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}");
}
var text = ParseOutputText(responseJson);
var httpClient = CreateHttpClient();
httpClient.BaseAddress = LiteLlmResponsesChatClient.NormalizeEndpoint(new Uri(endpoint));
using var chatClient = new LiteLlmResponsesChatClient(
httpClient,
key,
model,
enableThinking: false,
reasoningEffort: "none",
reconnectionAttempts: options.Agent.ReconnectionAttempts,
reconnectionDelay: options.Agent.ReconnectionDelay,
logger: logger,
firstRequestIsUser: false,
useStreaming: options.Agent.UseStreaming);
var response = await chatClient.GetResponseAsync(
[
new ChatMessage(
ChatRole.User,
[
new TextContent(CreatePrompt(prompt, imageBytes)),
new DataContent(imageBytes, "image/png")
])
],
cancellationToken: cancellationToken);
var text = response.Text ?? string.Empty;
logger.LogInformation("Screenshot OCR completed for {ScreenshotPath}", screenshotPath);
return ParseOcrResult(text);
}
@@ -64,35 +73,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
: new HttpClient(httpMessageHandlerFactory());
}
private static JsonObject CreatePayload(string model, string prompt, byte[] imageBytes)
{
return new JsonObject
{
["model"] = model,
["store"] = false,
["input"] = new JsonArray
{
new JsonObject
{
["role"] = "user",
["content"] = new JsonArray
{
new JsonObject
{
["type"] = "input_text",
["text"] = prompt
},
new JsonObject
{
["type"] = "input_image",
["image_url"] = "data:image/png;base64," + Convert.ToBase64String(imageBytes)
}
}
}
}
};
}
private static string CreatePrompt(string prompt, byte[] imageBytes)
{
return TryReadPngDimensions(imageBytes, out var width, out var height)
@@ -104,41 +84,69 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
private static ScreenshotOcrResult ParseOcrResult(string text)
{
ScreenshotCropCoordinates? crop = null;
IReadOnlyList<string> attendees = [];
var cleaned = JsonCodeBlockRegex().Replace(text, match =>
{
if (TryParseCrop(match.Groups["json"].Value, out var parsedCrop))
if (TryParseMetadata(match.Groups["json"].Value, out var parsedCrop, out var parsedAttendees))
{
crop = parsedCrop;
attendees = parsedAttendees;
return "";
}
return match.Value;
}).Trim();
return new ScreenshotOcrResult(cleaned, crop);
return new ScreenshotOcrResult(cleaned, crop, attendees);
}
private static bool TryParseCrop(string json, out ScreenshotCropCoordinates? crop)
private static bool TryParseMetadata(
string json,
out ScreenshotCropCoordinates? crop,
out IReadOnlyList<string> attendees)
{
crop = null;
attendees = [];
try
{
using var document = JsonDocument.Parse(json);
if (!document.RootElement.TryGetProperty("crop", out var cropElement) ||
cropElement.ValueKind == JsonValueKind.Null)
{
return true;
}
if (cropElement.ValueKind != JsonValueKind.Object ||
!TryGetInt(cropElement, "x", out var x) ||
!TryGetInt(cropElement, "y", out var y) ||
!TryGetInt(cropElement, "width", out var width) ||
!TryGetInt(cropElement, "height", out var height))
if (document.RootElement.ValueKind != JsonValueKind.Object)
{
return false;
}
crop = new ScreenshotCropCoordinates(x, y, width, height);
var hasMetadata = false;
if (document.RootElement.TryGetProperty("attendees", out var attendeesElement))
{
hasMetadata = true;
if (attendeesElement.ValueKind == JsonValueKind.Array)
{
attendees = MeetingAttendeeNames.NormalizeDistinct(attendeesElement
.EnumerateArray()
.Where(element => element.ValueKind == JsonValueKind.String)
.Select(element => element.GetString() ?? ""));
}
}
if (!document.RootElement.TryGetProperty("crop", out var cropElement))
{
return hasMetadata;
}
hasMetadata = true;
if (cropElement.ValueKind == JsonValueKind.Null)
{
return true;
}
if (cropElement.ValueKind == JsonValueKind.Object &&
TryGetInt(cropElement, "x", out var x) &&
TryGetInt(cropElement, "y", out var y) &&
TryGetInt(cropElement, "width", out var width) &&
TryGetInt(cropElement, "height", out var height))
{
crop = new ScreenshotCropCoordinates(x, y, width, height);
}
return true;
}
catch (JsonException)
@@ -181,46 +189,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
bytes[offset + 3];
}
private static string ParseOutputText(string responseJson)
{
using var document = JsonDocument.Parse(responseJson);
var root = document.RootElement;
var parts = new List<string>();
if (root.TryGetProperty("output_text", out var outputText) &&
outputText.ValueKind == JsonValueKind.String &&
!string.IsNullOrWhiteSpace(outputText.GetString()))
{
parts.Add(outputText.GetString()!);
}
if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array)
{
foreach (var item in output.EnumerateArray())
{
if (!item.TryGetProperty("content", out var content) || content.ValueKind != JsonValueKind.Array)
{
continue;
}
foreach (var block in content.EnumerateArray())
{
if (block.TryGetProperty("type", out var type) &&
type.GetString() == "output_text" &&
block.TryGetProperty("text", out var text) &&
text.ValueKind == JsonValueKind.String &&
!string.IsNullOrWhiteSpace(text.GetString()))
{
parts.Add(text.GetString()!);
}
}
}
}
return parts.Count == 0
? ""
: string.Join(Environment.NewLine + Environment.NewLine, parts);
}
private static string ResolveApiKey(MeetingAssistantOptions options)
{
if (!string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Key))
@@ -249,17 +217,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
$"No screenshot OCR API key configured. Set MeetingAssistant:Screenshots:Ocr:Key or environment variable '{options.Screenshots.Ocr.KeyEnv}'.");
}
private static Uri NormalizeEndpoint(Uri endpoint)
{
var value = endpoint.ToString().TrimEnd('/');
if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase))
{
value += "/v1";
}
return new Uri(value + "/");
}
[GeneratedRegex("```json\\s*(?<json>.*?)\\s*```", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
private static partial Regex JsonCodeBlockRegex();
}
@@ -2,7 +2,10 @@ using System.Collections.Concurrent;
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Speakers;
using MeetingAssistant.Workflow;
namespace MeetingAssistant.Screenshots;
@@ -22,7 +25,24 @@ public interface IScreenshotOcrClient
CancellationToken cancellationToken);
}
public sealed record ScreenshotOcrResult(string Text, ScreenshotCropCoordinates? Crop);
public sealed record ScreenshotOcrResult
{
public ScreenshotOcrResult(
string text,
ScreenshotCropCoordinates? crop,
IReadOnlyList<string>? attendees = null)
{
Text = text;
Crop = crop;
Attendees = attendees ?? [];
}
public string Text { get; init; }
public ScreenshotCropCoordinates? Crop { get; init; }
public IReadOnlyList<string> Attendees { get; init; }
}
public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height);
@@ -46,11 +66,36 @@ public interface IMeetingScreenshotService
CancellationToken cancellationToken);
}
public interface IMeetingNoteImageOcrService
{
Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken);
}
public interface IScreenshotOcrRetryRunner
{
Task<MeetingScreenshotOcrRetryTriggerResult?> TriggerOcrRetryAsync(
MeetingSessionArtifacts artifacts,
string screenshotPath,
string screenshotId,
MeetingAssistantOptions options,
CancellationToken cancellationToken);
}
public sealed record MeetingScreenshotCaptureResult(
string ScreenshotPath,
TimeSpan MeetingTimestamp,
bool OcrStarted);
public sealed record MeetingScreenshotOcrRetryTriggerResult(
string AssistantContextPath,
string ScreenshotPath,
string ScreenshotId);
public sealed record MeetingNoteImageOcrQueueResult(int QueuedCount);
public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
{
public static NoopMeetingScreenshotService Instance { get; } = new();
@@ -80,12 +125,32 @@ public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
{
return Task.CompletedTask;
}
}
public sealed class MeetingScreenshotService : IMeetingScreenshotService
public sealed class NoopMeetingNoteImageOcrService : IMeetingNoteImageOcrService
{
public static NoopMeetingNoteImageOcrService Instance { get; } = new();
public Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return Task.FromResult(new MeetingNoteImageOcrQueueResult(0));
}
}
public sealed partial class MeetingScreenshotService :
IMeetingScreenshotService,
IScreenshotOcrRetryRunner,
IMeetingNoteImageOcrService
{
private readonly IActiveWindowScreenshotCapture screenshotCapture;
private readonly IMeetingArtifactStore artifactStore;
private readonly IMeetingNoteStore meetingNoteStore;
private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer;
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
private readonly IScreenshotOcrClient ocrClient;
private readonly ILogger<MeetingScreenshotService> logger;
private readonly ConcurrentDictionary<string, List<PendingOcrTask>> pendingOcrByContext = new(StringComparer.OrdinalIgnoreCase);
@@ -94,11 +159,17 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
public MeetingScreenshotService(
IActiveWindowScreenshotCapture screenshotCapture,
IMeetingArtifactStore artifactStore,
IMeetingNoteStore meetingNoteStore,
ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer,
IScreenshotOcrClient ocrClient,
ILogger<MeetingScreenshotService> logger)
ILogger<MeetingScreenshotService> logger,
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
{
this.screenshotCapture = screenshotCapture;
this.artifactStore = artifactStore;
this.meetingNoteStore = meetingNoteStore;
this.attendeeCanonicalizer = attendeeCanonicalizer;
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
this.ocrClient = ocrClient;
this.logger = logger;
}
@@ -135,7 +206,13 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
TrackOcr(
artifacts,
ocrCancellation,
RunOcrAsync(artifacts, screenshotPath, screenshotId, options, ocrCancellation.Token));
RunOcrAsync(
artifacts,
screenshotPath,
screenshotId,
options,
ScreenshotOcrPolicy.CapturedScreenshot,
ocrCancellation.Token));
}
logger.LogInformation(
@@ -212,6 +289,100 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
await WaitForPendingOcrAsync(artifacts, timeout, cancellationToken);
}
public async Task<MeetingScreenshotOcrRetryTriggerResult?> TriggerOcrRetryAsync(
MeetingSessionArtifacts artifacts,
string screenshotPath,
string screenshotId,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(screenshotId) ||
string.IsNullOrWhiteSpace(screenshotPath) ||
!File.Exists(screenshotPath) ||
!File.Exists(artifacts.AssistantContextPath))
{
return null;
}
var replaced = await ReplaceOcrPlaceholderAsync(
artifacts.AssistantContextPath,
screenshotId,
"_OCR retry pending..._",
cancellationToken,
preserveMarkers: true,
appendWhenMissing: false);
if (!replaced)
{
return null;
}
var ocrCancellation = new CancellationTokenSource();
TrackOcr(
artifacts,
ocrCancellation,
RunOcrAsync(
artifacts,
screenshotPath,
screenshotId,
options,
ScreenshotOcrPolicy.CapturedScreenshot,
ocrCancellation.Token));
return new MeetingScreenshotOcrRetryTriggerResult(
artifacts.AssistantContextPath,
screenshotPath,
screenshotId);
}
public async Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
if (!options.Screenshots.Ocr.Enabled || !File.Exists(artifacts.MeetingNotePath))
{
return new MeetingNoteImageOcrQueueResult(0);
}
var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken);
var embeds = FindMeetingNoteImageEmbeds(
meetingNote.UserNotes,
artifacts.MeetingNotePath,
options);
var queuedEmbeds = new List<(MeetingNoteImageEmbed Embed, string OcrId)>();
foreach (var embed in embeds)
{
cancellationToken.ThrowIfCancellationRequested();
var ocrId = Guid.NewGuid().ToString("N");
await artifactStore.AppendAssistantContextAsync(
artifacts,
CreateMeetingNoteImageMarkdown(
ocrId,
embed.OriginalEmbed,
embed.ImagePath,
artifacts.AssistantContextPath),
cancellationToken);
queuedEmbeds.Add((embed, ocrId));
}
foreach (var (embed, ocrId) in queuedEmbeds)
{
cancellationToken.ThrowIfCancellationRequested();
var ocrCancellation = new CancellationTokenSource();
TrackOcr(
artifacts,
ocrCancellation,
RunOcrAsync(
artifacts,
embed.ImagePath,
ocrId,
options,
ScreenshotOcrPolicy.MeetingNoteImage,
ocrCancellation.Token));
}
return new MeetingNoteImageOcrQueueResult(queuedEmbeds.Count);
}
private async Task<string> SaveScreenshotAsync(
MeetingSessionArtifacts artifacts,
byte[] pngBytes,
@@ -234,6 +405,7 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
string screenshotPath,
string screenshotId,
MeetingAssistantOptions options,
ScreenshotOcrPolicy policy,
CancellationToken cancellationToken)
{
var timeout = options.Screenshots.Ocr.GetEffectiveTimeout();
@@ -248,17 +420,27 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
: options.Screenshots.Ocr.Prompt,
options,
timeoutSource.Token);
var cropMarkdown = await TryCreateCropMarkdownAsync(
artifacts.AssistantContextPath,
screenshotPath,
result.Crop,
timeoutSource.Token);
var cropMarkdown = policy.AllowCrop
? await TryCreateCropMarkdownAsync(
artifacts.AssistantContextPath,
screenshotPath,
result.Crop,
timeoutSource.Token)
: "";
await ReplaceOcrPlaceholderAsync(
artifacts.AssistantContextPath,
screenshotId,
cropMarkdown +
"### OCR" + Environment.NewLine + Environment.NewLine + result.Text.Trim(),
CancellationToken.None);
if (policy.AddAttendees)
{
await AddOcrAttendeesAsync(
artifacts,
result.Attendees,
options,
CancellationToken.None);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
@@ -269,8 +451,15 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
await ReplaceOcrPlaceholderAsync(
artifacts.AssistantContextPath,
screenshotId,
$"_OCR timed out after {timeout:g}._",
CancellationToken.None);
CreateOcrFailureMarkdown(
artifacts,
screenshotPath,
screenshotId,
options,
$"_OCR timed out after {timeout:g}._",
policy.IncludeRetryLink),
CancellationToken.None,
preserveMarkers: true);
}
catch (Exception exception)
{
@@ -278,16 +467,93 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
await ReplaceOcrPlaceholderAsync(
artifacts.AssistantContextPath,
screenshotId,
$"_OCR failed: {exception.Message}_",
CancellationToken.None);
CreateOcrFailureMarkdown(
artifacts,
screenshotPath,
screenshotId,
options,
$"_OCR failed: {exception.Message}_",
policy.IncludeRetryLink),
CancellationToken.None,
preserveMarkers: true);
}
}
private async Task ReplaceOcrPlaceholderAsync(
private async Task AddOcrAttendeesAsync(
MeetingSessionArtifacts artifacts,
IReadOnlyList<string> attendees,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var additions = MeetingAttendeeNames.NormalizeDistinct(attendees);
if (additions.Count == 0)
{
return;
}
try
{
var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken);
var transformedAdditions = await TransformAttendeesAsync(
artifacts,
additions,
options,
cancellationToken);
var canonicalized = await attendeeCanonicalizer.CanonicalizeAsync(
meetingNote.Frontmatter.Attendees.Concat(transformedAdditions).ToList(),
cancellationToken);
if (meetingNote.Frontmatter.Attendees.SequenceEqual(canonicalized, StringComparer.Ordinal))
{
return;
}
meetingNote.Frontmatter.Attendees = canonicalized.ToList();
await meetingNoteStore.SaveAsync(meetingNote, cancellationToken);
logger.LogInformation(
"Added {AttendeeCount} screenshot OCR attendee candidate(s) to meeting note {MeetingNotePath}",
additions.Count,
artifacts.MeetingNotePath);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogWarning(
exception,
"Could not add screenshot OCR attendees to meeting note {MeetingNotePath}",
artifacts.MeetingNotePath);
}
}
private async Task<IReadOnlyList<string>> TransformAttendeesAsync(
MeetingSessionArtifacts artifacts,
IReadOnlyList<string> attendees,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var transformed = new List<string>();
foreach (var attendee in attendees)
{
var value = await meetingWorkflowEngine.TransformAttendeeAsync(
MeetingWorkflowEvent.AttendeeAdded(artifacts, attendee),
options,
cancellationToken);
var normalized = MeetingAttendeeNames.NormalizeDisplayName(value);
if (!string.IsNullOrWhiteSpace(normalized) &&
!transformed.Contains(normalized, StringComparer.OrdinalIgnoreCase))
{
transformed.Add(normalized);
}
}
return transformed;
}
private async Task<bool> ReplaceOcrPlaceholderAsync(
string assistantContextPath,
string screenshotId,
string replacement,
CancellationToken cancellationToken)
CancellationToken cancellationToken,
bool preserveMarkers = false,
bool appendWhenMissing = true)
{
var fileLock = contextFileLocks.GetOrAdd(
NormalizeContextKey(assistantContextPath),
@@ -297,28 +563,41 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
{
if (!File.Exists(assistantContextPath))
{
return;
return false;
}
var content = await File.ReadAllTextAsync(assistantContextPath, cancellationToken);
var startMarker = $"<!-- screenshot-ocr:{screenshotId} -->";
var endMarker = $"<!-- /screenshot-ocr:{screenshotId} -->";
var startMarker = CreateOcrStartMarker(screenshotId);
var endMarker = CreateOcrEndMarker(screenshotId);
var startIndex = content.IndexOf(startMarker, StringComparison.Ordinal);
var endIndex = content.IndexOf(endMarker, StringComparison.Ordinal);
if (startIndex < 0 || endIndex < startIndex)
{
await File.AppendAllTextAsync(
assistantContextPath,
Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
cancellationToken);
return;
if (appendWhenMissing)
{
await File.AppendAllTextAsync(
assistantContextPath,
Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
cancellationToken);
}
return false;
}
endIndex += endMarker.Length;
var updated = content[..startIndex] +
replacement.TrimEnd() +
content[endIndex..];
var updated = preserveMarkers
? content[..startIndex] +
startMarker +
Environment.NewLine +
replacement.TrimEnd() +
Environment.NewLine +
endMarker +
content[endIndex..]
: content[..startIndex] +
replacement.TrimEnd() +
content[endIndex..];
await File.WriteAllTextAsync(assistantContextPath, updated, cancellationToken);
return true;
}
finally
{
@@ -326,6 +605,29 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
}
}
private static string CreateOcrFailureMarkdown(
MeetingSessionArtifacts artifacts,
string screenshotPath,
string screenshotId,
MeetingAssistantOptions options,
string status,
bool includeRetryLink = true)
{
var markdown = status.TrimEnd();
if (includeRetryLink)
{
markdown += Environment.NewLine +
Environment.NewLine +
MeetingNoteActionLinks.CreateScreenshotOcrRetryLink(
options.Api.PublicBaseUrl,
artifacts,
screenshotPath,
screenshotId);
}
return markdown;
}
private async Task<string> TryCreateCropMarkdownAsync(
string assistantContextPath,
string screenshotPath,
@@ -435,13 +737,189 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
return content +
Environment.NewLine +
Environment.NewLine +
$"<!-- screenshot-ocr:{screenshotId} -->" +
Environment.NewLine +
"_OCR pending..._" +
Environment.NewLine +
$"<!-- /screenshot-ocr:{screenshotId} -->";
CreateOcrBlock(screenshotId, "_OCR pending..._");
}
private static string CreateMeetingNoteImageMarkdown(
string ocrId,
string originalEmbed,
string imagePath,
string assistantContextPath)
{
var relativePath = ToMarkdownPath(Path.GetRelativePath(
Path.GetDirectoryName(assistantContextPath)!,
imagePath));
return "## Meeting Note Image" +
Environment.NewLine +
"Image from meeting note." +
Environment.NewLine +
$"Original embed: `{originalEmbed.Replace("`", "\\`", StringComparison.Ordinal)}`" +
Environment.NewLine +
$"![Meeting note image]({relativePath})" +
Environment.NewLine +
Environment.NewLine +
CreateOcrBlock(ocrId, "_OCR pending..._");
}
private static string CreateOcrBlock(string screenshotId, string content)
{
return CreateOcrStartMarker(screenshotId) +
Environment.NewLine +
content.TrimEnd() +
Environment.NewLine +
CreateOcrEndMarker(screenshotId);
}
private static string CreateOcrStartMarker(string screenshotId)
{
return $"<!-- screenshot-ocr:{screenshotId} -->";
}
private static string CreateOcrEndMarker(string screenshotId)
{
return $"<!-- /screenshot-ocr:{screenshotId} -->";
}
private static IReadOnlyList<MeetingNoteImageEmbed> FindMeetingNoteImageEmbeds(
string userNotes,
string meetingNotePath,
MeetingAssistantOptions options)
{
if (string.IsNullOrWhiteSpace(userNotes))
{
return [];
}
var result = new List<MeetingNoteImageEmbed>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (Match match in ObsidianImageEmbedRegex().Matches(userNotes))
{
AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options);
}
foreach (Match match in MarkdownImageEmbedRegex().Matches(userNotes))
{
AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options);
}
return result;
}
private static void AddResolvedEmbed(
List<MeetingNoteImageEmbed> embeds,
HashSet<string> seen,
string originalEmbed,
string target,
string meetingNotePath,
MeetingAssistantOptions options)
{
var imagePath = ResolveMeetingNoteImagePath(target, meetingNotePath, options);
if (imagePath is null || !seen.Add(imagePath))
{
return;
}
embeds.Add(new MeetingNoteImageEmbed(originalEmbed, imagePath));
}
private static string? ResolveMeetingNoteImagePath(
string target,
string meetingNotePath,
MeetingAssistantOptions options)
{
var cleaned = CleanEmbedTarget(target);
if (string.IsNullOrWhiteSpace(cleaned) ||
cleaned.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
cleaned.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ||
cleaned.StartsWith("data:", StringComparison.OrdinalIgnoreCase) ||
!IsSupportedImagePath(cleaned))
{
return null;
}
var candidates = new List<string>();
if (Path.IsPathRooted(cleaned))
{
candidates.Add(VaultPath.Resolve(cleaned));
}
else
{
candidates.Add(Path.GetFullPath(Path.Combine(Path.GetDirectoryName(meetingNotePath)!, cleaned)));
candidates.Add(VaultPath.Resolve(options.Vault, cleaned));
}
foreach (var candidate in candidates)
{
if (File.Exists(candidate))
{
return candidate;
}
}
if (Path.GetFileName(cleaned).Equals(cleaned, StringComparison.Ordinal))
{
try
{
var vaultRoot = VaultPath.Resolve(options.Vault.BaseFolder);
return Directory.EnumerateFiles(vaultRoot, cleaned, SearchOption.AllDirectories)
.FirstOrDefault(IsSupportedImagePath);
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or DirectoryNotFoundException)
{
return null;
}
}
return null;
}
private static string CleanEmbedTarget(string target)
{
var cleaned = target.Trim();
if (cleaned.StartsWith("<", StringComparison.Ordinal) &&
cleaned.EndsWith(">", StringComparison.Ordinal) &&
cleaned.Length > 1)
{
cleaned = cleaned[1..^1].Trim();
}
var pipeIndex = cleaned.IndexOf('|', StringComparison.Ordinal);
if (pipeIndex >= 0)
{
cleaned = cleaned[..pipeIndex].Trim();
}
var hashIndex = cleaned.IndexOf('#', StringComparison.Ordinal);
if (hashIndex >= 0)
{
cleaned = cleaned[..hashIndex].Trim();
}
try
{
return Uri.UnescapeDataString(cleaned.Replace('/', Path.DirectorySeparatorChar));
}
catch (UriFormatException)
{
return cleaned.Replace('/', Path.DirectorySeparatorChar);
}
}
private static bool IsSupportedImagePath(string path)
{
return Path.GetExtension(path).ToLowerInvariant() switch
{
".png" or ".jpg" or ".jpeg" or ".gif" or ".webp" or ".bmp" or ".tif" or ".tiff" => true,
_ => false
};
}
[GeneratedRegex(@"!\[\[(?<target>[^\]\r\n]+)\]\]", RegexOptions.CultureInvariant)]
private static partial Regex ObsidianImageEmbedRegex();
[GeneratedRegex(@"!\[[^\]\r\n]*\]\((?<target>[^)\r\n]+)\)", RegexOptions.CultureInvariant)]
private static partial Regex MarkdownImageEmbedRegex();
private static string ResolveAttachmentsFolder(string assistantContextPath, string configuredFolder)
{
var expanded = Environment.ExpandEnvironmentVariables(
@@ -479,4 +957,19 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
}
private sealed record PendingOcrTask(Task Task, CancellationTokenSource Cancellation);
private sealed record ScreenshotOcrPolicy(bool AllowCrop, bool AddAttendees, bool IncludeRetryLink)
{
public static ScreenshotOcrPolicy CapturedScreenshot { get; } = new(
AllowCrop: true,
AddAttendees: true,
IncludeRetryLink: true);
public static ScreenshotOcrPolicy MeetingNoteImage { get; } = new(
AllowCrop: false,
AddAttendees: false,
IncludeRetryLink: false);
}
private sealed record MeetingNoteImageEmbed(string OriginalEmbed, string ImagePath);
}
@@ -190,7 +190,7 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
return [];
}
return await finalizer.FinalizeAsync(
return await finalizer.FinalizeEnabledAsync(
wavPath,
[new TranscriptionSegment(TimeSpan.Zero, duration, "Unknown", "speaker identity validation sample")],
options.Diarization,
@@ -0,0 +1,631 @@
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Transcription;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public sealed class ResemblyzerSpeakerIdentificationService : ISpeakerIdentificationService
{
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
private readonly ISpeakerSnippetExtractor snippetExtractor;
private readonly IResemblyzerVoiceEncoder encoder;
private readonly ResemblyzerVoiceClusterMatcher clusterMatcher;
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
private readonly SpeakerIdentificationOptions options;
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
private readonly ILogger<ResemblyzerSpeakerIdentificationService> logger;
public ResemblyzerSpeakerIdentificationService(
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
ISpeakerSnippetExtractor snippetExtractor,
IResemblyzerVoiceEncoder encoder,
ResemblyzerVoiceClusterMatcher clusterMatcher,
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
IOptions<MeetingAssistantOptions> options,
ILogger<ResemblyzerSpeakerIdentificationService> logger)
{
this.dbContextFactory = dbContextFactory;
this.snippetExtractor = snippetExtractor;
this.encoder = encoder;
this.clusterMatcher = clusterMatcher;
this.outlierPruner = outlierPruner;
this.options = options.Value.SpeakerIdentification;
resemblyzerOptions = this.options.Resemblyzer;
this.logger = logger;
}
public Task<SpeakerIdentificationResult> IdentifyKnownSpeakersAsync(
SpeakerIdentificationRequest request,
CancellationToken cancellationToken)
{
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: false, cancellationToken);
}
public Task<SpeakerIdentificationResult> IdentifyFinishedSpeakersAsync(
SpeakerIdentificationRequest request,
CancellationToken cancellationToken)
{
return ProcessTranscriptAsync(request, final: false, allowAudioFallback: true, cancellationToken);
}
public Task<SpeakerIdentificationResult> ProcessFinishedTranscriptAsync(
SpeakerIdentificationRequest request,
CancellationToken cancellationToken)
{
return ProcessTranscriptAsync(request, final: true, allowAudioFallback: true, cancellationToken);
}
public async Task ApplySpeakerOverrideAsync(
SpeakerIdentificationRequest request,
string sourceSpeaker,
string targetSpeaker,
CancellationToken cancellationToken)
{
if (!options.Enabled ||
string.IsNullOrWhiteSpace(sourceSpeaker) ||
string.IsNullOrWhiteSpace(targetSpeaker) ||
string.Equals(sourceSpeaker, targetSpeaker, StringComparison.OrdinalIgnoreCase))
{
return;
}
var sourceLabel = sourceSpeaker.Trim();
var targetName = targetSpeaker.Trim();
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
var target = identities
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
.OrderBy(identity => string.Equals(identity.CanonicalName, targetName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
.ThenBy(identity => identity.Id)
.FirstOrDefault();
var reference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
var sourceCandidate = identities
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(targetName))
.OrderBy(identity => identity.Id)
.FirstOrDefault();
var vectors = await ResolveAvailableVectorsAsync(request, sourceLabel, cancellationToken);
if (target is null && sourceCandidate is null && vectors.Count == 0)
{
logger.LogWarning(
"Skipping Resemblyzer speaker override from {SourceSpeaker} to {TargetSpeaker} because no source evidence was available",
sourceLabel,
targetName);
return;
}
var now = DateTimeOffset.UtcNow;
if (target is null)
{
target = sourceCandidate ?? new SpeakerIdentity
{
CreatedAt = now
};
if (target.Id == 0)
{
context.SpeakerIdentities.Add(target);
}
}
else if (sourceCandidate is not null && sourceCandidate.Id != target.Id)
{
SpeakerIdentityMerger.MergeIntoAndPrune(
target,
sourceCandidate,
options.MaxSnippetsPerSpeaker,
resemblyzerOptions.MaxVectorsPerIdentity,
outlierPruner);
context.SpeakerIdentities.Remove(sourceCandidate);
}
target.CanonicalName = targetName;
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
SpeakerIdentityReferences.AddIfMissing(target, reference, now);
outlierPruner.AddAndPrune(
target,
vectors,
now);
target.UpdatedAt = now;
await context.SaveChangesAsync(cancellationToken);
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
target.References,
sourceLabel,
targetName,
cancellationToken);
}
public async Task DeleteSpeakerIdentityAsync(
string identity,
CancellationToken cancellationToken)
{
if (!options.Enabled || string.IsNullOrWhiteSpace(identity))
{
return;
}
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
var target = identities.FirstOrDefault(candidate =>
SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(identity.Trim()));
if (target is null)
{
return;
}
context.SpeakerIdentities.Remove(target);
await context.SaveChangesAsync(cancellationToken);
}
private async Task<SpeakerIdentificationResult> ProcessTranscriptAsync(
SpeakerIdentificationRequest request,
bool final,
bool allowAudioFallback,
CancellationToken cancellationToken)
{
if (!options.Enabled || request.Segments.Count == 0)
{
return EmptyResult(request.Segments);
}
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
var knownMappings = request.KnownSpeakerMappings ??
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var knownLabels = knownMappings.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase);
if (final && knownMappings.Count > 0)
{
await PersistMappedSpeakerEvidenceAsync(
context,
request,
knownMappings,
cancellationToken);
}
var identifiedNames = knownMappings.Values
.Where(name => !string.IsNullOrWhiteSpace(name))
.Select(name => name.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var segmentSpeaker in request.Segments
.Select(segment => segment.Speaker)
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
{
identifiedNames.Add(segmentSpeaker.Trim());
}
var mappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
var pendingAudits = new List<PendingIdentificationAudit>();
var matchedAcceptedNames = identifiedNames.ToHashSet(StringComparer.OrdinalIgnoreCase);
var unmatchedSpeakers = new List<(string Speaker, IReadOnlyList<float[]> Vectors)>();
foreach (var speaker in request.Segments
.Select(segment => segment.Speaker)
.Where(speaker => !string.IsNullOrWhiteSpace(speaker))
.Distinct(StringComparer.OrdinalIgnoreCase))
{
if (knownLabels.Contains(speaker) || identifiedNames.Contains(speaker) || !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker))
{
continue;
}
var vectors = await ResolveAutomaticVectorsAsync(
request,
speaker,
allowAudioFallback,
cancellationToken);
if (vectors.Count < resemblyzerOptions.RequiredVectorsPerSpeaker)
{
logger.LogInformation(
"Resemblyzer matching waits for more vectors for {Speaker}: {VectorCount}/{RequiredVectorCount}",
speaker,
vectors.Count,
resemblyzerOptions.RequiredVectorsPerSpeaker);
continue;
}
var (identity, decision) = await FindMatchAsync(
context,
attendees,
identifiedNames,
vectors,
cancellationToken);
if (identity is null)
{
if (final && decision.Cohesion >= resemblyzerOptions.MinimumClusterCohesion)
{
unmatchedSpeakers.Add((speaker, vectors));
}
continue;
}
var now = DateTimeOffset.UtcNow;
var previousCanonicalName = identity.CanonicalName;
var previousReferenceCount = identity.References.Count;
outlierPruner.AddAndPrune(
identity,
vectors,
now);
SpeakerIdentityReferences.AddIfMissing(
identity,
CreateReference(request.MeetingNote, now),
now);
if (identity.References.Count != previousReferenceCount)
{
identity.UpdatedAt = now;
}
if (final)
{
UpdateMatchedIdentity(identity, attendees);
if (string.IsNullOrWhiteSpace(previousCanonicalName) &&
!string.IsNullOrWhiteSpace(identity.CanonicalName))
{
pendingAudits.Add(new PendingIdentificationAudit(
identity,
speaker,
identity.CanonicalName));
}
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
{
matchedAcceptedNames.Add(acceptedName);
}
}
var displayName = identity.GetDisplayName();
if (!string.IsNullOrWhiteSpace(displayName))
{
mappings[speaker] = displayName;
identifiedNames.Add(displayName);
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
{
identifiedNames.Add(acceptedName);
}
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
displayName,
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
}
}
if (final)
{
LearnUnmatchedSpeakers(
context,
request.MeetingNote,
attendees,
matchedAcceptedNames,
unmatchedSpeakers,
pendingAudits);
}
await context.SaveChangesAsync(cancellationToken);
await AppendAuditsAsync(pendingAudits, cancellationToken);
var relabeled = request.Segments
.Select(segment => mappings.TryGetValue(segment.Speaker, out var name)
? segment with { Speaker = name }
: segment)
.ToList();
return new SpeakerIdentificationResult(relabeled, mappings, attendeeMatches);
}
private async Task<(SpeakerIdentity? Identity, ResemblyzerVoiceClusterMatchResult Decision)> FindMatchAsync(
SpeakerIdentityDbContext context,
IReadOnlyList<string> attendees,
IReadOnlySet<string> identifiedNames,
IReadOnlyList<float[]> queryVectors,
CancellationToken cancellationToken)
{
var activeCutoff = DateTimeOffset.UtcNow - options.MatchIdentityActiveAge;
var identities = await LoadIdentities(context)
.OrderByDescending(identity => identity.References.Count)
.ThenBy(identity => identity.Id)
.ToListAsync(cancellationToken);
var candidates = identities
.Select(identity => new
{
Identity = identity,
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
IsActive = identity.UpdatedAt >= activeCutoff
})
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
.Where(candidate => !SpeakerIdentityNaming.MatchesAnyAcceptedName(candidate.Identity, identifiedNames))
.Where(candidate => candidate.Identity.VoiceVectors.Any(vector =>
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
.OrderByDescending(candidate => candidate.IsAttendee)
.ThenByDescending(candidate => candidate.Identity.ReferenceCount)
.ThenBy(candidate => candidate.Identity.Id)
.Take(Math.Max(1, options.MaxMatchCandidates))
.Select(candidate => new ResemblyzerVoiceVectorCandidate(
candidate.Identity.Id,
SpeakerVoiceVectors.DecodeCompatible(
candidate.Identity,
resemblyzerOptions.ModelId,
logger)))
.Where(candidate => candidate.Vectors.Count > 0)
.ToList();
var match = clusterMatcher.Match(queryVectors, candidates);
return (
match.IdentityId is { } identityId
? identities.Single(identity => identity.Id == identityId)
: null,
match);
}
private void LearnUnmatchedSpeakers(
SpeakerIdentityDbContext context,
MeetingNote meetingNote,
IReadOnlyList<string> attendees,
IReadOnlySet<string> matchedAcceptedNames,
IReadOnlyList<(string Speaker, IReadOnlyList<float[]> Vectors)> unmatchedSpeakers,
ICollection<PendingIdentificationAudit> pendingAudits)
{
var candidates = attendees
.Except(matchedAcceptedNames, StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
if (candidates.Count == 0)
{
return;
}
foreach (var (speaker, vectors) in unmatchedSpeakers)
{
var now = DateTimeOffset.UtcNow;
var identity = new SpeakerIdentity
{
CanonicalName = candidates.Count == 1 ? candidates[0] : null,
CreatedAt = now,
UpdatedAt = now,
CandidateNames = candidates
.Select(name => new SpeakerCandidateName { Name = name })
.ToList(),
References = [CreateReference(meetingNote, now)]
};
outlierPruner.AddAndPrune(
identity,
vectors,
now);
context.SpeakerIdentities.Add(identity);
logger.LogInformation(
"Created Resemblyzer identity candidate for {Speaker} with {VectorCount} vector(s) and candidates {Candidates}",
speaker,
identity.VoiceVectors.Count,
string.Join(", ", candidates));
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
{
pendingAudits.Add(new PendingIdentificationAudit(
identity,
speaker,
identity.CanonicalName));
}
}
}
private async Task AppendAuditsAsync(
IEnumerable<PendingIdentificationAudit> pendingAudits,
CancellationToken cancellationToken)
{
foreach (var audit in pendingAudits)
{
try
{
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
audit.Identity.References,
audit.Speaker,
audit.Name,
cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogError(
exception,
"Resemblyzer identity {IdentityId} was saved, but its transcript identification audit could not be written",
audit.Identity.Id);
}
}
}
private async Task<IReadOnlyList<float[]>> ResolveAutomaticVectorsAsync(
SpeakerIdentificationRequest request,
string speaker,
bool allowAudioFallback,
CancellationToken cancellationToken)
{
var requiredCount = resemblyzerOptions.RequiredVectorsPerSpeaker;
var samples = await ResolveWavSamplesAsync(
request,
speaker,
resemblyzerOptions.MaxVectorsPerIdentity,
allowAudioFallback,
cancellationToken);
if (samples.Count < requiredCount)
{
return [];
}
return await encoder.EncodeAsync(samples, cancellationToken);
}
private async Task PersistMappedSpeakerEvidenceAsync(
SpeakerIdentityDbContext context,
SpeakerIdentificationRequest request,
IReadOnlyDictionary<string, string> knownMappings,
CancellationToken cancellationToken)
{
var identities = await LoadIdentities(context).ToListAsync(cancellationToken);
foreach (var (speaker, mappedName) in knownMappings)
{
var identity = identities
.Where(candidate => SpeakerIdentityNaming.GetAcceptedNames(candidate).Contains(mappedName))
.OrderBy(candidate =>
string.Equals(candidate.CanonicalName, mappedName, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
.ThenBy(candidate => candidate.Id)
.FirstOrDefault();
if (identity is null)
{
logger.LogWarning(
"Could not retain final Resemblyzer evidence for mapped speaker {Speaker}: identity {MappedName} was not found",
speaker,
mappedName);
continue;
}
var vectors = await ResolveAvailableVectorsAsync(request, speaker, cancellationToken);
if (vectors.Count == 0)
{
continue;
}
var now = DateTimeOffset.UtcNow;
var previousReferenceCount = identity.References.Count;
var vectorUpdate = outlierPruner.AddAndPrune(
identity,
vectors,
now);
SpeakerIdentityReferences.AddIfMissing(
identity,
CreateReference(request.MeetingNote, now),
now);
if (identity.References.Count != previousReferenceCount)
{
identity.UpdatedAt = now;
}
logger.LogInformation(
"Retained {AddedVectorCount} new final Resemblyzer vector(s) for mapped speaker {Speaker} as identity {IdentityId}",
vectorUpdate.AddedCount,
speaker,
identity.Id);
}
}
private async Task<IReadOnlyList<float[]>> ResolveAvailableVectorsAsync(
SpeakerIdentificationRequest request,
string speaker,
CancellationToken cancellationToken)
{
var samples = await ResolveWavSamplesAsync(
request,
speaker,
resemblyzerOptions.MaxVectorsPerIdentity,
allowAudioFallback: true,
cancellationToken);
if (samples.Count == 0)
{
return [];
}
return await encoder.EncodeAsync(samples, cancellationToken);
}
private async Task<IReadOnlyList<byte[]>> ResolveWavSamplesAsync(
SpeakerIdentificationRequest request,
string speaker,
int maxSamples,
bool allowAudioFallback,
CancellationToken cancellationToken)
{
var suppliedSamples = request.Samples?
.Where(sample => string.Equals(sample.Speaker, speaker, StringComparison.OrdinalIgnoreCase))
.Where(sample => sample.WavBytes.Length > 0)
.OrderByDescending(sample => sample.Score)
.Take(maxSamples)
.ToList()
?? [];
var wavSamples = suppliedSamples
.Select(sample => sample.WavBytes)
.ToList();
if (!allowAudioFallback || wavSamples.Count >= maxSamples)
{
return wavSamples;
}
var spans = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpans(
request.Segments,
speaker,
options.MinimumSampleSpeechDuration,
options.MaximumSampleDuration,
request.Segments.Count);
foreach (var span in spans.Where(span => !OverlapsSuppliedSample(span, suppliedSamples)))
{
var wavBytes = await snippetExtractor.ExtractSnippetAsync(
request.AudioPath,
span,
cancellationToken);
if (wavBytes.Length > 0)
{
wavSamples.Add(wavBytes);
}
if (wavSamples.Count >= maxSamples)
{
break;
}
}
logger.LogInformation(
"Resolved {SampleCount}/{RequestedSampleCount} Resemblyzer WAV samples for {Speaker}: {SuppliedSampleCount} supplied, {ExtractedSampleCount} extracted from completed audio",
wavSamples.Count,
maxSamples,
speaker,
suppliedSamples.Count,
wavSamples.Count - suppliedSamples.Count);
return wavSamples;
}
private static bool OverlapsSuppliedSample(
IReadOnlyList<TranscriptionSegment> span,
IReadOnlyList<SpeakerAudioSample> suppliedSamples)
{
if (span.Count == 0)
{
return false;
}
var start = span[0].Start;
var end = span[^1].End;
return suppliedSamples.Any(sample => sample.Segment.Start < end && sample.Segment.End > start);
}
private static IQueryable<SpeakerIdentity> LoadIdentities(SpeakerIdentityDbContext context)
{
return context.SpeakerIdentities
.AsSplitQuery()
.Include(identity => identity.Aliases)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.References);
}
private void UpdateMatchedIdentity(SpeakerIdentity identity, IReadOnlyList<string> attendees)
{
identity.UpdatedAt = DateTimeOffset.UtcNow;
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
{
return;
}
SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
}
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
{
return SpeakerIdentityReferences.Create(meetingNote.Path, meetingNote.Frontmatter.Transcript, timestamp);
}
private static SpeakerIdentificationResult EmptyResult(IReadOnlyList<TranscriptionSegment> segments)
{
return new SpeakerIdentificationResult(segments, new Dictionary<string, string>());
}
private sealed record PendingIdentificationAudit(
SpeakerIdentity Identity,
string Speaker,
string Name);
}
@@ -0,0 +1,172 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public sealed class ResemblyzerSpeakerIdentityMergeService : ISpeakerIdentityMergeService
{
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
private readonly ResemblyzerVoiceClusterMatcher matcher;
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
private readonly SpeakerIdentificationOptions options;
private readonly ResemblyzerSpeakerRecognitionOptions resemblyzerOptions;
private readonly ILogger<ResemblyzerSpeakerIdentityMergeService> logger;
public ResemblyzerSpeakerIdentityMergeService(
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
ResemblyzerVoiceClusterMatcher matcher,
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
IOptions<MeetingAssistantOptions> options,
ILogger<ResemblyzerSpeakerIdentityMergeService> logger)
{
this.dbContextFactory = dbContextFactory;
this.matcher = matcher;
this.outlierPruner = outlierPruner;
this.options = options.Value.SpeakerIdentification;
resemblyzerOptions = this.options.Resemblyzer;
this.logger = logger;
}
public async Task<SpeakerIdentityMergeResult> MergeRecentIdentitiesAsync(
TimeSpan? recentIdentityAge,
CancellationToken cancellationToken)
{
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var identities = await context.SpeakerIdentities
.AsSplitQuery()
.Include(identity => identity.Aliases)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.References)
.OrderByDescending(identity => identity.References.Count)
.ThenBy(identity => identity.Id)
.ToListAsync(cancellationToken);
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
var recentIds = identities
.Where(identity => identity.CreatedAt >= cutoff)
.Select(identity => identity.Id)
.ToList();
var required = resemblyzerOptions.RequiredVectorsPerSpeaker;
var attempts = 0;
var mergedPairs = 0;
var pendingAudits = new List<PendingMergeAudit>();
foreach (var sourceId in recentIds)
{
var source = identities.SingleOrDefault(identity => identity.Id == sourceId);
if (source is null)
{
continue;
}
var sourceVectors = SpeakerVoiceVectors.DecodeCompatible(
source,
resemblyzerOptions.ModelId,
logger)
.Take(required * 2)
.ToList();
if (sourceVectors.Count < required * 2)
{
logger.LogInformation(
"Skipping Resemblyzer merge source identity {SourceIdentityId}: {VectorCount}/{RequiredVectorCount} compatible vectors",
source.Id,
sourceVectors.Count,
required * 2);
continue;
}
var candidates = identities
.Where(identity => identity.Id != source.Id)
.Where(identity => identity.VoiceVectors.Any(vector =>
string.Equals(vector.ModelId, resemblyzerOptions.ModelId, StringComparison.Ordinal)))
.Take(Math.Max(1, options.MaxMatchCandidates))
.Select(identity => new ResemblyzerVoiceVectorCandidate(
identity.Id,
SpeakerVoiceVectors.DecodeCompatible(
identity,
resemblyzerOptions.ModelId,
logger)))
.Where(candidate => candidate.Vectors.Count > 0)
.ToList();
if (candidates.Count == 0)
{
continue;
}
attempts++;
var first = matcher.Match(sourceVectors.Take(required).ToList(), candidates);
if (!first.Accepted || first.IdentityId is not { } targetId)
{
continue;
}
attempts++;
var second = matcher.Match(sourceVectors.Skip(required).Take(required).ToList(), candidates);
if (!second.Accepted || second.IdentityId != targetId)
{
logger.LogInformation(
"Rejected Resemblyzer merge for source identity {SourceIdentityId}: disjoint clusters selected {FirstIdentityId} and {SecondIdentityId}",
source.Id,
first.IdentityId,
second.IdentityId);
continue;
}
var target = identities.SingleOrDefault(identity => identity.Id == targetId);
if (target is null)
{
continue;
}
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
SpeakerIdentityMerger.MergeIntoAndPrune(
target,
source,
options.MaxSnippetsPerSpeaker,
resemblyzerOptions.MaxVectorsPerIdentity,
outlierPruner);
pendingAudits.Add(new PendingMergeAudit(
target,
targetName,
sourceName));
context.SpeakerIdentities.Remove(source);
identities.Remove(source);
mergedPairs++;
}
await context.SaveChangesAsync(cancellationToken);
foreach (var audit in pendingAudits)
{
try
{
await SpeakerIdentityTranscriptAudit.AppendMergedAsync(
audit.Identity.References,
audit.TargetName,
audit.SourceName,
cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogError(
exception,
"Resemblyzer identity merge was saved for target {IdentityId}, but its transcript audit could not be written",
audit.Identity.Id);
}
}
return new SpeakerIdentityMergeResult(
recentIds.Count,
identities.Count,
attempts,
mergedPairs);
}
private sealed record PendingMergeAudit(
SpeakerIdentity Identity,
string TargetName,
string SourceName);
}
@@ -0,0 +1,204 @@
namespace MeetingAssistant.Speakers;
public sealed record ResemblyzerVoiceVectorCandidate(
int IdentityId,
IReadOnlyList<float[]> Vectors);
public sealed record ResemblyzerVoiceClusterMatchResult(
int? IdentityId,
string Reason,
double Cohesion,
double? BestSimilarity,
double? RunnerUpSimilarity)
{
public bool Accepted => IdentityId.HasValue;
}
public sealed class ResemblyzerVoiceClusterMatcher
{
private readonly ResemblyzerSpeakerRecognitionOptions options;
private readonly ILogger<ResemblyzerVoiceClusterMatcher> logger;
public ResemblyzerVoiceClusterMatcher(
ResemblyzerSpeakerRecognitionOptions options,
ILogger<ResemblyzerVoiceClusterMatcher> logger)
{
this.options = options;
this.logger = logger;
}
public ResemblyzerVoiceClusterMatchResult Match(
IReadOnlyList<float[]> queryVectors,
IReadOnlyList<ResemblyzerVoiceVectorCandidate> candidates)
{
var requiredCount = options.RequiredVectorsPerSpeaker;
if (queryVectors.Count < requiredCount)
{
return Reject(
$"insufficient query vectors ({queryVectors.Count}/{requiredCount})",
cohesion: 0,
bestSimilarity: null,
runnerUpSimilarity: null);
}
var normalizedQuery = queryVectors
.Take(requiredCount)
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
.ToList();
var cohesion = MeanPairwiseCosine(normalizedQuery);
if (cohesion < options.MinimumClusterCohesion)
{
return Reject(
$"query cohesion {cohesion:F4} is below {options.MinimumClusterCohesion:F4}",
cohesion,
bestSimilarity: null,
runnerUpSimilarity: null);
}
var scored = candidates
.Select(candidate => ScoreCandidate(normalizedQuery, candidate))
.Where(candidate => candidate is not null)
.Select(candidate => candidate!.Value)
.OrderByDescending(candidate => candidate.Similarity)
.ThenBy(candidate => candidate.IdentityId)
.ToList();
if (scored.Count == 0)
{
return Reject("no compatible identity vectors were available", cohesion, null, null);
}
var best = scored[0];
var runnerUp = scored.Count > 1 ? scored[1].Similarity : (double?)null;
if (best.Similarity < options.MinimumIdentitySimilarity)
{
return Reject(
$"best similarity {best.Similarity:F4} is below {options.MinimumIdentitySimilarity:F4}",
cohesion,
best.Similarity,
runnerUp);
}
if (runnerUp is { } runnerUpSimilarity &&
best.Similarity - runnerUpSimilarity < options.MinimumSimilarityMargin)
{
return Reject(
$"similarity margin {best.Similarity - runnerUpSimilarity:F4} is below {options.MinimumSimilarityMargin:F4}",
cohesion,
best.Similarity,
runnerUpSimilarity);
}
logger.LogInformation(
"Resemblyzer cluster accepted identity {IdentityId}: cohesion {Cohesion:F4}, similarity {Similarity:F4}, runner-up {RunnerUpSimilarity}",
best.IdentityId,
cohesion,
best.Similarity,
runnerUp);
return new ResemblyzerVoiceClusterMatchResult(
best.IdentityId,
"accepted",
cohesion,
best.Similarity,
runnerUp);
}
private ResemblyzerVoiceClusterMatchResult Reject(
string reason,
double cohesion,
double? bestSimilarity,
double? runnerUpSimilarity)
{
logger.LogInformation(
"Resemblyzer cluster rejected: {Reason}; cohesion {Cohesion:F4}, best {BestSimilarity}, runner-up {RunnerUpSimilarity}",
reason,
cohesion,
bestSimilarity,
runnerUpSimilarity);
return new ResemblyzerVoiceClusterMatchResult(
null,
reason,
cohesion,
bestSimilarity,
runnerUpSimilarity);
}
private (int IdentityId, double Similarity)? ScoreCandidate(
IReadOnlyList<float[]> normalizedQuery,
ResemblyzerVoiceVectorCandidate candidate)
{
if (candidate.Vectors.Count == 0)
{
return null;
}
try
{
var normalizedCandidate = candidate.Vectors
.Select(vector => SpeakerVoiceVectors.Normalize(vector, "Voice vector"))
.ToList();
var centroid = SpeakerVoiceVectors.Normalize(Centroid(normalizedCandidate), "Voice-vector centroid");
var similarities = normalizedQuery
.Select(vector => SpeakerVoiceVectors.Cosine(vector, centroid))
.Order()
.ToList();
return (candidate.IdentityId, Median(similarities));
}
catch (InvalidDataException exception)
{
logger.LogWarning(
exception,
"Skipping invalid Resemblyzer evidence for identity {IdentityId}",
candidate.IdentityId);
return null;
}
}
private static float[] Centroid(IReadOnlyList<float[]> vectors)
{
var centroid = new float[ResemblyzerVectorContract.Dimensions];
foreach (var vector in vectors)
{
for (var index = 0; index < centroid.Length; index++)
{
centroid[index] += vector[index];
}
}
for (var index = 0; index < centroid.Length; index++)
{
centroid[index] /= vectors.Count;
}
return centroid;
}
private static double MeanPairwiseCosine(IReadOnlyList<float[]> vectors)
{
if (vectors.Count < 2)
{
return 1;
}
double total = 0;
var pairs = 0;
for (var first = 0; first < vectors.Count - 1; first++)
{
for (var second = first + 1; second < vectors.Count; second++)
{
total += SpeakerVoiceVectors.Cosine(vectors[first], vectors[second]);
pairs++;
}
}
return total / pairs;
}
private static double Median(IReadOnlyList<double> sortedValues)
{
var middle = sortedValues.Count / 2;
return sortedValues.Count % 2 == 0
? (sortedValues[middle - 1] + sortedValues[middle]) / 2
: sortedValues[middle];
}
}
@@ -0,0 +1,208 @@
namespace MeetingAssistant.Speakers;
public sealed record ResemblyzerVoiceVectorUpdateResult(
int AddedCount,
int RemovedCount)
{
public bool Changed => AddedCount > 0 || RemovedCount > 0;
}
public sealed class ResemblyzerVoiceVectorOutlierPruner
{
private readonly ResemblyzerSpeakerRecognitionOptions options;
private readonly ILogger<ResemblyzerVoiceVectorOutlierPruner> logger;
public ResemblyzerVoiceVectorOutlierPruner(
ResemblyzerSpeakerRecognitionOptions options,
ILogger<ResemblyzerVoiceVectorOutlierPruner> logger)
{
this.options = options;
this.logger = logger;
}
public int Prune(SpeakerIdentity identity)
{
var compatible = new List<DecodedSpeakerVoiceVector>();
foreach (var entry in SpeakerVoiceVectors.DecodeCompatibleEntries(
identity,
options.ModelId,
logger))
{
try
{
compatible.Add(new DecodedSpeakerVoiceVector(
entry.Stored,
SpeakerVoiceVectors.Normalize(
entry.Vector,
$"Stored voice vector {entry.Stored.Id}")));
}
catch (InvalidDataException exception)
{
logger.LogWarning(
exception,
"Preserving invalid stored voice vector {VectorId} while pruning identity {IdentityId}",
entry.Stored.Id,
identity.Id);
}
}
var minimumVectors = options.OutlierPruningMinimumVectors;
if (compatible.Count < minimumVectors)
{
return 0;
}
var clusters = FindDensityClusters(compatible.Select(item => item.Vector).ToList());
var ranked = clusters
.GroupBy(cluster => cluster)
.Where(group => group.Key > 0)
.Select(group => new { Id = group.Key, Count = group.Count() })
.OrderByDescending(group => group.Count)
.ThenBy(group => group.Id)
.ToList();
if (ranked.Count == 0)
{
return Skip(identity, "no dense cluster was found");
}
if (ranked.Count > 1 && ranked[0].Count == ranked[1].Count)
{
return Skip(identity, "no uniquely largest dense cluster was found");
}
var dominant = ranked[0];
var ratio = (double)dominant.Count / compatible.Count;
if (ratio < options.OutlierPruningMinimumClusterRatio)
{
return Skip(
identity,
$"largest dense cluster ratio {ratio:F4} is below {options.OutlierPruningMinimumClusterRatio:F4}");
}
var removed = compatible
.Where((_, index) => clusters[index] != dominant.Id)
.Select(item => item.Stored)
.ToList();
foreach (var vector in removed)
{
identity.VoiceVectors.Remove(vector);
}
if (removed.Count > 0)
{
identity.UpdatedAt = DateTimeOffset.UtcNow;
}
logger.LogInformation(
"Pruned {RemovedCount} Resemblyzer voice-vector outliers from identity {IdentityId}; retained dominant cluster of {RetainedCount}/{CompatibleCount} vectors",
removed.Count,
identity.Id,
dominant.Count,
compatible.Count);
return removed.Count;
}
public ResemblyzerVoiceVectorUpdateResult AddAndPrune(
SpeakerIdentity identity,
IReadOnlyList<float[]> vectors,
DateTimeOffset createdAt)
{
var removedBeforeAdding = Prune(identity);
var added = SpeakerVoiceVectors.AddDistinct(
identity,
vectors,
options.ModelId,
options.MaxVectorsPerIdentity,
createdAt);
var removedAfterAdding = Prune(identity);
var result = new ResemblyzerVoiceVectorUpdateResult(
added,
removedBeforeAdding + removedAfterAdding);
if (result.Changed)
{
identity.UpdatedAt = createdAt;
}
return result;
}
private int[] FindDensityClusters(IReadOnlyList<float[]> vectors)
{
var neighbors = Enumerable.Range(0, vectors.Count)
.Select(index => Enumerable.Range(0, vectors.Count)
.Where(candidate => SpeakerVoiceVectors.Cosine(vectors[index], vectors[candidate]) >= options.OutlierPruningNeighborSimilarity)
.ToList())
.ToList();
var minimumNeighbors = options.OutlierPruningMinimumNeighbors;
var labels = new int[vectors.Count];
var clusterId = 0;
for (var point = 0; point < vectors.Count; point++)
{
if (labels[point] != 0)
{
continue;
}
if (neighbors[point].Count < minimumNeighbors)
{
labels[point] = -1;
continue;
}
clusterId++;
ExpandCluster(point, clusterId, neighbors, labels, minimumNeighbors);
}
return labels;
}
private static void ExpandCluster(
int seed,
int clusterId,
IReadOnlyList<List<int>> neighbors,
int[] labels,
int minimumNeighbors)
{
labels[seed] = clusterId;
var pending = new Queue<int>(neighbors[seed]);
var queued = neighbors[seed].ToHashSet();
while (pending.TryDequeue(out var point))
{
if (labels[point] == -1)
{
labels[point] = clusterId;
}
if (labels[point] != 0)
{
continue;
}
labels[point] = clusterId;
if (neighbors[point].Count < minimumNeighbors)
{
continue;
}
foreach (var neighbor in neighbors[point])
{
if (queued.Add(neighbor))
{
pending.Enqueue(neighbor);
}
}
}
}
private int Skip(
SpeakerIdentity identity,
string reason)
{
logger.LogInformation(
"Skipped Resemblyzer voice-vector pruning for identity {IdentityId}: {Reason}",
identity.Id,
reason);
return 0;
}
}
@@ -0,0 +1,54 @@
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public sealed class ResemblyzerWarmupHostedService : BackgroundService
{
private readonly IResemblyzerVoiceEncoder encoder;
private readonly bool speakerIdentificationEnabled;
private readonly ResemblyzerSpeakerRecognitionOptions options;
private readonly ILogger<ResemblyzerWarmupHostedService> logger;
public ResemblyzerWarmupHostedService(
IResemblyzerVoiceEncoder encoder,
IOptions<MeetingAssistantOptions> options,
ILogger<ResemblyzerWarmupHostedService> logger)
{
this.encoder = encoder;
speakerIdentificationEnabled = options.Value.SpeakerIdentification.Enabled;
this.options = options.Value.SpeakerIdentification.Resemblyzer;
this.logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
if (!speakerIdentificationEnabled || !options.Enabled)
{
return;
}
await Task.Yield();
try
{
logger.LogInformation(
"Starting Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
VaultPath.Resolve(options.RuntimeFolder),
options.ModelId);
await encoder.WarmUpAsync(stoppingToken);
logger.LogInformation(
"Finished Resemblyzer warm-up in runtime folder {RuntimeFolder} for model {ModelId}",
VaultPath.Resolve(options.RuntimeFolder),
options.ModelId);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
logger.LogInformation("Resemblyzer warm-up was cancelled during application shutdown");
}
catch (Exception exception)
{
logger.LogError(
exception,
"Resemblyzer warm-up failed in runtime folder {RuntimeFolder}; voice encoding can still retry on demand",
VaultPath.Resolve(options.RuntimeFolder));
}
}
}
+36 -1
View File
@@ -21,6 +21,8 @@ public sealed class SpeakerIdentity
public List<SpeakerSnippet> Snippets { get; set; } = [];
public List<SpeakerVoiceVector> VoiceVectors { get; set; } = [];
public List<SpeakerIdentityReference> References { get; set; } = [];
[NotMapped]
@@ -93,7 +95,7 @@ public static class SpeakerIdentityReferences
string.IsNullOrWhiteSpace(reference.TranscriptPath);
}
private static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
public static bool IsSame(SpeakerIdentityReference first, SpeakerIdentityReference second)
{
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
@@ -124,6 +126,25 @@ public sealed class SpeakerSnippet
public DateTimeOffset CreatedAt { get; set; }
}
public sealed class SpeakerVoiceVector
{
public int Id { get; set; }
public int SpeakerIdentityId { get; set; }
public SpeakerIdentity? SpeakerIdentity { get; set; }
public string ModelId { get; set; } = "";
public int Dimensions { get; set; }
public byte[] VectorBytes { get; set; } = [];
public string Fingerprint { get; set; } = "";
public DateTimeOffset CreatedAt { get; set; }
}
public sealed class SpeakerAlias
{
public int Id { get; set; }
@@ -150,6 +171,8 @@ public sealed class SpeakerIdentityDbContext : DbContext
public DbSet<SpeakerSnippet> SpeakerSnippets => Set<SpeakerSnippet>();
public DbSet<SpeakerVoiceVector> SpeakerVoiceVectors => Set<SpeakerVoiceVector>();
public DbSet<SpeakerIdentityReference> SpeakerIdentityReferences => Set<SpeakerIdentityReference>();
protected override void OnModelCreating(ModelBuilder modelBuilder)
@@ -175,6 +198,11 @@ public sealed class SpeakerIdentityDbContext : DbContext
.HasForeignKey(snippet => snippet.SpeakerIdentityId)
.OnDelete(DeleteBehavior.Cascade);
entity.HasMany(identity => identity.VoiceVectors)
.WithOne(vector => vector.SpeakerIdentity)
.HasForeignKey(vector => vector.SpeakerIdentityId)
.OnDelete(DeleteBehavior.Cascade);
entity.HasMany(identity => identity.References)
.WithOne(reference => reference.SpeakerIdentity)
.HasForeignKey(reference => reference.SpeakerIdentityId)
@@ -189,6 +217,13 @@ public sealed class SpeakerIdentityDbContext : DbContext
.HasIndex(alias => new { alias.SpeakerIdentityId, alias.Name })
.IsUnique();
modelBuilder.Entity<SpeakerVoiceVector>()
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.Fingerprint })
.IsUnique();
modelBuilder.Entity<SpeakerVoiceVector>()
.HasIndex(vector => new { vector.SpeakerIdentityId, vector.ModelId });
modelBuilder.Entity<SpeakerIdentityReference>()
.HasIndex(reference => new { reference.SpeakerIdentityId, reference.MeetingNotePath, reference.TranscriptPath })
.IsUnique();
@@ -114,11 +114,7 @@ public sealed class PassthroughSpeakerIdentityAttendeeCanonicalizer : ISpeakerId
IReadOnlyList<string> attendees,
CancellationToken cancellationToken)
{
var distinctAttendees = attendees
.Select(attendee => attendee.Trim())
.Where(name => !string.IsNullOrWhiteSpace(name))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
var distinctAttendees = MeetingAttendeeNames.NormalizeDistinct(attendees);
return Task.FromResult<IReadOnlyList<string>>(distinctAttendees);
}
}
@@ -20,17 +20,20 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
{
private readonly IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory;
private readonly ISpeakerIdentityMatcher matcher;
private readonly ResemblyzerVoiceVectorOutlierPruner outlierPruner;
private readonly SpeakerIdentificationOptions options;
private readonly ILogger<SpeakerIdentityMergeService> logger;
public SpeakerIdentityMergeService(
IDbContextFactory<SpeakerIdentityDbContext> dbContextFactory,
ISpeakerIdentityMatcher matcher,
ResemblyzerVoiceVectorOutlierPruner outlierPruner,
IOptions<MeetingAssistantOptions> options,
ILogger<SpeakerIdentityMergeService> logger)
{
this.dbContextFactory = dbContextFactory;
this.matcher = matcher;
this.outlierPruner = outlierPruner;
this.options = options.Value.SpeakerIdentification;
this.logger = logger;
}
@@ -44,9 +47,11 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
var cutoff = DateTimeOffset.UtcNow - (recentIdentityAge ?? options.MergeRecentIdentityAge);
var identities = await context.SpeakerIdentities
.AsSplitQuery()
.Include(identity => identity.Aliases)
.Include(identity => identity.CandidateNames)
.Include(identity => identity.Snippets)
.Include(identity => identity.VoiceVectors)
.Include(identity => identity.References)
.OrderByDescending(identity => identity.References.Count)
.ThenBy(identity => identity.Id)
@@ -154,10 +159,12 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
var targetName = target.GetDisplayName() ?? $"identity-{target.Id}";
var sourceName = source.GetDisplayName() ?? $"identity-{source.Id}";
SpeakerIdentityMerger.MergeInto(
SpeakerIdentityMerger.MergeIntoAndPrune(
target,
source,
options.MaxSnippetsPerSpeaker);
options.MaxSnippetsPerSpeaker,
options.Resemblyzer.MaxVectorsPerIdentity,
outlierPruner);
logger.LogInformation(
"Speaker identity merge diagnostics merging source identity {SourceIdentityId} ({SourceName}) into target identity {TargetIdentityId} ({TargetName})",
source.Id,
@@ -2,10 +2,24 @@ namespace MeetingAssistant.Speakers;
internal static class SpeakerIdentityMerger
{
public static void MergeIntoAndPrune(
SpeakerIdentity target,
SpeakerIdentity source,
int maxSnippets,
int maxVoiceVectors,
ResemblyzerVoiceVectorOutlierPruner outlierPruner)
{
outlierPruner.Prune(target);
outlierPruner.Prune(source);
MergeInto(target, source, maxSnippets, maxVoiceVectors);
outlierPruner.Prune(target);
}
public static void MergeInto(
SpeakerIdentity target,
SpeakerIdentity source,
int maxSnippets)
int maxSnippets,
int maxVoiceVectors = int.MaxValue)
{
AddAlias(target, source.CanonicalName);
foreach (var alias in source.Aliases)
@@ -35,6 +49,26 @@ internal static class SpeakerIdentityMerger
.ToList();
target.Snippets.Clear();
target.Snippets.AddRange(retainedSnippets);
var retainedVectors = target.VoiceVectors
.Concat(source.VoiceVectors)
.GroupBy(vector => vector.Fingerprint, StringComparer.Ordinal)
.Select(group => group.OrderByDescending(vector => vector.CreatedAt).First())
.OrderByDescending(vector => vector.CreatedAt)
.ThenBy(vector => vector.Fingerprint, StringComparer.Ordinal)
.Take(Math.Max(1, maxVoiceVectors))
.Select(vector => new SpeakerVoiceVector
{
SpeakerIdentity = target,
ModelId = vector.ModelId,
Dimensions = vector.Dimensions,
VectorBytes = vector.VectorBytes.ToArray(),
Fingerprint = vector.Fingerprint,
CreatedAt = vector.CreatedAt
})
.ToList();
target.VoiceVectors.Clear();
target.VoiceVectors.AddRange(retainedVectors);
target.UpdatedAt = DateTimeOffset.UtcNow;
}
@@ -0,0 +1,103 @@
using MeetingAssistant.MeetingNotes;
namespace MeetingAssistant.Speakers;
internal static class SpeakerIdentityNaming
{
public static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
{
return new[] { identity.CanonicalName }
.Concat(identity.Aliases.Select(alias => alias.Name))
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
.Where(name => !string.IsNullOrWhiteSpace(name))
.Select(name => name!.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
}
public static bool MatchesAnyAcceptedName(
SpeakerIdentity identity,
IReadOnlySet<string> names)
{
return GetAcceptedNames(identity).Any(names.Contains);
}
public static bool MatchesAttendees(
SpeakerIdentity identity,
IReadOnlyList<string> attendees)
{
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
}
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
{
return attendees
.Select(MeetingAttendeeNames.NormalizeDisplayName)
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
}
public static bool IsDiarizedSpeakerLabel(string speaker)
{
var normalized = speaker.Trim();
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
}
public static bool UpdateCandidateNames(
SpeakerIdentity identity,
IReadOnlyList<string> attendees)
{
if (!string.IsNullOrWhiteSpace(identity.CanonicalName) || attendees.Count == 0)
{
return false;
}
var currentCandidates = identity.CandidateNames
.Select(candidate => candidate.Name)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
var aliasToCandidate = identity.Aliases
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
var intersection = attendees
.Select(attendee => currentCandidates.Contains(attendee)
? attendee
: aliasToCandidate.GetValueOrDefault(attendee) ??
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
? fallbackAliasCandidate
: null))
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
.Select(candidate => candidate!)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
var resetToAttendees = intersection.Count == 0;
SetCandidates(identity, resetToAttendees ? attendees : intersection);
if (intersection.Count == 1)
{
identity.CanonicalName = intersection[0];
}
return resetToAttendees;
}
public static void SetCandidates(
SpeakerIdentity identity,
IEnumerable<string> candidates)
{
identity.CandidateNames.Clear();
identity.CandidateNames.AddRange(candidates
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
}
}
@@ -46,6 +46,33 @@ internal static class SpeakerIdentitySchema
ON "SpeakerIdentityReferences" ("SpeakerIdentityId", "MeetingNotePath", "TranscriptPath");
""",
cancellationToken);
await context.Database.ExecuteSqlRawAsync(
"""
CREATE TABLE IF NOT EXISTS "SpeakerVoiceVectors" (
"Id" INTEGER NOT NULL CONSTRAINT "PK_SpeakerVoiceVectors" PRIMARY KEY AUTOINCREMENT,
"SpeakerIdentityId" INTEGER NOT NULL,
"ModelId" TEXT NOT NULL,
"Dimensions" INTEGER NOT NULL,
"VectorBytes" BLOB NOT NULL,
"Fingerprint" TEXT NOT NULL,
"CreatedAt" TEXT NOT NULL,
CONSTRAINT "FK_SpeakerVoiceVectors_SpeakerIdentities_SpeakerIdentityId"
FOREIGN KEY ("SpeakerIdentityId") REFERENCES "SpeakerIdentities" ("Id") ON DELETE CASCADE
);
""",
cancellationToken);
await context.Database.ExecuteSqlRawAsync(
"""
CREATE UNIQUE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_Fingerprint"
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "Fingerprint");
""",
cancellationToken);
await context.Database.ExecuteSqlRawAsync(
"""
CREATE INDEX IF NOT EXISTS "IX_SpeakerVoiceVectors_SpeakerIdentityId_ModelId"
ON "SpeakerVoiceVectors" ("SpeakerIdentityId", "ModelId");
""",
cancellationToken);
}
private static async Task EnsureSpeakerIdentityTimestampColumnsAsync(
@@ -143,7 +143,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
target.CanonicalName = targetName;
target.UpdatedAt = now;
ResetCandidates(target, [targetName]);
SpeakerIdentityNaming.SetCandidates(target, [targetName]);
AddMeetingReference(target, meetingReference);
var snippetAdded = AddSnippetIfNeeded(target, snippet);
await context.SaveChangesAsync(cancellationToken);
@@ -207,7 +207,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var attendees = NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
var attendees = SpeakerIdentityNaming.NormalizeAttendees(request.MeetingNote.Frontmatter.Attendees);
var meetingReference = CreateReference(request.MeetingNote, DateTimeOffset.UtcNow);
var speakerMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var attendeeMatches = new List<SpeakerIdentityAttendeeMatch>();
@@ -222,7 +222,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var speaker in request.Segments
.Select(segment => segment.Speaker)
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !IsDiarizedSpeakerLabel(speaker)))
.Where(speaker => !string.IsNullOrWhiteSpace(speaker) && !SpeakerIdentityNaming.IsDiarizedSpeakerLabel(speaker)))
{
alreadyIdentifiedNames.Add(speaker.Trim());
}
@@ -360,7 +360,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
cancellationToken);
}
foreach (var acceptedName in GetAcceptedNames(identity))
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
{
matchedAcceptedNames.Add(acceptedName);
}
@@ -371,14 +371,14 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
{
speakerMappings[speaker] = speakerName;
alreadyIdentifiedNames.Add(speakerName);
foreach (var acceptedName in GetAcceptedNames(identity))
foreach (var acceptedName in SpeakerIdentityNaming.GetAcceptedNames(identity))
{
alreadyIdentifiedNames.Add(acceptedName);
}
attendeeMatches.Add(new SpeakerIdentityAttendeeMatch(
speakerName,
GetAcceptedNames(identity).ToList()));
SpeakerIdentityNaming.GetAcceptedNames(identity).ToList()));
}
}
@@ -440,7 +440,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.Select(identity => new
{
Identity = identity,
IsAttendee = MatchesAttendees(identity, attendees),
IsAttendee = SpeakerIdentityNaming.MatchesAttendees(identity, attendees),
IsActive = identity.UpdatedAt >= activeCutoff
})
.Where(candidate => candidate.IsAttendee || candidate.IsActive)
@@ -541,11 +541,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
string operation,
CancellationToken cancellationToken)
{
var span = SpeakerSampleSpanSelector.SelectBestContinuousSpan(
var span = SpeakerSampleSpanSelector.SelectBestSameSpeakerSpan(
request.Segments,
speaker,
options.MaximumSampleSegmentGap,
options.MinimumSampleSpeechDuration);
options.MinimumSampleSpeechDuration,
options.MaximumSampleDuration);
logger.LogInformation(
"{Operation} extracting fallback sample for {Speaker}: selected {SegmentCount} segment(s), span {SpanDuration}, minimum {MinimumDuration}",
operation,
@@ -569,7 +569,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.Include(identity => identity.References)
.ToListAsync(cancellationToken);
return identities
.Where(identity => GetAcceptedNames(identity).Contains(name))
.Where(identity => SpeakerIdentityNaming.GetAcceptedNames(identity).Contains(name))
.OrderBy(identity => string.Equals(identity.CanonicalName, name, StringComparison.OrdinalIgnoreCase) ? 0 : 1)
.ThenBy(identity => identity.Id)
.FirstOrDefault();
@@ -589,7 +589,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.Where(identity => string.IsNullOrWhiteSpace(identity.CanonicalName))
.ToListAsync(cancellationToken);
return candidates
.Where(identity => identity.References.Any(existing => IsSameReference(existing, reference)))
.Where(identity => identity.References.Any(existing => SpeakerIdentityReferences.IsSame(existing, reference)))
.Where(identity => identity.CandidateNames.Any(candidate =>
string.Equals(candidate.Name, targetName, StringComparison.OrdinalIgnoreCase)) ||
identity.Aliases.Any(alias =>
@@ -646,27 +646,11 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
identity.Aliases.Add(new SpeakerAlias { Name = alias.Trim() });
}
private static bool IsSameReference(
SpeakerIdentityReference first,
SpeakerIdentityReference second)
{
return string.Equals(first.MeetingNotePath, second.MeetingNotePath, StringComparison.OrdinalIgnoreCase) &&
string.Equals(first.TranscriptPath, second.TranscriptPath, StringComparison.OrdinalIgnoreCase);
}
private static bool MatchesAcceptedNames(
SpeakerIdentity identity,
IReadOnlySet<string> names)
{
return GetAcceptedNames(identity).Any(names.Contains);
}
private static bool MatchesAttendees(
SpeakerIdentity identity,
IReadOnlyList<string> attendees)
{
var attendeeSet = attendees.ToHashSet(StringComparer.OrdinalIgnoreCase);
return GetAcceptedNames(identity).Any(attendeeSet.Contains);
return SpeakerIdentityNaming.MatchesAnyAcceptedName(identity, names);
}
private static Task<SpeakerIdentity?> LoadIdentityAsync(
@@ -694,48 +678,25 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
var currentCandidates = identity.CandidateNames
.Select(candidate => candidate.Name)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var fallbackAliasCandidate = currentCandidates.Count == 1 ? currentCandidates.Single() : null;
var aliasToCandidate = identity.Aliases
.Where(alias => !string.IsNullOrWhiteSpace(alias.Name))
.SelectMany(alias => currentCandidates.Select(candidate => new { Alias = alias.Name, Candidate = candidate }))
.Where(pair => string.Equals(pair.Alias, pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
pair.Alias.Contains(pair.Candidate, StringComparison.OrdinalIgnoreCase) ||
pair.Candidate.Contains(pair.Alias, StringComparison.OrdinalIgnoreCase))
.ToDictionary(pair => pair.Alias, pair => pair.Candidate, StringComparer.OrdinalIgnoreCase);
var intersection = attendees
.Select(attendee => currentCandidates.Contains(attendee)
? attendee
: aliasToCandidate.GetValueOrDefault(attendee) ??
(identity.Aliases.Any(alias => string.Equals(alias.Name, attendee, StringComparison.OrdinalIgnoreCase))
? fallbackAliasCandidate
: null))
.Where(candidate => !string.IsNullOrWhiteSpace(candidate))
.Select(candidate => candidate!)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
if (intersection.Count == 0)
var resetToAttendees = SpeakerIdentityNaming.UpdateCandidateNames(identity, attendees);
if (resetToAttendees)
{
logger.LogInformation(
"Speaker identity candidate elimination for identity {IdentityId} had empty intersection; resetting candidates to attendees {Attendees} and replacing oldest snippet",
identity.Id,
FormatNames(attendees));
ResetCandidates(identity, attendees);
ReplaceOldestSnippet(identity, snippet);
return;
}
logger.LogInformation(
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, intersection {Intersection}",
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, remaining {RemainingCandidates}",
identity.Id,
FormatNames(currentCandidates),
FormatNames(attendees),
FormatNames(intersection));
ResetCandidates(identity, intersection);
if (intersection.Count == 1)
FormatNames(identity.CandidateNames.Select(candidate => candidate.Name)));
if (!string.IsNullOrWhiteSpace(identity.CanonicalName))
{
identity.CanonicalName = intersection[0];
logger.LogInformation(
"Speaker identity candidate elimination promoted identity {IdentityId} to canonical name {CanonicalName}",
identity.Id,
@@ -872,51 +833,6 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
}
}
private static void ResetCandidates(SpeakerIdentity identity, IReadOnlyList<string> candidates)
{
identity.CandidateNames.Clear();
identity.CandidateNames.AddRange(candidates
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.Select(candidate => new SpeakerCandidateName { Name = candidate }));
}
private static IReadOnlySet<string> GetAcceptedNames(SpeakerIdentity identity)
{
return new[]
{
identity.CanonicalName
}
.Concat(identity.Aliases.Select(alias => alias.Name))
.Concat(identity.CandidateNames.Select(candidate => candidate.Name))
.Where(name => !string.IsNullOrWhiteSpace(name))
.Select(name => name!.Trim())
.ToHashSet(StringComparer.OrdinalIgnoreCase);
}
private static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
{
return attendees
.Select(NormalizeAttendee)
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
}
private static string NormalizeAttendee(string attendee)
{
return MeetingAttendeeNames.NormalizeDisplayName(attendee);
}
private static bool IsDiarizedSpeakerLabel(string speaker)
{
var normalized = speaker.Trim();
return normalized.Equals("Unknown", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Guest", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Speaker", StringComparison.OrdinalIgnoreCase);
}
private static SpeakerIdentityReference CreateReference(MeetingNote meetingNote, DateTimeOffset timestamp)
{
return SpeakerIdentityReferences.Create(
@@ -0,0 +1,154 @@
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public sealed record SpeakerSampleCollectionPolicy(
int MinimumRetainedSamples,
bool RequireNonOverlappingSamples)
{
public static SpeakerSampleCollectionPolicy ExistingBackend { get; } = new(1, false);
public static SpeakerSampleCollectionPolicy IndependentVectors(int maximumVectors)
{
return new SpeakerSampleCollectionPolicy(Math.Max(1, maximumVectors), true);
}
public int ResolveRetainedSampleLimit(int configuredLimit)
{
return Math.Max(Math.Max(1, configuredLimit), MinimumRetainedSamples);
}
}
internal static class SpeakerSampleDurationConfiguration
{
public static void ValidateOrThrow(
SpeakerIdentificationOptions options,
string context)
{
ValidateOrThrow(
options.MinimumSampleSpeechDuration,
options.MaximumSampleDuration,
context);
}
public static void ValidateOrThrow(
TimeSpan minimumDuration,
TimeSpan maximumDuration,
string context)
{
var error = GetError(minimumDuration, maximumDuration);
if (error is not null)
{
throw new InvalidOperationException($"{context}: {error}");
}
}
public static string? GetError(
TimeSpan minimumDuration,
TimeSpan maximumDuration)
{
if (minimumDuration < TimeSpan.Zero)
{
return "SpeakerIdentification:MinimumSampleSpeechDuration must not be negative.";
}
if (maximumDuration <= TimeSpan.Zero)
{
return "SpeakerIdentification:MaximumSampleDuration must be greater than zero.";
}
return minimumDuration > maximumDuration
? "SpeakerIdentification:MinimumSampleSpeechDuration must not exceed SpeakerIdentification:MaximumSampleDuration."
: null;
}
}
internal static class ResemblyzerSpeakerRecognitionConfiguration
{
public static void ValidateOrThrow(
ResemblyzerSpeakerRecognitionOptions options,
string context)
{
var error = GetError(options);
if (error is not null)
{
throw new InvalidOperationException($"{context}: {error}");
}
}
public static string? GetError(ResemblyzerSpeakerRecognitionOptions options)
{
if (options.RequiredVectorsPerSpeaker < 1)
{
return "SpeakerIdentification:Resemblyzer:RequiredVectorsPerSpeaker must be at least one.";
}
if (options.MaxVectorsPerIdentity < options.RequiredVectorsPerSpeaker)
{
return "SpeakerIdentification:Resemblyzer:MaxVectorsPerIdentity must not be less than RequiredVectorsPerSpeaker.";
}
if (options.OutlierPruningMinimumVectors < options.RequiredVectorsPerSpeaker)
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not be less than RequiredVectorsPerSpeaker.";
}
if (options.OutlierPruningMinimumVectors > options.MaxVectorsPerIdentity)
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumVectors must not exceed MaxVectorsPerIdentity.";
}
if (options.OutlierPruningMinimumNeighbors < 1 ||
options.OutlierPruningMinimumNeighbors > options.OutlierPruningMinimumVectors)
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumNeighbors must be between one and OutlierPruningMinimumVectors.";
}
if (!IsCosine(options.OutlierPruningNeighborSimilarity))
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningNeighborSimilarity must be finite and between -1 and 1.";
}
if (!double.IsFinite(options.OutlierPruningMinimumClusterRatio) ||
options.OutlierPruningMinimumClusterRatio is <= 0 or > 1)
{
return "SpeakerIdentification:Resemblyzer:OutlierPruningMinimumClusterRatio must be finite, greater than zero, and at most one.";
}
if (!IsCosine(options.MinimumClusterCohesion))
{
return "SpeakerIdentification:Resemblyzer:MinimumClusterCohesion must be finite and between -1 and 1.";
}
if (!IsCosine(options.MinimumIdentitySimilarity))
{
return "SpeakerIdentification:Resemblyzer:MinimumIdentitySimilarity must be finite and between -1 and 1.";
}
return !double.IsFinite(options.MinimumSimilarityMargin) ||
options.MinimumSimilarityMargin is < 0 or > 2
? "SpeakerIdentification:Resemblyzer:MinimumSimilarityMargin must be finite and between 0 and 2."
: null;
}
private static bool IsCosine(double value)
{
return double.IsFinite(value) && value is >= -1 and <= 1;
}
}
internal sealed class MeetingAssistantSpeakerSampleOptionsValidator
: IValidateOptions<MeetingAssistantOptions>
{
public ValidateOptionsResult Validate(string? name, MeetingAssistantOptions options)
{
var error = SpeakerSampleDurationConfiguration.GetError(
options.SpeakerIdentification.MinimumSampleSpeechDuration,
options.SpeakerIdentification.MaximumSampleDuration) ??
ResemblyzerSpeakerRecognitionConfiguration.GetError(
options.SpeakerIdentification.Resemblyzer);
return error is null
? ValidateOptionsResult.Success
: ValidateOptionsResult.Fail(error);
}
}
@@ -0,0 +1,82 @@
using MeetingAssistant.Transcription;
namespace MeetingAssistant.Speakers;
internal sealed record SpeakerSampleSpan(
string Speaker,
TimeSpan Start,
TimeSpan End,
TimeSpan SpeechDuration,
TimeSpan MaximumDuration,
IReadOnlyList<TranscriptionSegment> Segments)
{
public static SpeakerSampleSpan? Create(
TranscriptionSegment segment,
TimeSpan maximumDuration)
{
if (maximumDuration <= TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(
nameof(maximumDuration),
maximumDuration,
"Speaker sample maximum duration must be greater than zero.");
}
var boundedEnd = Min(segment.End, segment.Start + maximumDuration);
return boundedEnd > segment.Start
? new SpeakerSampleSpan(
segment.Speaker,
segment.Start,
boundedEnd,
boundedEnd - segment.Start,
maximumDuration,
[segment with { End = boundedEnd }])
: null;
}
public bool TryExtend(
TranscriptionSegment segment,
out SpeakerSampleSpan extended)
{
extended = this;
if (!string.Equals(Speaker, segment.Speaker, StringComparison.OrdinalIgnoreCase) ||
segment.Start >= Start + MaximumDuration)
{
return false;
}
var boundedEnd = Min(segment.End, Start + MaximumDuration);
if (boundedEnd <= segment.Start)
{
return false;
}
var uncoveredStart = segment.Start > End ? segment.Start : End;
var additionalSpeechDuration = boundedEnd > uncoveredStart
? boundedEnd - uncoveredStart
: TimeSpan.Zero;
extended = this with
{
End = boundedEnd > End ? boundedEnd : End,
SpeechDuration = SpeechDuration + additionalSpeechDuration,
Segments = Segments.Append(segment with { End = boundedEnd }).ToList()
};
return true;
}
public TranscriptionSegment ToSegment()
{
return new TranscriptionSegment(
Start,
End,
Speaker,
string.Join(' ', Segments
.Select(segment => segment.Text)
.Where(text => !string.IsNullOrWhiteSpace(text))));
}
private static TimeSpan Min(TimeSpan left, TimeSpan right)
{
return left < right ? left : right;
}
}
@@ -4,56 +4,84 @@ namespace MeetingAssistant.Speakers;
internal static class SpeakerSampleSpanSelector
{
public static bool CanExtend(
string currentSpeaker,
TimeSpan currentEnd,
TranscriptionSegment nextSegment,
TimeSpan maximumSegmentGap)
{
return string.Equals(currentSpeaker, nextSegment.Speaker, StringComparison.OrdinalIgnoreCase) &&
nextSegment.Start - currentEnd <= maximumSegmentGap;
}
public static IReadOnlyList<TranscriptionSegment> SelectBestContinuousSpan(
public static IReadOnlyList<TranscriptionSegment> SelectBestSameSpeakerSpan(
IReadOnlyList<TranscriptionSegment> segments,
string speaker,
TimeSpan maximumSegmentGap,
TimeSpan minimumDuration)
TimeSpan minimumDuration,
TimeSpan maximumSampleDuration)
{
var best = new List<TranscriptionSegment>();
var current = new List<TranscriptionSegment>();
SpeakerSampleSpan? best = null;
SpeakerSampleSpan? current = null;
foreach (var segment in segments.OrderBy(segment => segment.Start))
{
if (current.Count == 0)
if (!IsSpeaker(segment, speaker))
{
if (IsSpeaker(segment, speaker))
{
current.Add(segment);
best = LongerSpan(current, best);
}
current = null;
continue;
}
if (!CanExtend(speaker, current[^1].End, segment, maximumSegmentGap))
current = ExtendOrStart(current, segment, maximumSampleDuration);
if (current is not null &&
(best is null || current.SpeechDuration > best.SpeechDuration))
{
current.Clear();
if (!IsSpeaker(segment, speaker))
{
continue;
}
best = current;
}
current.Add(segment);
best = LongerSpan(current, best);
}
return SpanDuration(best) >= minimumDuration
? best
return best is not null && best.SpeechDuration >= minimumDuration
? best.Segments
: [];
}
public static IReadOnlyList<IReadOnlyList<TranscriptionSegment>> SelectBestSameSpeakerSpans(
IReadOnlyList<TranscriptionSegment> segments,
string speaker,
TimeSpan minimumDuration,
TimeSpan maximumSampleDuration,
int maxSpans)
{
if (maxSpans <= 0)
{
return [];
}
var completed = new List<SpeakerSampleSpan>();
SpeakerSampleSpan? current = null;
TimeSpan? lastCompletedEnd = null;
foreach (var segment in segments.OrderBy(segment => segment.Start))
{
if (!IsSpeaker(segment, speaker))
{
current = null;
continue;
}
if (lastCompletedEnd is { } end && segment.Start < end)
{
current = null;
continue;
}
current = ExtendOrStart(current, segment, maximumSampleDuration);
if (current is null || current.SpeechDuration < minimumDuration)
{
continue;
}
completed.Add(current);
lastCompletedEnd = current.End;
current = null;
}
return completed
.OrderByDescending(span => span.SpeechDuration)
.ThenBy(span => span.Start)
.Take(maxSpans)
.OrderBy(span => span.Start)
.Select(span => span.Segments)
.ToList();
}
public static TimeSpan SpanDuration(IReadOnlyList<TranscriptionSegment> segments)
{
return segments.Count == 0
@@ -61,19 +89,20 @@ internal static class SpeakerSampleSpanSelector
: segments[^1].End - segments[0].Start;
}
private static SpeakerSampleSpan? ExtendOrStart(
SpeakerSampleSpan? current,
TranscriptionSegment segment,
TimeSpan maximumSampleDuration)
{
return current is not null && current.TryExtend(segment, out var extended)
? extended
: SpeakerSampleSpan.Create(segment, maximumSampleDuration);
}
private static bool IsSpeaker(
TranscriptionSegment segment,
string speaker)
{
return string.Equals(segment.Speaker, speaker, StringComparison.OrdinalIgnoreCase);
}
private static List<TranscriptionSegment> LongerSpan(
List<TranscriptionSegment> current,
List<TranscriptionSegment> best)
{
return SpanDuration(current) > SpanDuration(best)
? current.ToList()
: best;
}
}
@@ -0,0 +1,191 @@
using System.Buffers.Binary;
using System.Security.Cryptography;
using System.Text;
namespace MeetingAssistant.Speakers;
internal sealed record DecodedSpeakerVoiceVector(
SpeakerVoiceVector Stored,
float[] Vector);
internal static class ResemblyzerVectorContract
{
public const int Dimensions = 256;
}
internal static class SpeakerVoiceVectors
{
public static float[] Decode(SpeakerVoiceVector stored)
{
if (stored.Dimensions != ResemblyzerVectorContract.Dimensions ||
stored.VectorBytes.Length != stored.Dimensions * sizeof(float))
{
throw new InvalidDataException(
$"Stored voice vector {stored.Id} has invalid dimension or byte length metadata.");
}
var vector = new float[stored.Dimensions];
for (var index = 0; index < vector.Length; index++)
{
vector[index] = BinaryPrimitives.ReadSingleLittleEndian(
stored.VectorBytes.AsSpan(index * sizeof(float), sizeof(float)));
if (!float.IsFinite(vector[index]))
{
throw new InvalidDataException($"Stored voice vector {stored.Id} contains a non-finite value.");
}
}
return vector;
}
public static int AddDistinct(
SpeakerIdentity identity,
IEnumerable<float[]> vectors,
string modelId,
int maximumCount,
DateTimeOffset createdAt)
{
var limit = Math.Max(1, maximumCount);
var fingerprints = identity.VoiceVectors
.Select(vector => vector.Fingerprint)
.ToHashSet(StringComparer.Ordinal);
var added = 0;
foreach (var vector in vectors)
{
if (identity.VoiceVectors.Count >= limit)
{
break;
}
var bytes = Encode(vector);
var fingerprint = Fingerprint(modelId, bytes);
if (!fingerprints.Add(fingerprint))
{
continue;
}
identity.VoiceVectors.Add(new SpeakerVoiceVector
{
ModelId = modelId,
Dimensions = vector.Length,
VectorBytes = bytes,
Fingerprint = fingerprint,
CreatedAt = createdAt
});
added++;
}
return added;
}
public static IReadOnlyList<float[]> DecodeCompatible(
SpeakerIdentity identity,
string modelId,
ILogger logger)
{
return DecodeCompatibleEntries(identity, modelId, logger)
.Select(entry => entry.Vector)
.ToList();
}
public static IReadOnlyList<DecodedSpeakerVoiceVector> DecodeCompatibleEntries(
SpeakerIdentity identity,
string modelId,
ILogger logger)
{
var vectors = new List<DecodedSpeakerVoiceVector>();
foreach (var stored in identity.VoiceVectors
.Where(vector => string.Equals(vector.ModelId, modelId, StringComparison.Ordinal))
.OrderBy(vector => vector.CreatedAt)
.ThenBy(vector => vector.Id))
{
try
{
vectors.Add(new DecodedSpeakerVoiceVector(stored, Decode(stored)));
}
catch (InvalidDataException exception)
{
logger.LogWarning(
exception,
"Ignoring invalid stored voice vector {VectorId} for identity {IdentityId}",
stored.Id,
identity.Id);
}
}
return vectors;
}
public static double Cosine(float[] first, float[] second)
{
double dot = 0;
for (var index = 0; index < first.Length; index++)
{
dot += first[index] * second[index];
}
return dot;
}
public static float[] Normalize(float[]? vector, string description)
{
if (vector is null || vector.Length != ResemblyzerVectorContract.Dimensions)
{
throw new InvalidDataException(
$"{description} has {vector?.Length ?? 0} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
}
double magnitudeSquared = 0;
foreach (var value in vector)
{
if (!float.IsFinite(value))
{
throw new InvalidDataException($"{description} contains a non-finite value.");
}
magnitudeSquared += value * value;
}
var magnitude = Math.Sqrt(magnitudeSquared);
if (magnitude <= double.Epsilon)
{
throw new InvalidDataException($"{description} has zero magnitude.");
}
return vector.Select(value => (float)(value / magnitude)).ToArray();
}
private static byte[] Encode(float[] vector)
{
if (vector.Length != ResemblyzerVectorContract.Dimensions)
{
throw new InvalidDataException(
$"Voice vector has {vector.Length} dimensions; expected {ResemblyzerVectorContract.Dimensions}.");
}
var bytes = new byte[vector.Length * sizeof(float)];
for (var index = 0; index < vector.Length; index++)
{
if (!float.IsFinite(vector[index]))
{
throw new InvalidDataException("Voice vector contains a non-finite value.");
}
BinaryPrimitives.WriteSingleLittleEndian(
bytes.AsSpan(index * sizeof(float), sizeof(float)),
vector[index]);
}
return bytes;
}
private static string Fingerprint(string modelId, byte[] vectorBytes)
{
var modelBytes = Encoding.UTF8.GetBytes(modelId);
var input = new byte[modelBytes.Length + 1 + vectorBytes.Length];
modelBytes.CopyTo(input, 0);
input[modelBytes.Length] = 0;
vectorBytes.CopyTo(input, modelBytes.Length + 1);
return Convert.ToHexString(SHA256.HashData(input));
}
}
@@ -0,0 +1,391 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Speakers;
public interface IResemblyzerVoiceEncoder
{
Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken);
Task WarmUpAsync(CancellationToken cancellationToken);
}
public sealed partial class VenvResemblyzerVoiceEncoder : IResemblyzerVoiceEncoder
{
private const string EnvironmentSchemaVersion = "venv-v2";
private const string JsonStart = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_START__";
private const string JsonEnd = "__MEETING_ASSISTANT_RESEMBLYZER_JSON_END__";
private const string NumpyBeforePython313Requirement = "numpy<2; python_version < '3.13'";
private const string NumpyFromPython313Requirement = "numpy>=2,<3; python_version >= '3.13'";
private const string LibrosaRequirement = "librosa>=0.9.1";
private const string ScipyRequirement = "scipy>=1.2.1";
private readonly SemaphoreSlim commandLock = new(1, 1);
private readonly ICommandRunner commandRunner;
private readonly ResemblyzerSpeakerRecognitionOptions options;
private readonly ILogger<VenvResemblyzerVoiceEncoder> logger;
private string? verifiedEnvironmentPythonPath;
public VenvResemblyzerVoiceEncoder(
ICommandRunner commandRunner,
IOptions<MeetingAssistantOptions> options,
ILogger<VenvResemblyzerVoiceEncoder> logger)
{
this.commandRunner = commandRunner;
this.options = options.Value.SpeakerIdentification.Resemblyzer;
this.logger = logger;
}
public async Task<IReadOnlyList<float[]>> EncodeAsync(
IReadOnlyList<byte[]> wavSamples,
CancellationToken cancellationToken)
{
if (wavSamples.Count == 0)
{
return [];
}
if (wavSamples.Any(sample => sample.Length == 0))
{
throw new InvalidDataException("Resemblyzer cannot encode an empty WAV sample.");
}
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
var inputFolder = Path.Combine(runtimeFolder, "input", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(inputFolder);
try
{
for (var index = 0; index < wavSamples.Count; index++)
{
await File.WriteAllBytesAsync(
Path.Combine(inputFolder, $"{index:D4}.wav"),
wavSamples[index],
cancellationToken);
}
await commandLock.WaitAsync(cancellationToken);
try
{
return await RunEncodingAsync(inputFolder, wavSamples.Count, cancellationToken);
}
finally
{
commandLock.Release();
}
}
finally
{
try
{
Directory.Delete(inputFolder, recursive: true);
}
catch (DirectoryNotFoundException)
{
}
catch (IOException exception)
{
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
}
catch (UnauthorizedAccessException exception)
{
logger.LogWarning(exception, "Could not remove Resemblyzer temporary input folder {InputFolder}", inputFolder);
}
}
}
private async Task<IReadOnlyList<float[]>> RunEncodingAsync(
string inputFolder,
int expectedCount,
CancellationToken cancellationToken)
{
return await RunWithTimeoutAsync(async token =>
{
var pythonPath = await EnsureEnvironmentAsync(token);
var scriptPath = Path.Combine(VaultPath.Resolve(options.RuntimeFolder), "encode.py");
await File.WriteAllTextAsync(scriptPath, BuildEncodingScript(), token);
var result = await RunRequiredAsync(
pythonPath,
[scriptPath, Path.GetFullPath(inputFolder)],
"encoding",
token);
return ParseAndValidateVectors(result.StandardOutput, expectedCount);
}, cancellationToken);
}
public async Task WarmUpAsync(CancellationToken cancellationToken)
{
await commandLock.WaitAsync(cancellationToken);
try
{
await RunWithTimeoutAsync(async token =>
{
await EnsureEnvironmentAsync(token);
return true;
}, cancellationToken);
}
finally
{
commandLock.Release();
}
}
private async Task<string> EnsureEnvironmentAsync(CancellationToken cancellationToken)
{
ValidateDependencySettings();
var runtimeFolder = VaultPath.Resolve(options.RuntimeFolder);
var environmentFolder = Path.Combine(runtimeFolder, "venv", BuildEnvironmentKey());
var pythonPath = GetEnvironmentPythonPath(environmentFolder);
var readyPath = Path.Combine(environmentFolder, ".ready");
if (string.Equals(verifiedEnvironmentPythonPath, pythonPath, StringComparison.OrdinalIgnoreCase)
&& File.Exists(pythonPath)
&& File.Exists(readyPath))
{
return pythonPath;
}
if (File.Exists(pythonPath) && File.Exists(readyPath))
{
var verification = await commandRunner.RunAsync(
pythonPath,
WarmupArguments,
cancellationToken);
if (verification.ExitCode == 0)
{
verifiedEnvironmentPythonPath = pythonPath;
return pythonPath;
}
logger.LogWarning(
"Existing Resemblyzer environment verification failed with exit code {ExitCode}: {Error}. Reprovisioning it.",
verification.ExitCode,
verification.StandardError);
File.Delete(readyPath);
}
Directory.CreateDirectory(environmentFolder);
await RunRequiredAsync(
options.PythonCommand,
["-m", "venv", "--clear", environmentFolder],
"virtual environment creation",
cancellationToken);
await RunRequiredAsync(
pythonPath,
["-m", "pip", "install", "--upgrade", "pip"],
"pip upgrade",
cancellationToken);
await RunRequiredAsync(
pythonPath,
[
"-m", "pip", "install",
"--index-url", options.TorchIndexUrl,
TorchRequirement
],
"CPU PyTorch installation",
cancellationToken);
await RunRequiredAsync(
pythonPath,
[
"-m", "pip", "install",
.. RuntimeDependencyRequirements
],
"Resemblyzer dependency installation",
cancellationToken);
await RunRequiredAsync(
pythonPath,
[
"-m", "pip", "install", "--no-deps",
ResemblyzerRequirement
],
"Resemblyzer installation",
cancellationToken);
await RunRequiredAsync(
pythonPath,
WarmupArguments,
"environment verification",
cancellationToken);
await File.WriteAllTextAsync(readyPath, BuildEnvironmentKey(), cancellationToken);
verifiedEnvironmentPythonPath = pythonPath;
return pythonPath;
}
private async Task<CommandResult> RunRequiredAsync(
string fileName,
IReadOnlyList<string> arguments,
string operation,
CancellationToken cancellationToken)
{
var result = await commandRunner.RunAsync(fileName, arguments, cancellationToken);
ThrowIfFailed(result, operation);
return result;
}
private static string BuildEncodingScript()
{
return
"import json\n"
+ "import sys\n"
+ "import numpy as np\n"
+ "from pathlib import Path\n"
+ "from resemblyzer import VoiceEncoder, preprocess_wav\n"
+ "encoder = VoiceEncoder('cpu', verbose=False)\n"
+ "vectors = []\n"
+ "for path in sorted(Path(sys.argv[1]).glob('*.wav')):\n"
+ " wav = preprocess_wav(path)\n"
+ " vector = encoder.embed_utterance(wav)\n"
+ " vectors.append(np.asarray(vector, dtype=np.float32).tolist())\n"
+ $"print('{JsonStart}')\n"
+ "print(json.dumps(vectors, allow_nan=False))\n"
+ $"print('{JsonEnd}')\n";
}
private static IReadOnlyList<float[]> ParseAndValidateVectors(string output, int expectedCount)
{
var json = ExtractJson(output);
float[][]? vectors;
try
{
vectors = JsonSerializer.Deserialize<float[][]>(json);
}
catch (JsonException exception)
{
throw new InvalidDataException("Resemblyzer returned malformed vector JSON.", exception);
}
if (vectors is null || vectors.Length != expectedCount)
{
throw new InvalidDataException(
$"Resemblyzer returned {vectors?.Length ?? 0} vectors for {expectedCount} WAV samples.");
}
return vectors
.Select((vector, index) => SpeakerVoiceVectors.Normalize(vector, $"Resemblyzer vector {index}"))
.ToList();
}
private static string ExtractJson(string output)
{
var start = output.IndexOf(JsonStart, StringComparison.Ordinal);
if (start < 0)
{
throw new InvalidDataException("Resemblyzer output did not contain the JSON start marker.");
}
start += JsonStart.Length;
var end = output.IndexOf(JsonEnd, start, StringComparison.Ordinal);
if (end < 0)
{
throw new InvalidDataException("Resemblyzer output did not contain the JSON end marker.");
}
return output[start..end].Trim();
}
private string BuildEnvironmentKey()
{
var settings = string.Join('\n', EnvironmentManifest);
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(settings));
return Convert.ToHexString(hash)[..16].ToLowerInvariant();
}
private string TorchRequirement => $"torch=={options.TorchVersion}";
private string ResemblyzerRequirement => $"Resemblyzer=={options.PackageVersion}";
private static string[] WarmupArguments =>
[
"-c",
"from resemblyzer import VoiceEncoder; VoiceEncoder('cpu', verbose=False); print('Resemblyzer warm-up complete')"
];
private string[] RuntimeDependencyRequirements =>
[
NumpyBeforePython313Requirement,
NumpyFromPython313Requirement,
LibrosaRequirement,
ScipyRequirement,
$"webrtcvad-wheels=={options.WebRtcVadVersion}"
];
private IEnumerable<string> EnvironmentManifest =>
[
EnvironmentSchemaVersion,
options.PythonCommand,
options.TorchIndexUrl,
TorchRequirement,
.. RuntimeDependencyRequirements,
ResemblyzerRequirement
];
private static string GetEnvironmentPythonPath(string environmentFolder)
{
return OperatingSystem.IsWindows()
? Path.Combine(environmentFolder, "Scripts", "python.exe")
: Path.Combine(environmentFolder, "bin", "python");
}
private void ValidateDependencySettings()
{
if (string.IsNullOrWhiteSpace(options.PythonCommand))
{
throw new InvalidOperationException("The Resemblyzer Python command cannot be empty.");
}
ValidateVersion(options.PackageVersion, "Resemblyzer package");
ValidateVersion(options.TorchVersion, "PyTorch");
ValidateVersion(options.WebRtcVadVersion, "webrtcvad-wheels");
if (!Uri.TryCreate(options.TorchIndexUrl, UriKind.Absolute, out var indexUri)
|| indexUri.Scheme != Uri.UriSchemeHttps)
{
throw new InvalidOperationException(
$"Invalid Resemblyzer PyTorch index URL '{options.TorchIndexUrl}'.");
}
}
private static void ValidateVersion(string version, string dependency)
{
if (!PackageVersionPattern().IsMatch(version))
{
throw new InvalidOperationException($"Invalid {dependency} version '{version}'.");
}
}
private async Task<T> RunWithTimeoutAsync<T>(
Func<CancellationToken, Task<T>> operation,
CancellationToken cancellationToken)
{
using var timeoutSource = options.CommandTimeout > TimeSpan.Zero
? new CancellationTokenSource(options.CommandTimeout)
: null;
using var linkedSource = timeoutSource is null
? null
: CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
try
{
return await operation(linkedSource?.Token ?? cancellationToken);
}
catch (OperationCanceledException) when (
!cancellationToken.IsCancellationRequested && timeoutSource?.IsCancellationRequested == true)
{
throw new TimeoutException(
$"Resemblyzer command timed out after {options.CommandTimeout.ToString(null, CultureInfo.InvariantCulture)}.");
}
}
private static void ThrowIfFailed(CommandResult result, string operation)
{
if (result.ExitCode != 0)
{
throw new InvalidOperationException(
$"Resemblyzer {operation} failed with exit code {result.ExitCode}: {result.StandardError}");
}
}
[GeneratedRegex("^[0-9A-Za-z.+-]+$", RegexOptions.CultureInvariant)]
private static partial Regex PackageVersionPattern();
}
@@ -18,6 +18,12 @@ public sealed class BoundMeetingProjectResolver
public string ProjectsRoot => VaultPath.Resolve(options.Vault, options.Vault.ProjectsFolder);
public List<BoundMeetingProject> GetAllProjects() => !Directory.Exists(ProjectsRoot) ? [] :
Directory.EnumerateDirectories(ProjectsRoot)
.Select(path => new BoundMeetingProject(Path.GetFileName(path), path))
.OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase)
.ToList();
public async Task<List<BoundMeetingProject>> GetBoundProjectsAsync(
MeetingSessionArtifacts artifacts,
CancellationToken cancellationToken = default)
@@ -33,10 +39,8 @@ public sealed class BoundMeetingProjectResolver
return [];
}
return Directory.EnumerateDirectories(ProjectsRoot)
.Select(path => new BoundMeetingProject(Path.GetFileName(path), path))
return GetAllProjects()
.Where(project => projectNames.Contains(project.Name))
.OrderBy(project => project.Name, StringComparer.OrdinalIgnoreCase)
.ToList();
}
@@ -0,0 +1,26 @@
using System.Text.Json.Nodes;
using Microsoft.Extensions.AI;
namespace MeetingAssistant.Summary;
internal static class FunctionInvocationGuard
{
public static ValueTask<object?> InvokeAsync(
FunctionInvocationContext context,
CancellationToken cancellationToken)
{
if (context.CallContent.Exception is not null)
{
return ValueTask.FromResult<object?>(new JsonObject
{
["error"] = new JsonObject
{
["code"] = "invalid_tool_arguments",
["message"] = "Tool arguments must be a valid JSON object."
}
});
}
return context.Function.InvokeAsync(context.Arguments, cancellationToken);
}
}

Some files were not shown because too many files have changed in this diff Show More