Add inactivity safeguard and speaker diagnostics
PR and Push Build/Test / build-and-test (push) Failing after 8m31s

This commit is contained in:
2026-06-02 13:16:02 +02:00
parent c56ecb6ab3
commit 0e9d525b63
24 changed files with 1780 additions and 117 deletions
@@ -179,6 +179,212 @@ public sealed class RecordingCoordinatorTests
Assert.Equal(started.SummaryPath, artifactCleaner.DeletedArtifacts?.SummaryPath);
}
[Fact]
public async Task InactivitySafeguardStopsNormallyWhenPromptIsAcceptedAndUsesTranscriptEndTime()
{
var clock = new ManualMeetingInactivityClock(DateTimeOffset.Parse("2026-06-02T10:00:00+02:00"));
var promptService = new CapturingMeetingInactivityPromptService(MeetingInactivityPromptResponse.Stop);
var audioSource = new ControlledAudioSource();
var transcriptStore = new InMemoryTranscriptStore();
var noteStore = new InMemoryMeetingNoteStore();
var summaryPipeline = new CapturingMeetingSummaryPipeline();
var coordinator = new MeetingRecordingCoordinator(
audioSource,
new TestSpeechRecognitionPipelineFactory(new FixedSegmentStreamingTranscriptionProvider(
new TranscriptionSegment(
TimeSpan.FromMinutes(3),
TimeSpan.FromMinutes(4),
"Guest-01",
"The latest transcript text before silence."))),
transcriptStore,
noteStore,
new CapturingMeetingNoteOpener(),
new InMemoryMeetingArtifactStore(),
new InMemoryRecordedAudioStore(),
summaryPipeline,
Options.Create(new MeetingAssistantOptions
{
Recording =
{
InactivitySafeguard =
{
FirstPromptAfter = TimeSpan.FromSeconds(2),
ReminderPromptAfter = [],
AutoStopAfter = TimeSpan.FromMinutes(30),
InferredEndPadding = TimeSpan.FromMinutes(1),
CheckInterval = TimeSpan.FromSeconds(1)
}
}
}),
NullLogger<MeetingRecordingCoordinator>.Instance,
inactivityPromptService: promptService,
inactivityClock: clock);
await coordinator.StartAsync(CancellationToken.None);
await audioSource.WriteAsync(new AudioChunk([1, 0], 16000, 1), CancellationToken.None);
await WaitUntilAsync(() => transcriptStore.Segments.Any(segment =>
segment.Text.Contains("latest transcript text", StringComparison.Ordinal)));
clock.Advance(TimeSpan.FromSeconds(2));
await promptService.WaitForPromptAsync();
await WaitUntilAsync(() => !coordinator.CurrentStatus.IsRecording);
Assert.Single(promptService.Requests);
Assert.False(coordinator.CurrentStatus.IsRecording);
Assert.True(summaryPipeline.WasRun);
Assert.Equal(
DateTimeOffset.Parse("2026-06-02T10:05:00+02:00"),
noteStore.SavedNote?.Frontmatter.EndTime);
}
[Fact]
public async Task InactivitySafeguardAutoStopsNormallyAndUsesMeetingStartWhenNoTranscriptArrives()
{
var clock = new ManualMeetingInactivityClock(DateTimeOffset.Parse("2026-06-02T11:00:00+02:00"));
var promptService = new CapturingMeetingInactivityPromptService();
var audioSource = new ControlledAudioSource();
var noteStore = new InMemoryMeetingNoteStore();
var summaryPipeline = new CapturingMeetingSummaryPipeline();
var coordinator = new MeetingRecordingCoordinator(
audioSource,
new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()),
new InMemoryTranscriptStore(),
noteStore,
new CapturingMeetingNoteOpener(),
new InMemoryMeetingArtifactStore(),
new InMemoryRecordedAudioStore(),
summaryPipeline,
Options.Create(new MeetingAssistantOptions
{
Recording =
{
InactivitySafeguard =
{
FirstPromptAfter = TimeSpan.Zero,
ReminderPromptAfter = [],
AutoStopAfter = TimeSpan.FromSeconds(5),
InferredEndPadding = TimeSpan.FromMinutes(1),
CheckInterval = TimeSpan.FromSeconds(1)
}
}
}),
NullLogger<MeetingRecordingCoordinator>.Instance,
inactivityPromptService: promptService,
inactivityClock: clock);
await coordinator.StartAsync(CancellationToken.None);
clock.Advance(TimeSpan.FromSeconds(5));
await WaitUntilAsync(() => !coordinator.CurrentStatus.IsRecording);
Assert.Empty(promptService.Requests);
Assert.False(coordinator.CurrentStatus.IsRecording);
Assert.True(summaryPipeline.WasRun);
Assert.Equal(
DateTimeOffset.Parse("2026-06-02T11:01:00+02:00"),
noteStore.SavedNote?.Frontmatter.EndTime);
}
[Fact]
public async Task InactivitySafeguardContinuesMonitoringWhenPromptIsIgnored()
{
var clock = new ManualMeetingInactivityClock(DateTimeOffset.Parse("2026-06-02T11:30:00+02:00"));
var promptService = new IgnoringMeetingInactivityPromptService();
var audioSource = new ControlledAudioSource();
var summaryPipeline = new CapturingMeetingSummaryPipeline();
var coordinator = new MeetingRecordingCoordinator(
audioSource,
new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()),
new InMemoryTranscriptStore(),
new InMemoryMeetingNoteStore(),
new CapturingMeetingNoteOpener(),
new InMemoryMeetingArtifactStore(),
new InMemoryRecordedAudioStore(),
summaryPipeline,
Options.Create(new MeetingAssistantOptions
{
Recording =
{
InactivitySafeguard =
{
FirstPromptAfter = TimeSpan.FromSeconds(2),
ReminderPromptAfter = [],
AutoStopAfter = TimeSpan.FromSeconds(5),
CheckInterval = TimeSpan.FromSeconds(1)
}
}
}),
NullLogger<MeetingRecordingCoordinator>.Instance,
inactivityPromptService: promptService,
inactivityClock: clock);
await coordinator.StartAsync(CancellationToken.None);
clock.Advance(TimeSpan.FromSeconds(2));
await promptService.WaitForPromptAsync();
await WaitUntilAsync(() => clock.PendingDelayCount > 0);
clock.Advance(TimeSpan.FromSeconds(3));
await WaitUntilAsync(() => !coordinator.CurrentStatus.IsRecording);
Assert.Single(promptService.Requests);
Assert.True(summaryPipeline.WasRun);
}
[Fact]
public async Task InactivitySafeguardResetsPromptScheduleWhenNewTranscriptTextArrives()
{
var clock = new ManualMeetingInactivityClock(DateTimeOffset.Parse("2026-06-02T12:00:00+02:00"));
var promptService = new CapturingMeetingInactivityPromptService(
MeetingInactivityPromptResponse.Continue,
MeetingInactivityPromptResponse.Continue);
var audioSource = new ControlledAudioSource();
var transcriptStore = new InMemoryTranscriptStore();
var coordinator = new MeetingRecordingCoordinator(
audioSource,
new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()),
transcriptStore,
new InMemoryMeetingNoteStore(),
new CapturingMeetingNoteOpener(),
new InMemoryMeetingArtifactStore(),
new InMemoryRecordedAudioStore(),
new CapturingMeetingSummaryPipeline(),
Options.Create(new MeetingAssistantOptions
{
Recording =
{
InactivitySafeguard =
{
FirstPromptAfter = TimeSpan.FromSeconds(2),
ReminderPromptAfter = [TimeSpan.FromSeconds(5)],
AutoStopAfter = TimeSpan.FromMinutes(30),
CheckInterval = TimeSpan.FromSeconds(1)
}
}
}),
NullLogger<MeetingRecordingCoordinator>.Instance,
inactivityPromptService: promptService,
inactivityClock: clock);
await coordinator.StartAsync(CancellationToken.None);
await audioSource.WriteAsync(new AudioChunk([1, 0], 16000, 1), CancellationToken.None);
await WaitUntilAsync(() => transcriptStore.Segments.Any(segment =>
segment.Text.Contains("chunk:2", StringComparison.Ordinal)));
clock.Advance(TimeSpan.FromSeconds(2));
await WaitUntilAsync(() => promptService.Requests.Count == 1);
await audioSource.WriteAsync(new AudioChunk([1, 0, 2, 0], 16000, 1), CancellationToken.None);
await WaitUntilAsync(() => transcriptStore.Segments.Any(segment =>
segment.Text.Contains("chunk:4", StringComparison.Ordinal)));
clock.Advance(TimeSpan.FromSeconds(2));
await WaitUntilAsync(() => promptService.Requests.Count == 2);
Assert.Equal(
[TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(2)],
promptService.Requests.Select(request => request.Threshold).ToArray());
await coordinator.StopAsync(CancellationToken.None);
}
[Fact]
public async Task StartUsesCurrentOutlookMeetingMetadataWhenAvailable()
{
@@ -3225,6 +3431,150 @@ public sealed class RecordingCoordinatorTests
}
}
private sealed class ManualMeetingInactivityClock : IMeetingInactivityClock
{
private readonly object gate = new();
private readonly List<ScheduledDelay> delays = [];
public ManualMeetingInactivityClock(DateTimeOffset now)
{
Now = now;
}
public DateTimeOffset Now { get; private set; }
public int PendingDelayCount
{
get
{
lock (gate)
{
return delays.Count;
}
}
}
public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
if (delay <= TimeSpan.Zero)
{
return Task.CompletedTask;
}
lock (gate)
{
if (cancellationToken.IsCancellationRequested)
{
return Task.FromCanceled(cancellationToken);
}
var scheduledDelay = new ScheduledDelay(
Now + delay,
new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously));
var registration = cancellationToken.Register(
static state =>
{
var delay = (ScheduledDelay)state!;
delay.Completion.TrySetCanceled();
},
scheduledDelay);
scheduledDelay.CancellationRegistration = registration;
delays.Add(scheduledDelay);
return scheduledDelay.Completion.Task;
}
}
public void Advance(TimeSpan duration)
{
List<ScheduledDelay> due;
lock (gate)
{
Now += duration;
due = delays.Where(delay => delay.DueAt <= Now).ToList();
foreach (var delay in due)
{
delays.Remove(delay);
}
}
foreach (var delay in due)
{
delay.CancellationRegistration.Dispose();
delay.Completion.TrySetResult();
}
}
private sealed class ScheduledDelay
{
public ScheduledDelay(DateTimeOffset dueAt, TaskCompletionSource completion)
{
DueAt = dueAt;
Completion = completion;
}
public DateTimeOffset DueAt { get; }
public TaskCompletionSource Completion { get; }
public CancellationTokenRegistration CancellationRegistration { get; set; }
}
}
private sealed class CapturingMeetingInactivityPromptService : IMeetingInactivityPromptService
{
private readonly Queue<MeetingInactivityPromptResponse> responses;
private TaskCompletionSource promptObserved =
new(TaskCreationOptions.RunContinuationsAsynchronously);
public CapturingMeetingInactivityPromptService(params MeetingInactivityPromptResponse[] responses)
{
this.responses = new Queue<MeetingInactivityPromptResponse>(responses);
}
public List<MeetingInactivityPromptRequest> Requests { get; } = [];
public async Task ShowStopPromptAsync(
MeetingInactivityPromptRequest request,
Func<MeetingInactivityPromptResponse, CancellationToken, Task> handleResponseAsync,
CancellationToken cancellationToken)
{
Requests.Add(request);
promptObserved.TrySetResult();
var response = responses.Count > 0
? responses.Dequeue()
: MeetingInactivityPromptResponse.Continue;
await handleResponseAsync(response, cancellationToken);
}
public Task WaitForPromptAsync()
{
return promptObserved.Task.WaitAsync(TimeSpan.FromSeconds(5));
}
}
private sealed class IgnoringMeetingInactivityPromptService : IMeetingInactivityPromptService
{
private readonly TaskCompletionSource promptObserved =
new(TaskCreationOptions.RunContinuationsAsynchronously);
public List<MeetingInactivityPromptRequest> Requests { get; } = [];
public Task ShowStopPromptAsync(
MeetingInactivityPromptRequest request,
Func<MeetingInactivityPromptResponse, CancellationToken, Task> handleResponseAsync,
CancellationToken cancellationToken)
{
Requests.Add(request);
promptObserved.TrySetResult();
return Task.CompletedTask;
}
public Task WaitForPromptAsync()
{
return promptObserved.Task.WaitAsync(TimeSpan.FromSeconds(5));
}
}
private sealed class CapturedChunkThenCancelAudioSource : IMeetingAudioSource
{
private readonly AudioChunk chunk;
@@ -3257,6 +3607,27 @@ public sealed class RecordingCoordinatorTests
}
}
private sealed class FixedSegmentStreamingTranscriptionProvider : IStreamingTranscriptionProvider
{
private readonly TranscriptionSegment segment;
public FixedSegmentStreamingTranscriptionProvider(TranscriptionSegment segment)
{
this.segment = segment;
}
public async IAsyncEnumerable<TranscriptionSegment> TranscribeAsync(
IAsyncEnumerable<AudioChunk> audio,
SpeechRecognitionPipelineOptions options,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
await foreach (var _ in audio.WithCancellation(cancellationToken))
{
yield return segment;
}
}
}
private sealed class EchoStreamingTranscriptionProvider : IStreamingTranscriptionProvider
{
public bool FirstChunkWasObservedBeforeSourceCompleted { get; private set; }
@@ -1,5 +1,6 @@
using MeetingAssistant.Recording;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Logging;
using NAudio.Wave;
namespace MeetingAssistant.Tests;
@@ -47,6 +48,27 @@ public sealed class SpeakerAudioSampleCollectorTests
Assert.DoesNotContain(collector.Snapshot(), sample => sample.Speaker == "Guest01");
}
[Fact]
public void CollectorLogsWhenSampleIsDiscardedBecauseSpeechIsTooShort()
{
var logger = new CapturingLogger();
var collector = new SpeakerAudioSampleCollector(
TimeSpan.FromMinutes(2),
maxSamplesPerSpeaker: 3,
minimumUninterruptedSpeechDuration: TimeSpan.FromSeconds(30),
maximumSegmentGap: TimeSpan.FromSeconds(1),
logger: logger);
collector.AppendAudio(CreateAudio(TimeSpan.FromSeconds(35)));
collector.TryAdd(Segment(0, 12, "Guest01", "one two three four five."));
var message = Assert.Single(
logger.Messages,
message => message.Contains("Discarding speaker identity sample for Guest01", StringComparison.Ordinal));
Assert.Contains("duration", message);
Assert.Contains("minimum duration", message);
}
private static TranscriptionSegment Segment(
double start,
double end,
@@ -73,4 +95,30 @@ public sealed class SpeakerAudioSampleCollectorTests
using var reader = new WaveFileReader(new MemoryStream(wavBytes));
return reader.TotalTime;
}
private sealed class CapturingLogger : ILogger
{
public List<string> Messages { get; } = [];
public IDisposable? BeginScope<TState>(TState state)
where TState : notnull
{
return null;
}
public bool IsEnabled(LogLevel logLevel)
{
return true;
}
public void Log<TState>(
LogLevel logLevel,
EventId eventId,
TState state,
Exception? exception,
Func<TState, Exception?, string> formatter)
{
Messages.Add(formatter(state, exception));
}
}
}
@@ -3,6 +3,7 @@ using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
@@ -149,6 +150,28 @@ public sealed class SpeakerIdentityServiceTests
});
}
[Fact]
public async Task RejectedUnmatchedSpeakerSampleIsLoggedWithSpeakerAndCandidateContext()
{
await using var fixture = await SpeakerIdentityFixture.CreateAsync();
fixture.MatchValidator.SampleIsValid = false;
var logger = new CapturingLogger<SpeakerIdentityService>();
var service = fixture.CreateService(logger);
await service.ProcessFinishedTranscriptAsync(
fixture.CreateRequest(
["John", "Mike"],
[new TranscriptionSegment(TimeSpan.Zero, TimeSpan.FromSeconds(4), "Guest01", "unknown speaker")]),
CancellationToken.None);
var message = Assert.Single(
logger.Messages,
message => message.Contains("Skipping speaker identity candidate for Guest01", StringComparison.Ordinal));
Assert.Contains("Skipping speaker identity candidate for Guest01", message);
Assert.Contains("candidate names John, Mike", message);
Assert.Contains("sample bytes 3", message);
}
[Fact]
public async Task UnmatchedSpeakersExcludeAliasesOfMatchedIdentitiesFromCandidates()
{
@@ -648,14 +671,14 @@ public sealed class SpeakerIdentityServiceTests
return new SpeakerIdentityFixture(tempDirectory, dbPath, context, options);
}
public SpeakerIdentityService CreateService()
public SpeakerIdentityService CreateService(ILogger<SpeakerIdentityService>? logger = null)
{
return new SpeakerIdentityService(
new TestSpeakerIdentityDbContextFactory(dbPath),
SnippetExtractor,
Matcher,
Options.Create(new MeetingAssistantOptions { SpeakerIdentification = options }),
NullLogger<SpeakerIdentityService>.Instance,
logger ?? NullLogger<SpeakerIdentityService>.Instance,
MatchValidator);
}
@@ -836,4 +859,30 @@ public sealed class SpeakerIdentityServiceTests
return Task.FromResult(true);
}
}
private sealed class CapturingLogger<T> : ILogger<T>
{
public List<string> Messages { get; } = [];
public IDisposable? BeginScope<TState>(TState state)
where TState : notnull
{
return null;
}
public bool IsEnabled(LogLevel logLevel)
{
return true;
}
public void Log<TState>(
LogLevel logLevel,
EventId eventId,
TState state,
Exception? exception,
Func<TState, Exception?, string> formatter)
{
Messages.Add(formatter(state, exception));
}
}
}
@@ -1,6 +1,8 @@
using MeetingAssistant.LaunchProfiles;
using MeetingAssistant.Recording;
using MeetingAssistant.Taskbar;
using System.Drawing;
using System.Runtime.Versioning;
namespace MeetingAssistant.Tests;
@@ -81,6 +83,29 @@ public sealed class TaskbarIconTests
Assert.Equal(RecordingProcessState.Recording, menu.State);
}
[Fact]
[SupportedOSPlatform("windows")]
public void TaskbarIconGlyphsAreVisuallyCentered()
{
foreach (var state in new[]
{
RecordingProcessState.Idle,
RecordingProcessState.Recording,
RecordingProcessState.Summarizing
})
{
using var bitmap = TaskbarIconRenderer.RenderBitmap(state);
var bounds = GetVisibleGlyphBounds(bitmap);
Assert.True(
IsCentered(GetCenter(bounds.Left, bounds.Right)),
$"{state} glyph is horizontally off-center: {bounds}");
Assert.True(
IsCentered(GetCenter(bounds.Top, bounds.Bottom)),
$"{state} glyph is vertically off-center: {bounds}");
}
}
private static LaunchProfile Profile(string name, string hotkey = "")
{
return new LaunchProfile(name, new MeetingAssistantOptions
@@ -106,4 +131,42 @@ public sealed class TaskbarIconTests
state,
profile);
}
[SupportedOSPlatform("windows")]
private static Rectangle GetVisibleGlyphBounds(Bitmap bitmap)
{
var left = bitmap.Width;
var top = bitmap.Height;
var right = -1;
var bottom = -1;
for (var y = 0; y < bitmap.Height; y++)
{
for (var x = 0; x < bitmap.Width; x++)
{
var pixel = bitmap.GetPixel(x, y);
if (pixel.A < 32 || pixel.GetBrightness() < 0.58f)
{
continue;
}
left = Math.Min(left, x);
top = Math.Min(top, y);
right = Math.Max(right, x);
bottom = Math.Max(bottom, y);
}
}
Assert.True(right >= left && bottom >= top, "Expected the icon to contain a visible white glyph.");
return Rectangle.FromLTRB(left, top, right, bottom);
}
private static double GetCenter(int start, int end)
{
return (start + end) / 2.0;
}
private static bool IsCentered(double center)
{
return center is >= 7.25 and <= 8.75;
}
}
+2 -1
View File
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFrameworks>net10.0;net10.0-windows</TargetFrameworks>
<TargetFrameworks>net10.0;net10.0-windows10.0.19041.0</TargetFrameworks>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<PreserveCompilationContext>true</PreserveCompilationContext>
@@ -35,6 +35,7 @@
</ItemGroup>
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">
<PackageReference Include="CommunityToolkit.WinUI.Notifications" Version="7.1.2" />
<PackageReference Include="H.NotifyIcon.Uno.WinUI" Version="2.4.1" />
</ItemGroup>
@@ -131,6 +131,27 @@ public sealed class RecordingOptions
public TimeSpan OpenMeetingNoteDelay { get; set; } = TimeSpan.FromSeconds(1);
public string TemporaryRecordingsFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Recordings";
public RecordingInactivitySafeguardOptions InactivitySafeguard { get; set; } = new();
}
public sealed class RecordingInactivitySafeguardOptions
{
public bool Enabled { get; set; } = true;
public TimeSpan FirstPromptAfter { get; set; } = TimeSpan.FromMinutes(2);
public TimeSpan[] ReminderPromptAfter { get; set; } =
[
TimeSpan.FromMinutes(5),
TimeSpan.FromMinutes(10)
];
public TimeSpan AutoStopAfter { get; set; } = TimeSpan.FromMinutes(30);
public TimeSpan InferredEndPadding { get; set; } = TimeSpan.FromMinutes(1);
public TimeSpan CheckInterval { get; set; } = TimeSpan.FromSeconds(15);
}
public sealed class WhisperLocalOptions
+6
View File
@@ -38,6 +38,12 @@ builder.Services.AddSingleton<IMeetingArtifactStore, MarkdownMeetingArtifactStor
builder.Services.AddSingleton<IMeetingRunArtifactCleaner, MeetingRunArtifactCleaner>();
builder.Services.AddSingleton<IRecordedAudioStore, TemporaryRecordedAudioStore>();
builder.Services.AddSingleton<IDictationWordStore, MarkdownDictationWordStore>();
builder.Services.AddSingleton<IMeetingInactivityClock, SystemMeetingInactivityClock>();
#if WINDOWS
builder.Services.AddSingleton<IMeetingInactivityPromptService, WindowsMeetingInactivityPromptService>();
#else
builder.Services.AddSingleton<IMeetingInactivityPromptService, NoopMeetingInactivityPromptService>();
#endif
#if WINDOWS
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, ActiveWindowScreenshotCapture>();
#else
@@ -0,0 +1,18 @@
namespace MeetingAssistant.Recording;
public interface IMeetingInactivityClock
{
DateTimeOffset Now { get; }
Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken);
}
public sealed class SystemMeetingInactivityClock : IMeetingInactivityClock
{
public DateTimeOffset Now => DateTimeOffset.Now;
public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
return Task.Delay(delay, cancellationToken);
}
}
@@ -0,0 +1,33 @@
namespace MeetingAssistant.Recording;
public interface IMeetingInactivityPromptService
{
Task ShowStopPromptAsync(
MeetingInactivityPromptRequest request,
Func<MeetingInactivityPromptResponse, CancellationToken, Task> handleResponseAsync,
CancellationToken cancellationToken);
}
public sealed record MeetingInactivityPromptRequest(
TimeSpan InactivityDuration,
DateTimeOffset SuggestedEndTime,
int PromptIndex,
TimeSpan Threshold);
public enum MeetingInactivityPromptResponse
{
Dismissed,
Continue,
Stop
}
public sealed class NoopMeetingInactivityPromptService : IMeetingInactivityPromptService
{
public Task ShowStopPromptAsync(
MeetingInactivityPromptRequest request,
Func<MeetingInactivityPromptResponse, CancellationToken, Task> handleResponseAsync,
CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
@@ -29,6 +29,8 @@ public sealed class MeetingRecordingCoordinator
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
private readonly IMeetingScreenshotService screenshotService;
private readonly IMeetingRunArtifactCleaner artifactCleaner;
private readonly IMeetingInactivityPromptService inactivityPromptService;
private readonly IMeetingInactivityClock inactivityClock;
private readonly MeetingAssistantOptions options;
private readonly ILogger<MeetingRecordingCoordinator> logger;
private readonly SemaphoreSlim gate = new(1, 1);
@@ -55,7 +57,9 @@ public sealed class MeetingRecordingCoordinator
ILaunchProfileOptionsProvider? launchProfiles = null,
IMeetingWorkflowEngine? meetingWorkflowEngine = null,
IMeetingScreenshotService? screenshotService = null,
IMeetingRunArtifactCleaner? artifactCleaner = null)
IMeetingRunArtifactCleaner? artifactCleaner = null,
IMeetingInactivityPromptService? inactivityPromptService = null,
IMeetingInactivityClock? inactivityClock = null)
{
this.audioSource = audioSource;
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
@@ -73,6 +77,8 @@ public sealed class MeetingRecordingCoordinator
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
this.screenshotService = screenshotService ?? NoopMeetingScreenshotService.Instance;
this.artifactCleaner = artifactCleaner ?? new MeetingRunArtifactCleaner();
this.inactivityPromptService = inactivityPromptService ?? new NoopMeetingInactivityPromptService();
this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock();
this.options = options.Value;
this.logger = logger;
}
@@ -152,7 +158,7 @@ public sealed class MeetingRecordingCoordinator
var runOptions = launchProfile.Options;
currentSession = await transcriptStore.CreateSessionAsync(runOptions, cancellationToken);
var recordedAudio = await recordedAudioStore.CreateSessionAsync(runOptions, cancellationToken);
var startedAt = DateTimeOffset.Now;
var startedAt = inactivityClock.Now;
var assistantContextPath = GetAssistantContextPath(startedAt, runOptions);
var summaryPath = GetSummaryPath(startedAt, runOptions);
currentMeetingNote = await meetingNoteStore.SaveAsync(
@@ -206,8 +212,16 @@ public sealed class MeetingRecordingCoordinator
startedAt,
launchProfile.Name,
runOptions.SpeakerIdentification.LiveSampleBufferDuration,
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker);
runOptions.SpeakerIdentification.MaxSnippetsPerSpeaker,
logger);
run.Task = Task.Run(() => RecordAsync(run), CancellationToken.None);
if (runOptions.Recording.InactivitySafeguard.Enabled)
{
_ = Task.Run(
() => RunInactivitySafeguardAsync(run),
CancellationToken.None);
}
currentRun = run;
_ = Task.Run(
() => ApplyMeetingMetadataWhenAvailableAsync(startedAt, run),
@@ -231,6 +245,13 @@ public sealed class MeetingRecordingCoordinator
}
public async Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
{
return await StopAsync(null, cancellationToken);
}
private async Task<RecordingStatus> StopAsync(
DateTimeOffset? inferredEndTime,
CancellationToken cancellationToken)
{
RecordingRun run;
var abortAsTooShort = false;
@@ -244,6 +265,11 @@ public sealed class MeetingRecordingCoordinator
}
run = currentRun;
if (inferredEndTime is not null)
{
run.SetInferredEndTime(inferredEndTime.Value);
}
abortAsTooShort = ShouldAbortRunOnStop(run, DateTimeOffset.Now);
if (abortAsTooShort)
{
@@ -461,6 +487,7 @@ public sealed class MeetingRecordingCoordinator
var completedMeetingNote = await CompleteMeetingNoteAsync(
run.MeetingNotePath,
run.Options,
run.InferredEndTime,
run.TranscriptionCancellation);
if (completedMeetingNote is not null)
{
@@ -547,13 +574,121 @@ public sealed class MeetingRecordingCoordinator
{
await foreach (var segment in pipeline.ReadLiveTranscriptAsync(cancellationToken))
{
run.TryAddSpeakerSample(segment);
run.RecordTranscriptActivity(segment, inactivityClock.Now);
var sample = run.TryAddSpeakerSample(segment);
if (sample is not null)
{
logger.LogInformation(
"Collected speaker identity sample for {Speaker}: start {Start}, end {End}, duration {Duration}, score {Score}, sample bytes {SampleBytes}",
sample.Speaker,
sample.Segment.Start,
sample.Segment.End,
sample.Segment.End - sample.Segment.Start,
sample.Score,
sample.WavBytes.Length);
}
var relabeledSegment = run.Relabel(segment);
run.AddLiveSegment(relabeledSegment);
await transcriptStore.AppendAsync(run.Session, relabeledSegment, cancellationToken);
}
}
private async Task RunInactivitySafeguardAsync(RecordingRun run)
{
var safeguardOptions = run.Options.Recording.InactivitySafeguard;
var promptThresholds = GetInactivityPromptThresholds(safeguardOptions);
var promptedThresholds = new HashSet<TimeSpan>();
var lastActivityVersion = run.GetTranscriptActivitySnapshot().ActivityVersion;
logger.LogInformation(
"Recording inactivity safeguard started with prompt thresholds {PromptThresholds}, auto-stop {AutoStopAfter}, check interval {CheckInterval}",
string.Join(", ", promptThresholds),
safeguardOptions.AutoStopAfter,
safeguardOptions.CheckInterval);
try
{
while (!run.CaptureCancellation.IsCancellationRequested && !run.IsAborted)
{
var snapshot = run.GetTranscriptActivitySnapshot();
if (snapshot.ActivityVersion != lastActivityVersion)
{
promptedThresholds.Clear();
lastActivityVersion = snapshot.ActivityVersion;
logger.LogInformation(
"Recording inactivity safeguard reset after transcript activity at {LastTranscriptActivityAt}",
snapshot.LastTranscriptActivityAt);
}
var now = inactivityClock.Now;
var inactivityDuration = now - snapshot.LastTranscriptActivityAt;
if (safeguardOptions.AutoStopAfter > TimeSpan.Zero &&
inactivityDuration >= safeguardOptions.AutoStopAfter)
{
var inferredEndTime = run.GetInactivitySafeguardEndTime(safeguardOptions.InferredEndPadding);
logger.LogWarning(
"Recording inactivity safeguard auto-stopping meeting after {InactivityDuration} without transcript text; inferred end time {InferredEndTime}",
inactivityDuration,
inferredEndTime);
await StopAsync(inferredEndTime, CancellationToken.None);
return;
}
foreach (var threshold in promptThresholds)
{
if (promptedThresholds.Contains(threshold) || inactivityDuration < threshold)
{
continue;
}
promptedThresholds.Add(threshold);
var inferredEndTime = run.GetInactivitySafeguardEndTime(safeguardOptions.InferredEndPadding);
logger.LogInformation(
"Recording inactivity safeguard prompting after {InactivityDuration} without transcript text at threshold {Threshold}; inferred end time {InferredEndTime}",
inactivityDuration,
threshold,
inferredEndTime);
await inactivityPromptService.ShowStopPromptAsync(
new MeetingInactivityPromptRequest(
inactivityDuration,
inferredEndTime,
promptedThresholds.Count,
threshold),
async (response, cancellationToken) =>
{
logger.LogInformation(
"Recording inactivity safeguard prompt response {Response} at threshold {Threshold}",
response,
threshold);
if (response == MeetingInactivityPromptResponse.Stop)
{
await StopAsync(inferredEndTime, CancellationToken.None);
}
},
run.CaptureCancellation);
break;
}
await inactivityClock.DelayAsync(
GetInactivitySafeguardCheckInterval(safeguardOptions),
run.CaptureCancellation);
}
}
catch (OperationCanceledException) when (run.CaptureCancellation.IsCancellationRequested)
{
}
catch (ObjectDisposedException) when (run.CaptureCancellation.IsCancellationRequested)
{
}
catch (Exception exception)
{
logger.LogError(exception, "Recording inactivity safeguard failed");
}
}
private async Task IdentifyLiveSpeakersAsync(
RecordingRun run,
CancellationToken cancellationToken)
@@ -596,6 +731,10 @@ public sealed class MeetingRecordingCoordinator
var samples = run.GetSpeakerSamplesSnapshot();
if (segments.Count == 0 || samples.Count == 0)
{
logger.LogInformation(
"Skipping live speaker identity matching: {SegmentCount} live segment(s), {SampleCount} collected sample(s)",
segments.Count,
samples.Count);
return;
}
@@ -605,9 +744,15 @@ public sealed class MeetingRecordingCoordinator
var checkpoint = run.CreateLiveIdentificationCheckpoint(meetingNote);
if (checkpoint is null)
{
logger.LogInformation(
"Skipping live speaker identity matching because no new unmapped sample speakers or attendee changes were found");
return;
}
logger.LogInformation(
"Running live speaker identity matching for speakers {Speakers} with {SampleCount} sample(s)",
string.Join(", ", checkpoint.Speakers),
samples.Count);
var result = await speakerIdentificationService!.IdentifyKnownSpeakersAsync(
new SpeakerIdentificationRequest(
"",
@@ -830,11 +975,18 @@ public sealed class MeetingRecordingCoordinator
if (run.HasSwitchedProfile)
{
var liveSegments = run.GetLiveSegmentsSnapshot();
var liveSamples = run.GetSpeakerSamplesSnapshot();
var liveMappings = run.GetSpeakerMappingsSnapshot();
run.SetPendingFinalSpeakerIdentification(
run.RecordedAudio.AudioPath,
liveSegments,
run.GetSpeakerSamplesSnapshot(),
run.GetSpeakerMappingsSnapshot());
liveSamples,
liveMappings);
logger.LogInformation(
"Queued pending final speaker identity processing after profile switch: {SegmentCount} segment(s), {SampleCount} sample(s), {KnownMappingCount} known mapping(s)",
liveSegments.Count,
liveSamples.Count,
liveMappings.Count);
await transcriptStore.ReplaceAsync(run.Session, liveSegments, cancellationToken);
return;
}
@@ -875,6 +1027,11 @@ public sealed class MeetingRecordingCoordinator
finishedSpeakerResult.Segments,
samples,
knownSpeakerMappings);
logger.LogInformation(
"Queued pending final speaker identity processing: {SegmentCount} segment(s), {SampleCount} sample(s), {KnownMappingCount} known mapping(s)",
finishedSpeakerResult.Segments.Count,
samples.Count,
knownSpeakerMappings.Count);
await transcriptStore.ReplaceAsync(run.Session, finishedSpeakerResult.Segments, cancellationToken);
}
@@ -937,6 +1094,11 @@ public sealed class MeetingRecordingCoordinator
try
{
logger.LogInformation(
"Starting pending final speaker identity processing: {SegmentCount} segment(s), {SampleCount} sample(s), {KnownMappingCount} known mapping(s)",
pending.Segments.Count,
pending.Samples.Count,
pending.KnownSpeakerMappings.Count);
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken);
var segments = pending.Segments.ToList();
var knownSpeakerMappings = pending.KnownSpeakerMappings.ToDictionary(
@@ -948,6 +1110,10 @@ public sealed class MeetingRecordingCoordinator
cancellationToken);
foreach (var speakerOverride in speakerOverrides)
{
logger.LogInformation(
"Applying summary speaker override before final identity learning: {SourceSpeaker} -> {TargetSpeaker}",
speakerOverride.From,
speakerOverride.To);
await speakerIdentificationService.ApplySpeakerOverrideAsync(
new SpeakerIdentificationRequest(
pending.AudioPath,
@@ -971,6 +1137,10 @@ public sealed class MeetingRecordingCoordinator
cancellationToken);
foreach (var deletion in identityDeletions)
{
logger.LogInformation(
"Applying summary speaker identity deletion before final identity learning: {Identity} -> {Replacement}",
deletion.Identity,
deletion.Replacement);
await speakerIdentificationService.DeleteSpeakerIdentityAsync(
deletion.Identity,
cancellationToken);
@@ -989,6 +1159,10 @@ public sealed class MeetingRecordingCoordinator
pending.Samples,
knownSpeakerMappings),
cancellationToken);
logger.LogInformation(
"Completed final speaker identity learning: {MappingCount} mapping(s), {SegmentCount} segment(s)",
result.SpeakerMappings.Count,
result.Segments.Count);
await AddIdentifiedSpeakersToMeetingAttendeesAsync(
result.AttendeeMatches,
run.Artifacts,
@@ -1089,6 +1263,7 @@ public sealed class MeetingRecordingCoordinator
private async Task<MeetingNote?> CompleteMeetingNoteAsync(
string meetingNotePath,
MeetingAssistantOptions runOptions,
DateTimeOffset? endTime,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(meetingNotePath))
@@ -1097,7 +1272,7 @@ public sealed class MeetingRecordingCoordinator
}
var meetingNote = await meetingNoteStore.ReadAsync(meetingNotePath, cancellationToken);
meetingNote.Frontmatter.EndTime = DateTimeOffset.Now;
meetingNote.Frontmatter.EndTime = endTime ?? DateTimeOffset.Now;
var savedMeetingNote = await meetingNoteStore.SaveAsync(meetingNote, runOptions, cancellationToken);
if (string.Equals(currentMeetingNote?.Path, savedMeetingNote.Path, StringComparison.OrdinalIgnoreCase))
{
@@ -1294,6 +1469,26 @@ public sealed class MeetingRecordingCoordinator
return minimumDuration > TimeSpan.Zero && stoppedAt - run.StartedAt < minimumDuration;
}
private static IReadOnlyList<TimeSpan> GetInactivityPromptThresholds(
RecordingInactivitySafeguardOptions options)
{
return new[] { options.FirstPromptAfter }
.Concat(options.ReminderPromptAfter ?? [])
.Where(threshold => threshold > TimeSpan.Zero)
.Where(threshold => options.AutoStopAfter <= TimeSpan.Zero || threshold < options.AutoStopAfter)
.Distinct()
.Order()
.ToList();
}
private static TimeSpan GetInactivitySafeguardCheckInterval(
RecordingInactivitySafeguardOptions options)
{
return options.CheckInterval > TimeSpan.Zero
? options.CheckInterval
: TimeSpan.FromSeconds(15);
}
private async Task CompleteRunAsync(RecordingRun run)
{
if (!run.TryComplete())
@@ -1326,6 +1521,7 @@ public sealed class MeetingRecordingCoordinator
private readonly object liveSegmentsGate = new();
private readonly object liveTranscriptTasksGate = new();
private readonly object liveIdentificationGate = new();
private readonly object transcriptActivityGate = new();
private readonly List<TranscriptionSegment> liveSegments = [];
private readonly List<Task> liveTranscriptTasks = [];
private readonly List<ISpeechRecognitionPipeline> pipelines = [];
@@ -1333,6 +1529,10 @@ public sealed class MeetingRecordingCoordinator
private readonly SpeakerAudioSampleCollector speakerSampleCollector;
private LiveIdentificationCheckpoint? lastLiveIdentificationCheckpoint;
private FinalSpeakerIdentificationWork? pendingFinalSpeakerIdentification;
private DateTimeOffset lastTranscriptActivityAt;
private TimeSpan? lastTranscriptSegmentEnd;
private long transcriptActivityVersion;
private DateTimeOffset? inferredEndTime;
public RecordingRun(
CancellationTokenSource captureCancellation,
@@ -1347,7 +1547,8 @@ public sealed class MeetingRecordingCoordinator
DateTimeOffset startedAt,
string launchProfileName,
TimeSpan liveSampleBufferDuration,
int maxSpeakerSamples)
int maxSpeakerSamples,
ILogger logger)
{
CaptureCancellationSource = captureCancellation;
TranscriptionCancellationSource = transcriptionCancellation;
@@ -1360,13 +1561,15 @@ public sealed class MeetingRecordingCoordinator
PipelineOptions = pipelineOptions;
Options = options;
StartedAt = startedAt;
lastTranscriptActivityAt = startedAt;
LaunchProfileName = launchProfileName;
pipelines.Add(pipeline);
speakerSampleCollector = new SpeakerAudioSampleCollector(
liveSampleBufferDuration,
maxSpeakerSamples,
options.SpeakerIdentification.MinimumSampleSpeechDuration,
options.SpeakerIdentification.MaximumSampleSegmentGap);
options.SpeakerIdentification.MaximumSampleSegmentGap,
logger);
}
public CancellationTokenSource CaptureCancellationSource { get; }
@@ -1411,6 +1614,17 @@ public sealed class MeetingRecordingCoordinator
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
public DateTimeOffset? InferredEndTime
{
get
{
lock (transcriptActivityGate)
{
return inferredEndTime;
}
}
}
public void StopCapture()
{
CaptureCancellationSource.Cancel();
@@ -1557,14 +1771,59 @@ public sealed class MeetingRecordingCoordinator
}
}
public void RecordTranscriptActivity(TranscriptionSegment segment, DateTimeOffset arrivedAt)
{
if (string.IsNullOrWhiteSpace(segment.Text))
{
return;
}
lock (transcriptActivityGate)
{
lastTranscriptActivityAt = arrivedAt;
var segmentEnd = segment.End > TimeSpan.Zero ? segment.End : segment.Start;
lastTranscriptSegmentEnd = lastTranscriptSegmentEnd is null
? segmentEnd
: TimeSpan.FromTicks(Math.Max(lastTranscriptSegmentEnd.Value.Ticks, segmentEnd.Ticks));
transcriptActivityVersion++;
}
}
public TranscriptActivitySnapshot GetTranscriptActivitySnapshot()
{
lock (transcriptActivityGate)
{
return new TranscriptActivitySnapshot(
lastTranscriptActivityAt,
lastTranscriptSegmentEnd,
transcriptActivityVersion);
}
}
public DateTimeOffset GetInactivitySafeguardEndTime(TimeSpan padding)
{
lock (transcriptActivityGate)
{
return StartedAt + (lastTranscriptSegmentEnd ?? TimeSpan.Zero) + padding;
}
}
public void SetInferredEndTime(DateTimeOffset endTime)
{
lock (transcriptActivityGate)
{
inferredEndTime = endTime;
}
}
public void AppendAudio(AudioChunk chunk)
{
speakerSampleCollector.AppendAudio(chunk);
}
public void TryAddSpeakerSample(TranscriptionSegment segment)
public SpeakerAudioSample? TryAddSpeakerSample(TranscriptionSegment segment)
{
speakerSampleCollector.TryAdd(segment);
return speakerSampleCollector.TryAdd(segment);
}
public IReadOnlyList<TranscriptionSegment> GetLiveSegmentsSnapshot()
@@ -1683,6 +1942,11 @@ public sealed class MeetingRecordingCoordinator
PipelineGate.Dispose();
}
public sealed record TranscriptActivitySnapshot(
DateTimeOffset LastTranscriptActivityAt,
TimeSpan? LastTranscriptSegmentEnd,
long ActivityVersion);
private static int StateRank(AssistantContextState state)
{
return state switch
@@ -1,5 +1,6 @@
using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription;
using Microsoft.Extensions.Logging;
namespace MeetingAssistant.Recording;
@@ -11,6 +12,7 @@ internal sealed class SpeakerAudioSampleCollector
private readonly int maxSamplesPerSpeaker;
private readonly TimeSpan minimumUninterruptedSpeechDuration;
private readonly TimeSpan maximumSegmentGap;
private readonly ILogger? logger;
private PendingSpeakerSpan? pendingSpan;
public SpeakerAudioSampleCollector(TimeSpan bufferDuration, int maxSamplesPerSpeaker)
@@ -18,7 +20,8 @@ internal sealed class SpeakerAudioSampleCollector
bufferDuration,
maxSamplesPerSpeaker,
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(1))
TimeSpan.FromSeconds(1),
logger: null)
{
}
@@ -26,7 +29,8 @@ internal sealed class SpeakerAudioSampleCollector
TimeSpan bufferDuration,
int maxSamplesPerSpeaker,
TimeSpan minimumUninterruptedSpeechDuration,
TimeSpan maximumSegmentGap)
TimeSpan maximumSegmentGap,
ILogger? logger = null)
{
audioBuffer = new RollingAudioBuffer(bufferDuration);
this.maxSamplesPerSpeaker = Math.Max(1, maxSamplesPerSpeaker);
@@ -36,6 +40,7 @@ internal sealed class SpeakerAudioSampleCollector
this.maximumSegmentGap = maximumSegmentGap >= TimeSpan.Zero
? maximumSegmentGap
: TimeSpan.Zero;
this.logger = logger;
}
public void AppendAudio(AudioChunk chunk)
@@ -53,32 +58,61 @@ internal sealed class SpeakerAudioSampleCollector
}
}
public void TryAdd(TranscriptionSegment segment)
public SpeakerAudioSample? TryAdd(TranscriptionSegment segment)
{
if (!IsDiarizedSpeaker(segment.Speaker))
{
return;
logger?.LogInformation(
"Discarding speaker identity sample for {Speaker} because the segment has no diarized speaker label",
segment.Speaker);
return null;
}
TranscriptionSegment sampleSegment;
PendingSpanReset? reset;
lock (gate)
{
sampleSegment = ExtendPendingSpan(segment);
(sampleSegment, reset) = ExtendPendingSpan(segment);
}
if (reset is not null)
{
logger?.LogInformation(
"Reset speaker identity sample span from {PreviousSpeaker} to {Speaker}: previous end {PreviousEnd}, next start {NextStart}, gap {Gap}, maximum gap {MaximumGap}",
reset.PreviousSpeaker,
segment.Speaker,
reset.PreviousEnd,
segment.Start,
reset.Gap,
maximumSegmentGap);
}
var score = Score(sampleSegment, minimumUninterruptedSpeechDuration);
if (score <= 0)
if (!score.Accepted)
{
return;
logger?.LogInformation(
"Discarding speaker identity sample for {Speaker} because {Reason}: duration {Duration}, minimum duration {MinimumDuration}, word count {WordCount}",
sampleSegment.Speaker,
score.Reason,
sampleSegment.End - sampleSegment.Start,
minimumUninterruptedSpeechDuration,
score.WordCount);
return null;
}
var wavBytes = audioBuffer.TryExtractWav(sampleSegment.Start, sampleSegment.End);
if (wavBytes.Length == 0)
{
return;
logger?.LogInformation(
"Discarding speaker identity sample for {Speaker} because no audio could be extracted from rolling buffer: start {Start}, end {End}, duration {Duration}",
sampleSegment.Speaker,
sampleSegment.Start,
sampleSegment.End,
sampleSegment.End - sampleSegment.Start);
return null;
}
var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score);
var sample = new SpeakerAudioSample(sampleSegment.Speaker, sampleSegment, wavBytes, score.Value);
lock (gate)
{
if (!samplesBySpeaker.TryGetValue(sampleSegment.Speaker, out var samples))
@@ -88,13 +122,27 @@ internal sealed class SpeakerAudioSampleCollector
}
samples.Add(sample);
var beforeCount = samples.Count;
var bestSamples = samples
.OrderByDescending(candidate => candidate.Score)
.Take(maxSamplesPerSpeaker)
.ToList();
var retained = bestSamples.Contains(sample);
samples.Clear();
samples.AddRange(bestSamples);
if (!retained)
{
logger?.LogInformation(
"Discarding speaker identity sample for {Speaker} because it was not among the best {MaxSamplesPerSpeaker} retained sample(s): score {Score}, candidate count {CandidateCount}",
sample.Speaker,
maxSamplesPerSpeaker,
sample.Score,
beforeCount);
return null;
}
}
return sample;
}
public IReadOnlyList<SpeakerAudioSample> Snapshot()
@@ -115,39 +163,43 @@ internal sealed class SpeakerAudioSampleCollector
!string.Equals(speaker, "Unknown", StringComparison.OrdinalIgnoreCase);
}
private TranscriptionSegment ExtendPendingSpan(TranscriptionSegment segment)
private (TranscriptionSegment Segment, PendingSpanReset? Reset) ExtendPendingSpan(TranscriptionSegment segment)
{
if (pendingSpan is null ||
!SpeakerSampleSpanSelector.CanExtend(pendingSpan.Speaker, pendingSpan.End, segment, maximumSegmentGap))
{
var reset = pendingSpan is null
? null
: new PendingSpanReset(
pendingSpan.Speaker,
pendingSpan.End,
segment.Start - pendingSpan.End);
pendingSpan = new PendingSpeakerSpan(
segment.Speaker,
segment.Start,
segment.End,
[segment.Text]);
return pendingSpan.ToSegment();
return (pendingSpan.ToSegment(), reset);
}
pendingSpan = pendingSpan.Extend(segment);
return pendingSpan.ToSegment();
return (pendingSpan.ToSegment(), null);
}
private static double Score(
private static SampleScore Score(
TranscriptionSegment segment,
TimeSpan minimumUninterruptedSpeechDuration)
{
var durationSeconds = (segment.End - segment.Start).TotalSeconds;
if (durationSeconds < minimumUninterruptedSpeechDuration.TotalSeconds)
{
return 0;
return new SampleScore(false, 0, "speech duration is below the configured minimum", WordCount(segment.Text));
}
var words = segment.Text
.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Length;
var words = WordCount(segment.Text);
if (words < 3)
{
return 0;
return new SampleScore(false, 0, "word count is below the minimum useful sample length", words);
}
var durationScore = Math.Min(durationSeconds / Math.Max(1, minimumUninterruptedSpeechDuration.TotalSeconds), 2);
@@ -157,9 +209,20 @@ internal sealed class SpeakerAudioSampleCollector
segment.Text.TrimEnd().EndsWith('!')
? 5
: 0;
return durationScore * 70 + wordScore * 30 + sentenceBonus;
return new SampleScore(true, durationScore * 70 + wordScore * 30 + sentenceBonus, null, words);
}
private static int WordCount(string text)
{
return text
.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Length;
}
private sealed record SampleScore(bool Accepted, double Value, string? Reason, int WordCount);
private sealed record PendingSpanReset(string PreviousSpeaker, TimeSpan PreviousEnd, TimeSpan Gap);
private sealed record PendingSpeakerSpan(
string Speaker,
TimeSpan Start,
@@ -0,0 +1,214 @@
#if WINDOWS
using System.Collections.Concurrent;
using CommunityToolkit.WinUI.Notifications;
namespace MeetingAssistant.Recording;
public sealed class WindowsMeetingInactivityPromptService : IMeetingInactivityPromptService, IDisposable
{
private const string NotificationSource = "meeting-inactivity-safeguard";
private const string NotificationGroup = "meeting-inactivity";
private readonly ConcurrentDictionary<string, PendingPrompt> pendingPrompts = new(StringComparer.OrdinalIgnoreCase);
private readonly ILogger<WindowsMeetingInactivityPromptService> logger;
private readonly object registrationGate = new();
private bool registered;
public WindowsMeetingInactivityPromptService(ILogger<WindowsMeetingInactivityPromptService> logger)
{
this.logger = logger;
}
public Task ShowStopPromptAsync(
MeetingInactivityPromptRequest request,
Func<MeetingInactivityPromptResponse, CancellationToken, Task> handleResponseAsync,
CancellationToken cancellationToken)
{
if (!OperatingSystem.IsWindowsVersionAtLeast(10, 0, 17763))
{
logger.LogWarning("Windows app notifications require Windows 10 version 1809 or later");
return Task.CompletedTask;
}
if (cancellationToken.IsCancellationRequested)
{
return Task.FromCanceled(cancellationToken);
}
try
{
EnsureRegistered();
var promptId = Guid.NewGuid().ToString("N");
pendingPrompts[promptId] = new PendingPrompt(handleResponseAsync);
var notification = BuildNotification(promptId, request);
notification.Show(toast =>
{
toast.Group = NotificationGroup;
toast.Tag = promptId;
toast.ExpirationTime = DateTimeOffset.Now.AddMinutes(15);
});
logger.LogInformation(
"Displayed native Windows inactivity notification {PromptId} at threshold {Threshold}",
promptId,
request.Threshold);
}
catch (Exception exception)
{
logger.LogError(exception, "Failed to display native Windows inactivity notification");
}
return Task.CompletedTask;
}
public void Dispose()
{
if (!registered)
{
return;
}
try
{
ToastNotificationManagerCompat.OnActivated -= OnNotificationInvoked;
ToastNotificationManagerCompat.Uninstall();
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to unregister native Windows toast notifications");
}
}
private void EnsureRegistered()
{
if (registered)
{
return;
}
lock (registrationGate)
{
if (registered)
{
return;
}
ToastNotificationManagerCompat.OnActivated += OnNotificationInvoked;
registered = true;
logger.LogInformation("Registered native Windows toast notification activation handler");
}
}
private static ToastContentBuilder BuildNotification(
string promptId,
MeetingInactivityPromptRequest request)
{
var yesButton = new ToastButton()
.SetContent("Yes")
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", "stop")
.SetBackgroundActivation();
var noButton = new ToastButton()
.SetContent("No")
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", "continue")
.SetBackgroundActivation();
return new ToastContentBuilder()
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddText("Stop meeting?")
.AddText($"No transcript text has arrived for {FormatDuration(request.InactivityDuration)}.")
.AddButton(yesButton)
.AddButton(noButton);
}
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
{
var arguments = ParseArguments(args.Argument);
if (!TryGetArgument(arguments, "source", out var source) ||
!string.Equals(source, NotificationSource, StringComparison.OrdinalIgnoreCase) ||
!TryGetArgument(arguments, "promptId", out var promptId) ||
!pendingPrompts.TryRemove(promptId, out var pendingPrompt))
{
return;
}
var response = TryGetArgument(arguments, "response", out var responseValue) &&
string.Equals(responseValue, "stop", StringComparison.OrdinalIgnoreCase)
? MeetingInactivityPromptResponse.Stop
: MeetingInactivityPromptResponse.Continue;
_ = Task.Run(async () =>
{
try
{
logger.LogInformation(
"Native Windows inactivity notification {PromptId} activated with response {Response}",
promptId,
response);
await pendingPrompt.HandleResponseAsync(response, CancellationToken.None);
}
catch (Exception exception)
{
logger.LogError(
exception,
"Failed to handle native Windows inactivity notification response {Response}",
response);
}
});
}
private static IReadOnlyDictionary<string, string> ParseArguments(string argumentText)
{
if (string.IsNullOrWhiteSpace(argumentText))
{
return new Dictionary<string, string>();
}
var arguments = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var pair in argumentText.Split('&', StringSplitOptions.RemoveEmptyEntries))
{
var separatorIndex = pair.IndexOf('=');
if (separatorIndex < 0)
{
arguments[Uri.UnescapeDataString(pair)] = "";
continue;
}
var key = Uri.UnescapeDataString(pair[..separatorIndex]);
var value = Uri.UnescapeDataString(pair[(separatorIndex + 1)..]);
arguments[key] = value;
}
return arguments;
}
private static bool TryGetArgument(
IReadOnlyDictionary<string, string> arguments,
string key,
out string value)
{
if (arguments.TryGetValue(key, out value!))
{
return true;
}
value = "";
return false;
}
private static string FormatDuration(TimeSpan duration)
{
if (duration.TotalMinutes >= 1)
{
return $"{Math.Round(duration.TotalMinutes):0} minutes";
}
return $"{Math.Max(1, Math.Round(duration.TotalSeconds)):0} seconds";
}
private sealed record PendingPrompt(
Func<MeetingInactivityPromptResponse, CancellationToken, Task> HandleResponseAsync);
}
#endif
@@ -63,6 +63,13 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
return null;
}
logger.LogInformation(
"Speaker identity matching {DiarizedSpeaker} composite layout: unknown {UnknownStart}-{UnknownEnd}, {KnownSegmentCount} known segment(s) across identity ids {IdentityIds}",
request.DiarizedSpeaker,
layout.UnknownSegment.Value.Start,
layout.UnknownSegment.Value.End,
layout.KnownSegments.Count,
string.Join(", ", layout.KnownSegments.Select(segment => segment.IdentityId).Distinct().Order()));
var segments = await DiarizeWithTimeoutAsync(tempPath, cancellationToken);
if (segments.Count == 0)
{
@@ -76,6 +83,16 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
"Speaker identity matching {DiarizedSpeaker} received {SegmentCount} diarized segment(s)",
request.DiarizedSpeaker,
segments.Count);
foreach (var segment in segments.OrderBy(segment => segment.Start))
{
logger.LogInformation(
"Speaker identity matching {DiarizedSpeaker} diarized turn {Start}-{End} as {Speaker}",
request.DiarizedSpeaker,
segment.Start,
segment.End,
segment.Speaker);
}
var unknownSpeaker = FindBestSpeaker(layout.UnknownSegment.Value, segments);
if (string.IsNullOrWhiteSpace(unknownSpeaker))
{
@@ -93,6 +110,14 @@ public sealed class AzureSpeechSpeakerIdentityMatcher : ISpeakerIdentityMatcher
unknownSpeaker,
StringComparison.Ordinal));
var requiredMatches = known.Count() > 1 ? 2 : 1;
logger.LogInformation(
"Speaker identity matching {DiarizedSpeaker} compared unknown speaker {UnknownSpeaker} with identity {IdentityId}: {MatchingSnippetCount}/{KnownSnippetCount} matching known snippet(s), required {RequiredMatches}",
request.DiarizedSpeaker,
unknownSpeaker,
known.Key,
matchingSnippetCount,
known.Count(),
requiredMatches);
if (matchingSnippetCount >= requiredMatches)
{
var validationRequest = new SpeakerIdentityMatchValidationRequest(
@@ -30,14 +30,29 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
}
var segments = await DiarizeBytesAsync(wavBytes, cancellationToken);
var result = HasSingleSpeaker(segments);
if (!result)
var result = AnalyzeSingleSpeakerCoverage(segments);
if (!result.Accepted)
{
logger.LogInformation(
"pyannote rejected speaker identity sample because it did not contain one dominant speaker");
"pyannote rejected speaker identity sample because it did not contain one dominant speaker: segments {SegmentCount}, speakers {SpeakerCount}, dominant speaker {DominantSpeaker}, coverage {Coverage:P2}, required coverage {RequiredCoverage:P2}, duration {Duration}",
segments.Count,
result.SpeakerCount,
result.DominantSpeaker,
result.Coverage,
Math.Clamp(options.MinimumSingleSpeakerCoverage, 0, 1),
result.TotalDuration);
return false;
}
return result;
logger.LogInformation(
"pyannote accepted speaker identity sample: segments {SegmentCount}, speakers {SpeakerCount}, dominant speaker {DominantSpeaker}, coverage {Coverage:P2}, duration {Duration}, sample bytes {SampleBytes}",
segments.Count,
result.SpeakerCount,
result.DominantSpeaker,
result.Coverage,
result.TotalDuration,
wavBytes.Length);
return true;
}
public async Task<bool> ValidateMatchAsync(
@@ -63,10 +78,20 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
TimeSpan.FromSeconds(1));
if (layout.UnknownSegment is null || layout.KnownSegments.Count == 0)
{
logger.LogInformation(
"pyannote rejected speaker identity match for {DiarizedSpeaker} and identity {IdentityId} because no readable composite snippets were available",
request.DiarizedSpeaker,
request.IdentityId);
return false;
}
var segments = await DiarizePathAsync(tempPath, cancellationToken);
logger.LogInformation(
"pyannote diarized speaker identity match for {DiarizedSpeaker} and identity {IdentityId}: {SegmentCount} segment(s), {KnownSnippetCount} known snippet(s)",
request.DiarizedSpeaker,
request.IdentityId,
segments.Count,
layout.KnownSegments.Count);
var unknownSpeaker = FindBestSpeaker(layout.UnknownSegment.Value, segments);
if (string.IsNullOrWhiteSpace(unknownSpeaker))
{
@@ -98,11 +123,22 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
if (!accepted)
{
logger.LogInformation(
"pyannote rejected speaker identity match for {DiarizedSpeaker} and identity {IdentityId}: {Matching}/{Compared} known sample(s) matched",
"pyannote rejected speaker identity match for {DiarizedSpeaker} and identity {IdentityId}: {Matching}/{Compared} known sample(s) matched, required ratio {RequiredRatio:P2}",
request.DiarizedSpeaker,
request.IdentityId,
matching,
compared);
compared,
minimumRatio);
}
else
{
logger.LogInformation(
"pyannote accepted speaker identity match for {DiarizedSpeaker} and identity {IdentityId}: {Matching}/{Compared} known sample(s) matched, required ratio {RequiredRatio:P2}",
request.DiarizedSpeaker,
request.IdentityId,
matching,
compared,
minimumRatio);
}
return accepted;
@@ -162,7 +198,7 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
cancellationToken);
}
private bool HasSingleSpeaker(IReadOnlyList<TranscriptionSegment> segments)
private SingleSpeakerCoverage AnalyzeSingleSpeakerCoverage(IReadOnlyList<TranscriptionSegment> segments)
{
var durationsBySpeaker = segments
.Where(segment => !string.IsNullOrWhiteSpace(segment.Speaker) && segment.End > segment.Start)
@@ -176,17 +212,29 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
.ToList();
if (durationsBySpeaker.Count == 0)
{
return false;
}
if (durationsBySpeaker.Count == 1)
{
return true;
return new SingleSpeakerCoverage(false, 0, null, 0, TimeSpan.Zero);
}
var total = durationsBySpeaker.Sum(group => group.Duration);
var dominant = durationsBySpeaker.Max(group => group.Duration);
return total > 0 && dominant / total >= Math.Clamp(options.MinimumSingleSpeakerCoverage, 0, 1);
var dominant = durationsBySpeaker.OrderByDescending(group => group.Duration).First();
var coverage = total > 0 ? dominant.Duration / total : 0;
if (durationsBySpeaker.Count == 1)
{
return new SingleSpeakerCoverage(
true,
durationsBySpeaker.Count,
dominant.Speaker,
coverage,
TimeSpan.FromSeconds(total));
}
var accepted = total > 0 && coverage >= Math.Clamp(options.MinimumSingleSpeakerCoverage, 0, 1);
return new SingleSpeakerCoverage(
accepted,
durationsBySpeaker.Count,
dominant.Speaker,
coverage,
TimeSpan.FromSeconds(total));
}
private CompositeLayout WriteCompositeWav(
@@ -234,4 +282,11 @@ public sealed class PyannoteSpeakerIdentityMatchValidator : ISpeakerIdentityMatc
}
private sealed record CompositeLayout(TimeSegment? UnknownSegment, IReadOnlyList<TimeSegment> KnownSegments);
private sealed record SingleSpeakerCoverage(
bool Accepted,
int SpeakerCount,
string? DominantSpeaker,
double Coverage,
TimeSpan TotalDuration);
}
@@ -58,53 +58,97 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
.ToHashSet();
var mergedPairs = 0;
var attempts = 0;
logger.LogInformation(
"Speaker identity merge diagnostics started: cutoff {Cutoff}, recent identity ids {RecentIdentityIds}, candidate identity count {CandidateIdentityCount}",
cutoff,
FormatIds(recentIds),
identities.Count);
foreach (var sourceId in recentIds.ToList())
{
var source = identities.SingleOrDefault(identity => identity.Id == sourceId);
if (source is null || source.Snippets.Count == 0)
{
logger.LogInformation(
"Speaker identity merge diagnostics skipped source identity {SourceIdentityId} because it was missing or had no snippets",
sourceId);
continue;
}
var targets = identities
.Where(identity => identity.Id != source.Id && identity.Snippets.Count > 0)
.ToList();
logger.LogInformation(
"Speaker identity merge diagnostics evaluating source identity {SourceIdentityId} ({SourceName}) with {SnippetCount} snippet(s) against target ids {TargetIdentityIds}",
source.Id,
source.GetDisplayName(),
source.Snippets.Count,
FormatIds(targets.Select(target => target.Id)));
foreach (var batch in targets.Chunk(Math.Max(1, options.MatchBatchSize)))
{
var firstSnippet = SelectSnippet(source, excludedSnippet: null);
if (firstSnippet is null)
{
logger.LogInformation(
"Speaker identity merge diagnostics stopped evaluating source identity {SourceIdentityId} because no first snippet was available",
source.Id);
break;
}
attempts++;
logger.LogInformation(
"Speaker identity merge diagnostics round 1 attempt {Attempt} for source identity {SourceIdentityId}: target ids {TargetIdentityIds}, source snippet {SnippetId}",
attempts,
source.Id,
FormatIds(batch.Select(target => target.Id)),
firstSnippet.Id);
var firstMatch = await matcher.MatchAsync(
CreateRequest(source, firstSnippet, batch),
cancellationToken);
if (firstMatch is null || firstMatch.IdentityId == source.Id)
{
logger.LogInformation(
"Speaker identity merge diagnostics round 1 found no usable target for source identity {SourceIdentityId}",
source.Id);
continue;
}
var target = batch.SingleOrDefault(identity => identity.Id == firstMatch.IdentityId);
if (target is null)
{
logger.LogInformation(
"Speaker identity merge diagnostics round 1 matched identity {IdentityId}, but it was not in the current target batch",
firstMatch.IdentityId);
continue;
}
var secondSnippet = SelectSnippet(source, excludedSnippet: firstSnippet);
if (secondSnippet is null)
{
logger.LogInformation(
"Speaker identity merge diagnostics could not validate source identity {SourceIdentityId} against target identity {TargetIdentityId} because no second source snippet was available",
source.Id,
target.Id);
continue;
}
attempts++;
logger.LogInformation(
"Speaker identity merge diagnostics round 2 attempt {Attempt} for source identity {SourceIdentityId}: target identity {TargetIdentityId}, source snippet {SnippetId}",
attempts,
source.Id,
target.Id,
secondSnippet.Id);
var secondMatch = await matcher.MatchAsync(
CreateRequest(source, secondSnippet, batch),
cancellationToken);
if (secondMatch?.IdentityId != target.Id)
{
logger.LogInformation(
"Speaker identity merge diagnostics rejected merge source identity {SourceIdentityId} into target identity {TargetIdentityId}: second match was {SecondMatchIdentityId}",
source.Id,
target.Id,
secondMatch?.IdentityId);
continue;
}
@@ -114,6 +158,12 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
target,
source,
options.MaxSnippetsPerSpeaker);
logger.LogInformation(
"Speaker identity merge diagnostics merging source identity {SourceIdentityId} ({SourceName}) into target identity {TargetIdentityId} ({TargetName})",
source.Id,
sourceName,
target.Id,
targetName);
await SpeakerIdentityTranscriptAudit.AppendMergedAsync(
target.References,
targetName,
@@ -164,4 +214,13 @@ public sealed class SpeakerIdentityMergeService : ISpeakerIdentityMergeService
.FirstOrDefault(snippet => excludedSnippet is null || snippet.Id != excludedSnippet.Id);
}
private static string FormatIds(IEnumerable<int> ids)
{
var formatted = ids
.Distinct()
.Order()
.Select(id => id.ToString(System.Globalization.CultureInfo.InvariantCulture))
.ToList();
return formatted.Count == 0 ? "<none>" : string.Join(", ", formatted);
}
}
@@ -67,18 +67,32 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
var sourceLabel = sourceSpeaker.Trim();
var targetName = targetSpeaker.Trim();
logger.LogInformation(
"Applying speaker override from {SourceSpeaker} to {TargetSpeaker} for meeting {MeetingNotePath}",
sourceLabel,
targetName,
request.MeetingNote.Path);
await using var context = await dbContextFactory.CreateDbContextAsync(cancellationToken);
await SpeakerIdentitySchema.EnsureCreatedOrUpdatedAsync(context, cancellationToken);
var now = DateTimeOffset.UtcNow;
var meetingReference = CreateReference(request.MeetingNote, now);
var snippet = await ResolveOverrideSnippetAsync(request, sourceLabel, cancellationToken);
var snippetResolution = await ResolveOverrideSnippetAsync(request, sourceLabel, cancellationToken);
var snippet = snippetResolution.WavBytes;
var target = await FindIdentityByAcceptedNameAsync(context, targetName, cancellationToken);
var sourceCandidate = await FindCurrentRunCandidateAsync(
context,
meetingReference,
targetName,
cancellationToken);
logger.LogInformation(
"Speaker override from {SourceSpeaker} to {TargetSpeaker} resolved sample source {SampleSource}, sample bytes {SampleBytes}, target identity {TargetIdentityId}, source candidate identity {SourceCandidateIdentityId}",
sourceLabel,
targetName,
snippetResolution.Source,
snippet.Length,
target?.Id,
sourceCandidate?.Id);
if (target is null)
{
@@ -99,10 +113,19 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
if (target.Id == 0)
{
context.SpeakerIdentities.Add(target);
logger.LogInformation(
"Creating speaker identity from override for {TargetSpeaker} using source {SourceSpeaker}",
targetName,
sourceLabel);
}
}
else if (sourceCandidate is not null && sourceCandidate.Id != target.Id)
{
logger.LogInformation(
"Merging override source candidate identity {SourceCandidateIdentityId} into target identity {TargetIdentityId} for {TargetSpeaker}",
sourceCandidate.Id,
target.Id,
targetName);
MergeOverrideCandidate(target, sourceCandidate);
context.SpeakerIdentities.Remove(sourceCandidate);
}
@@ -111,8 +134,16 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
target.UpdatedAt = now;
ResetCandidates(target, [targetName]);
AddMeetingReference(target, meetingReference);
AddSnippetIfNeeded(target, snippet);
var snippetAdded = AddSnippetIfNeeded(target, snippet);
await context.SaveChangesAsync(cancellationToken);
logger.LogInformation(
"Applied speaker override from {SourceSpeaker} to {TargetSpeaker}: identity {IdentityId}, snippet added {SnippetAdded}, snippet count {SnippetCount}, reference count {ReferenceCount}",
sourceLabel,
targetName,
target.Id,
snippetAdded,
target.Snippets.Count,
target.References.Count);
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
target.References,
sourceLabel,
@@ -134,9 +165,20 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
var target = await FindIdentityByAcceptedNameAsync(context, identity.Trim(), cancellationToken);
if (target is null)
{
logger.LogInformation(
"Speaker identity deletion skipped because {IdentityName} was not found",
identity.Trim());
return;
}
logger.LogInformation(
"Deleting speaker identity {IdentityId} ({IdentityName}) with {SnippetCount} snippet(s), {CandidateCount} candidate(s), {AliasCount} alias(es), {ReferenceCount} reference(s)",
target.Id,
target.GetDisplayName() ?? identity.Trim(),
target.Snippets.Count,
target.CandidateNames.Count,
target.Aliases.Count,
target.References.Count);
context.SpeakerIdentities.Remove(target);
await context.SaveChangesAsync(cancellationToken);
}
@@ -180,6 +222,15 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
matchedAcceptedNames.Add(name);
}
logger.LogInformation(
"Speaker identity processing started in {Mode} mode for meeting {MeetingNotePath}: {SegmentCount} segment(s), {SampleCount} supplied sample(s), {KnownMappingCount} known mapping(s), attendees {Attendees}",
mode,
request.MeetingNote.Path,
request.Segments.Count,
request.Samples?.Count ?? 0,
knownSpeakerMappings.Count,
FormatNames(attendees));
var unmatchedSpeakers = new List<(string Speaker, byte[] Snippet)>();
var samplesBySpeaker = request.Samples?
.Where(sample => !string.IsNullOrWhiteSpace(sample.Speaker) && sample.WavBytes.Length > 0)
@@ -189,6 +240,12 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
group => group.OrderByDescending(sample => sample.Score).Select(sample => sample.WavBytes).First(),
StringComparer.OrdinalIgnoreCase)
?? new Dictionary<string, byte[]>(StringComparer.OrdinalIgnoreCase);
if (samplesBySpeaker.Count > 0)
{
logger.LogInformation(
"Speaker identity processing has supplied samples for {SampleSpeakers}",
string.Join(", ", samplesBySpeaker.Select(pair => $"{pair.Key}:{pair.Value.Length} bytes")));
}
foreach (var group in request.Segments
.Where(segment => !string.IsNullOrWhiteSpace(segment.Speaker))
@@ -198,26 +255,55 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
var speaker = group.Key;
if (knownDiarizedSpeakers.Contains(speaker) || alreadyIdentifiedNames.Contains(speaker))
{
logger.LogInformation(
"Speaker identity processing skipped {Speaker} because it is already known or identified",
speaker);
continue;
}
if (speakerMappings.ContainsKey(speaker))
{
logger.LogInformation(
"Speaker identity processing skipped {Speaker} because a mapping already exists to {MappedSpeaker}",
speaker,
speakerMappings[speaker]);
continue;
}
var snippetSource = "supplied";
var snippet = samplesBySpeaker.TryGetValue(speaker, out var suppliedSnippet)
? suppliedSnippet
: await snippetExtractor.ExtractSnippetAsync(
request.AudioPath,
SpeakerSampleSpanSelector.SelectBestContinuousSpan(
request.Segments,
speaker,
options.MaximumSampleSegmentGap,
options.MinimumSampleSpeechDuration),
cancellationToken);
: [];
if (snippet.Length == 0)
{
snippetSource = "extracted";
var span = SpeakerSampleSpanSelector.SelectBestContinuousSpan(
request.Segments,
speaker,
options.MaximumSampleSegmentGap,
options.MinimumSampleSpeechDuration);
logger.LogInformation(
"Speaker identity processing extracting sample for {Speaker}: selected {SegmentCount} segment(s), span {SpanDuration}, minimum {MinimumDuration}",
speaker,
span.Count,
SpeakerSampleSpanSelector.SpanDuration(span),
options.MinimumSampleSpeechDuration);
snippet = await snippetExtractor.ExtractSnippetAsync(
request.AudioPath,
span,
cancellationToken);
}
logger.LogInformation(
"Speaker identity processing resolved {SampleSource} sample for {Speaker}: {SampleBytes} byte(s)",
snippetSource,
speaker,
snippet.Length);
if (snippet.Length == 0)
{
logger.LogInformation(
"Speaker identity processing cannot match or learn {Speaker} because no usable sample was available",
speaker);
unmatchedSpeakers.Add((speaker, snippet));
continue;
}
@@ -231,6 +317,9 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
cancellationToken);
if (match is null)
{
logger.LogInformation(
"Speaker identity processing found no existing identity match for {Speaker}",
speaker);
unmatchedSpeakers.Add((speaker, snippet));
continue;
}
@@ -238,6 +327,10 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
var identity = await LoadIdentityAsync(context, match.IdentityId, cancellationToken);
if (identity is null)
{
logger.LogWarning(
"Speaker identity processing matched {Speaker} to identity {IdentityId}, but the identity could not be loaded",
speaker,
match.IdentityId);
unmatchedSpeakers.Add((speaker, snippet));
continue;
}
@@ -247,6 +340,15 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
var previousCanonicalName = identity.CanonicalName;
AddMeetingReference(identity, meetingReference);
UpdateMatchedIdentity(identity, attendees, snippet);
logger.LogInformation(
"Speaker identity processing updated matched identity {IdentityId} for {Speaker}: previous canonical {PreviousCanonicalName}, canonical {CanonicalName}, candidates {CandidateNames}, snippets {SnippetCount}, references {ReferenceCount}",
identity.Id,
speaker,
previousCanonicalName,
identity.CanonicalName,
FormatNames(identity.CandidateNames.Select(candidate => candidate.Name)),
identity.Snippets.Count,
identity.References.Count);
if (string.IsNullOrWhiteSpace(previousCanonicalName) &&
!string.IsNullOrWhiteSpace(identity.CanonicalName))
{
@@ -279,18 +381,34 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
}
}
IReadOnlyList<SpeakerIdentity> learnedIdentities = [];
if (mode == SpeakerIdentityProcessingMode.Final)
{
await LearnUnmatchedSpeakersAsync(
learnedIdentities = await LearnUnmatchedSpeakersAsync(
context,
attendees,
matchedAcceptedNames,
unmatchedSpeakers,
meetingReference,
cancellationToken);
logger.LogInformation(
"Speaker identity processing queued {LearnedIdentityCount} new unmatched speaker identity candidate(s)",
learnedIdentities.Count);
}
await context.SaveChangesAsync(cancellationToken);
if (learnedIdentities.Count > 0)
{
logger.LogInformation(
"Speaker identity processing saved learned identity candidate ids {IdentityIds}",
FormatIds(learnedIdentities.Select(identity => identity.Id)));
}
logger.LogInformation(
"Speaker identity processing completed in {Mode} mode for meeting {MeetingNotePath}: {MappingCount} mapping(s), {AttendeeMatchCount} attendee match(es)",
mode,
request.MeetingNote.Path,
speakerMappings.Count,
attendeeMatches.Count);
var relabeledSegments = request.Segments
.Select(segment => speakerMappings.TryGetValue(segment.Speaker, out var speakerName)
@@ -333,8 +451,16 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.Select(candidate => candidate.Identity)
.ToList();
logger.LogInformation(
"Speaker identity match candidate selection for {Speaker}: {CandidateCount} candidate(s) after active/attendee/name filters, ids {IdentityIds}",
speaker,
identities.Count,
FormatIds(identities.Select(identity => identity.Id)));
var round = 0;
foreach (var batch in identities.Chunk(Math.Max(1, options.MatchBatchSize)))
{
round++;
var request = new SpeakerIdentityMatchRequest(
speaker,
snippet,
@@ -344,17 +470,31 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
identity.ReferenceCount,
identity.Snippets.Select(storedSnippet => storedSnippet.WavBytes).ToList()))
.ToList());
logger.LogInformation(
"Speaker identity match round {Round} for {Speaker}: candidate identity ids {IdentityIds}",
round,
speaker,
FormatIds(request.Candidates.Select(candidate => candidate.IdentityId)));
var match = await matcher.MatchAsync(request, cancellationToken);
if (match is not null)
{
logger.LogInformation(
"Speaker identity match round {Round} for {Speaker} matched identity {IdentityId}",
round,
speaker,
match.IdentityId);
return match;
}
}
logger.LogInformation(
"Speaker identity match candidate selection for {Speaker} completed with no match after {RoundCount} round(s)",
speaker,
round);
return null;
}
private async Task<byte[]> ResolveOverrideSnippetAsync(
private async Task<SpeakerSnippetResolution> ResolveOverrideSnippetAsync(
SpeakerIdentificationRequest request,
string sourceSpeaker,
CancellationToken cancellationToken)
@@ -365,15 +505,29 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.FirstOrDefault();
if (sample is not null)
{
return sample.WavBytes;
return new SpeakerSnippetResolution(
sample.WavBytes,
"supplied-sample",
SegmentCount: 1,
sample.Score,
sample.Segment.End - sample.Segment.Start);
}
var segments = request.Segments
.Where(segment => string.Equals(segment.Speaker, sourceSpeaker, StringComparison.OrdinalIgnoreCase))
.ToList();
return segments.Count == 0
? []
: await snippetExtractor.ExtractSnippetAsync(request.AudioPath, segments, cancellationToken);
if (segments.Count == 0)
{
return new SpeakerSnippetResolution([], "missing-segments", SegmentCount: 0, Score: null, Duration: TimeSpan.Zero);
}
var snippet = await snippetExtractor.ExtractSnippetAsync(request.AudioPath, segments, cancellationToken);
return new SpeakerSnippetResolution(
snippet,
"extracted-from-recording",
segments.Count,
Score: null,
segments.Max(segment => segment.End) - segments.Min(segment => segment.Start));
}
private static async Task<SpeakerIdentity?> FindIdentityByAcceptedNameAsync(
@@ -421,6 +575,10 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
SpeakerIdentity target,
SpeakerIdentity source)
{
logger.LogInformation(
"Speaker override candidate merge started: source identity {SourceIdentityId} into target identity {TargetIdentityId}",
source.Id,
target.Id);
AddAlias(target, source.CanonicalName);
foreach (var alias in source.Aliases)
{
@@ -441,6 +599,12 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
{
AddSnippetIfNeeded(target, snippet.WavBytes);
}
logger.LogInformation(
"Speaker override candidate merge completed: target identity {TargetIdentityId} now has {AliasCount} alias(es), {SnippetCount} snippet(s), {ReferenceCount} reference(s)",
target.Id,
target.Aliases.Count,
target.Snippets.Count,
target.References.Count);
}
private static void AddAlias(SpeakerIdentity identity, string? alias)
@@ -526,22 +690,36 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
if (intersection.Count == 0)
{
logger.LogInformation(
"Speaker identity candidate elimination for identity {IdentityId} had empty intersection; resetting candidates to attendees {Attendees} and replacing oldest snippet",
identity.Id,
FormatNames(attendees));
ResetCandidates(identity, attendees);
ReplaceOldestSnippet(identity, snippet);
return;
}
logger.LogInformation(
"Speaker identity candidate elimination for identity {IdentityId}: candidates {CurrentCandidates}, attendees {Attendees}, intersection {Intersection}",
identity.Id,
FormatNames(currentCandidates),
FormatNames(attendees),
FormatNames(intersection));
ResetCandidates(identity, intersection);
if (intersection.Count == 1)
{
identity.CanonicalName = intersection[0];
logger.LogInformation(
"Speaker identity candidate elimination promoted identity {IdentityId} to canonical name {CanonicalName}",
identity.Id,
identity.CanonicalName);
}
}
AddSnippetIfNeeded(identity, snippet);
}
private async Task LearnUnmatchedSpeakersAsync(
private async Task<IReadOnlyList<SpeakerIdentity>> LearnUnmatchedSpeakersAsync(
SpeakerIdentityDbContext context,
IReadOnlyList<string> attendees,
IEnumerable<string> matchedCanonicalNames,
@@ -555,16 +733,26 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
.ToList();
if (remainingCandidates.Count == 0)
{
return;
logger.LogInformation(
"Speaker identity candidate learning skipped because no attendee candidates remain. Matched names {MatchedNames}",
FormatNames(matchedCanonicalNames));
return [];
}
logger.LogInformation(
"Speaker identity candidate learning started for {UnmatchedSpeakerCount} unmatched speaker(s) with remaining candidate names {CandidateNames}",
unmatchedSpeakers.Count,
FormatNames(remainingCandidates));
var learned = new List<SpeakerIdentity>();
foreach (var (speaker, snippet) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length > 0))
{
if (!await matchValidator.ValidateSampleAsync(snippet, cancellationToken))
{
logger.LogInformation(
"Skipping speaker identity candidate for {Speaker} because secondary validation rejected the sample",
speaker);
"Skipping speaker identity candidate for {Speaker} because secondary validation rejected the sample: candidate names {CandidateNames}, sample bytes {SampleBytes}",
speaker,
FormatNames(remainingCandidates),
snippet.Length);
continue;
}
@@ -595,6 +783,14 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
]
};
context.SpeakerIdentities.Add(identity);
learned.Add(identity);
logger.LogInformation(
"Created speaker identity candidate for {Speaker}: canonical {CanonicalName}, candidate names {CandidateNames}, sample bytes {SampleBytes}, meeting reference {MeetingNotePath}",
speaker,
canonicalName,
FormatNames(remainingCandidates),
snippet.Length,
meetingReference.MeetingNotePath);
if (!string.IsNullOrWhiteSpace(canonicalName))
{
await SpeakerIdentityTranscriptAudit.AppendIdentifiedAsync(
@@ -605,14 +801,22 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
}
}
await Task.CompletedTask;
foreach (var (speaker, _) in unmatchedSpeakers.Where(speaker => speaker.Snippet.Length == 0))
{
logger.LogInformation(
"Skipping speaker identity candidate for {Speaker} because no sample was available: candidate names {CandidateNames}",
speaker,
FormatNames(remainingCandidates));
}
return learned;
}
private void AddSnippetIfNeeded(SpeakerIdentity identity, byte[] snippet)
private bool AddSnippetIfNeeded(SpeakerIdentity identity, byte[] snippet)
{
if (snippet.Length == 0 || identity.Snippets.Count >= options.MaxSnippetsPerSpeaker)
{
return;
return false;
}
identity.Snippets.Add(new SpeakerSnippet
@@ -620,6 +824,7 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
WavBytes = snippet,
CreatedAt = DateTimeOffset.UtcNow
});
return true;
}
private static void ReplaceOldestSnippet(SpeakerIdentity identity, byte[] snippet)
@@ -700,6 +905,34 @@ public sealed class SpeakerIdentityService : ISpeakerIdentificationService
SpeakerIdentityReferences.AddIfMissing(identity, reference, DateTimeOffset.UtcNow);
}
private static string FormatNames(IEnumerable<string?> names)
{
var formatted = names
.Where(name => !string.IsNullOrWhiteSpace(name))
.Select(name => name!.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order(StringComparer.OrdinalIgnoreCase)
.ToList();
return formatted.Count == 0 ? "<none>" : string.Join(", ", formatted);
}
private static string FormatIds(IEnumerable<int> ids)
{
var formatted = ids
.Distinct()
.Order()
.Select(id => id.ToString(System.Globalization.CultureInfo.InvariantCulture))
.ToList();
return formatted.Count == 0 ? "<none>" : string.Join(", ", formatted);
}
private sealed record SpeakerSnippetResolution(
byte[] WavBytes,
string Source,
int SegmentCount,
double? Score,
TimeSpan Duration);
private enum SpeakerIdentityProcessingMode
{
LiveReadOnly,
@@ -0,0 +1,74 @@
using System.Drawing;
using System.Drawing.Drawing2D;
using System.Drawing.Imaging;
using System.Drawing.Text;
using System.Runtime.InteropServices;
using System.Runtime.Versioning;
using MeetingAssistant.Recording;
namespace MeetingAssistant.Taskbar;
[SupportedOSPlatform("windows")]
internal static class TaskbarIconRenderer
{
private const int IconSize = 16;
private const float GlyphPixelGridOffsetX = -0.5f;
public static Icon CreateIcon(RecordingProcessState state)
{
using var bitmap = RenderBitmap(state);
var handle = bitmap.GetHicon();
try
{
return (Icon)Icon.FromHandle(handle).Clone();
}
finally
{
DestroyIcon(handle);
}
}
internal static Bitmap RenderBitmap(RecordingProcessState state)
{
var (color, text) = state switch
{
RecordingProcessState.Recording => (Color.FromArgb(220, 38, 38), "R"),
RecordingProcessState.Summarizing => (Color.FromArgb(37, 99, 235), "S"),
_ => (Color.FromArgb(75, 85, 99), "I")
};
var bitmap = new Bitmap(IconSize, IconSize, PixelFormat.Format32bppArgb);
using var graphics = Graphics.FromImage(bitmap);
graphics.Clear(Color.Transparent);
graphics.SmoothingMode = SmoothingMode.AntiAlias;
graphics.TextRenderingHint = TextRenderingHint.AntiAliasGridFit;
using var brush = new SolidBrush(color);
graphics.FillEllipse(brush, 1, 1, 14, 14);
DrawCenteredGlyph(graphics, text);
return bitmap;
}
private static void DrawCenteredGlyph(Graphics graphics, string text)
{
using var path = new GraphicsPath();
using var format = StringFormat.GenericTypographic;
path.AddString(
text,
FontFamily.GenericSansSerif,
(int)FontStyle.Bold,
9,
Point.Empty,
format);
var bounds = path.GetBounds();
using var transform = new Matrix();
transform.Translate(
(IconSize - bounds.Width) / 2 - bounds.Left + GlyphPixelGridOffsetX,
(IconSize - bounds.Height) / 2 - bounds.Top);
path.Transform(transform);
using var textBrush = new SolidBrush(Color.White);
graphics.FillPath(textBrush, path);
}
[DllImport("user32.dll", SetLastError = true)]
private static extern bool DestroyIcon(IntPtr hIcon);
}
@@ -1,6 +1,5 @@
#if WINDOWS
using System.Drawing;
using System.Runtime.InteropServices;
using H.NotifyIcon.Core;
using MeetingAssistant.LaunchProfiles;
using MeetingAssistant.Recording;
@@ -47,7 +46,7 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
var menu = BuildMenu();
currentMenuSignature = BuildMenuSignature(menu);
currentIcon = CreateIcon(menu.State);
currentIcon = TaskbarIconRenderer.CreateIcon(menu.State);
currentState = menu.State;
trayIcon = new TrayIconWithContextMenu(TrayIconId)
{
@@ -126,7 +125,7 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
if (currentState != menu.State)
{
var previousIcon = currentIcon;
var nextIcon = CreateIcon(menu.State);
var nextIcon = TaskbarIconRenderer.CreateIcon(menu.State);
if (!trayIcon.UpdateIcon(nextIcon.Handle))
{
logger.LogWarning("Taskbar icon update returned false for state {State}", menu.State);
@@ -248,45 +247,5 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
menu.Items.Select(item => $"{item.Action}:{item.ProfileName}:{item.Text}"));
}
private static Icon CreateIcon(RecordingProcessState state)
{
var (color, text) = state switch
{
RecordingProcessState.Recording => (Color.FromArgb(220, 38, 38), "R"),
RecordingProcessState.Summarizing => (Color.FromArgb(37, 99, 235), "S"),
_ => (Color.FromArgb(75, 85, 99), "I")
};
using var bitmap = new Bitmap(16, 16);
using (var graphics = Graphics.FromImage(bitmap))
{
graphics.Clear(Color.Transparent);
graphics.SmoothingMode = System.Drawing.Drawing2D.SmoothingMode.AntiAlias;
using var brush = new SolidBrush(color);
graphics.FillEllipse(brush, 1, 1, 14, 14);
using var font = new Font(FontFamily.GenericSansSerif, 8, FontStyle.Bold, GraphicsUnit.Pixel);
using var textBrush = new SolidBrush(Color.White);
var size = graphics.MeasureString(text, font);
graphics.DrawString(
text,
font,
textBrush,
(16 - size.Width) / 2,
(16 - size.Height) / 2);
}
var handle = bitmap.GetHicon();
try
{
return (Icon)Icon.FromHandle(handle).Clone();
}
finally
{
DestroyIcon(handle);
}
}
[DllImport("user32.dll", SetLastError = true)]
private static extern bool DestroyIcon(IntPtr hIcon);
}
#endif
+12 -1
View File
@@ -25,7 +25,18 @@
"BackgroundMetadataLookupTimeout": "00:01:00",
"MaxMetadataAttendeeImportCount": 30,
"OpenMeetingNoteDelay": "00:00:01",
"TemporaryRecordingsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Recordings"
"TemporaryRecordingsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Recordings",
"InactivitySafeguard": {
"Enabled": true,
"FirstPromptAfter": "00:02:00",
"ReminderPromptAfter": [
"00:05:00",
"00:10:00"
],
"AutoStopAfter": "00:30:00",
"InferredEndPadding": "00:01:00",
"CheckInterval": "00:00:15"
}
},
"FunAsr": {
"Endpoint": "ws://127.0.0.1:10095",
+1 -1
View File
@@ -51,7 +51,7 @@ For the local Windows workstation, use the snippets restart helper instead of ru
powershell -ExecutionPolicy Bypass -File C:\Manuel\snippets\restart-meeting-assistant.ps1
```
The restart helper publishes `MeetingAssistant\MeetingAssistant.csproj` as `net10.0-windows` into a fresh temp folder under `C:\Manuel\meeting-assistant\tmp\meeting-assistant-runtime`. Only after that publish succeeds does it stop any active Meeting Assistant instance on port `5090` and start the newly published executable detached from the script.
The restart helper publishes `MeetingAssistant\MeetingAssistant.csproj` as `net10.0-windows10.0.19041.0` into a fresh temp folder under `C:\Manuel\meeting-assistant\tmp\meeting-assistant-runtime`. Only after that publish succeeds does it stop any active Meeting Assistant instance on port `5090` and start the newly published executable detached from the script.
The detached starter writes logs and the last process id to:
+21 -1
View File
@@ -31,7 +31,15 @@ This example is abbreviated so the most common shape is readable. The checked-in
"StopProcessingTimeout": "00:10:00",
"MinimumCompletedMeetingDuration": "00:01:00",
"MaxMetadataAttendeeImportCount": 30,
"TemporaryRecordingsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Recordings"
"TemporaryRecordingsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Recordings",
"InactivitySafeguard": {
"Enabled": true,
"FirstPromptAfter": "00:02:00",
"ReminderPromptAfter": [ "00:05:00", "00:10:00" ],
"AutoStopAfter": "00:30:00",
"InferredEndPadding": "00:01:00",
"CheckInterval": "00:00:15"
}
},
"Automation": {
"RulesPath": "meeting-rules.local.yaml"
@@ -111,6 +119,7 @@ The default profile is always named `default`. Non-default profile hotkeys are r
| `MaxMetadataAttendeeImportCount` | Maximum Outlook attendee count that will be copied into meeting frontmatter. |
| `OpenMeetingNoteDelay` | Delay before opening the newly created Obsidian note, giving the file system and app time to settle. |
| `TemporaryRecordingsFolder` | Folder for temporary mixed WAV files while a run is active. |
| `InactivitySafeguard` | Controls prompts and automatic normal stop for active meetings with no new transcript text. |
During recording, Meeting Assistant captures microphone and system loopback separately, buffers both streams to align samples, cleans the microphone stream through a local adaptive echo canceller using loopback as the far-end reference, then mixes the cleaned microphone and system streams into the normal 16 kHz mono PCM chunks. If one source is quiet beyond the alignment timeout, the available source is mixed with synthetic silence so microphone-only speech and system-only playback keep flowing to transcription.
@@ -118,6 +127,17 @@ During recording, Meeting Assistant captures microphone and system loopback sepa
`Recording:MaxMetadataAttendeeImportCount` limits how many attendees Outlook metadata enrichment imports into meeting-note frontmatter. The default is `30`; when an appointment has more attendees than that, Meeting Assistant still imports title, agenda, and scheduled end time, but leaves attendees empty because large invites are usually presentation-style meetings.
`Recording:InactivitySafeguard` watches active recordings for long periods without transcript text. The timer starts at meeting start and resets whenever a live transcript segment with text arrives. By default the app asks whether to stop after 2, 5, and 10 minutes of inactivity through native Windows app notifications with action buttons, and automatically stops normally after 30 minutes. Ignoring a notification does not block later checks or auto-stop. Safeguard stops are not aborts: transcription drain, speaker processing, screenshots, and summary generation continue through the normal stop flow. When the safeguard stops a run, the meeting end time is inferred as the last transcript segment timestamp plus `InferredEndPadding`; if no transcript text arrived, it uses meeting start plus the same padding.
| Setting | Purpose |
| --- | --- |
| `Enabled` | Enables the inactivity safeguard for active recordings. |
| `FirstPromptAfter` | First transcript-inactivity duration before showing the stop prompt. |
| `ReminderPromptAfter` | Additional transcript-inactivity durations before showing another stop prompt. |
| `AutoStopAfter` | Transcript-inactivity duration after which Meeting Assistant stops the recording normally without prompting again. |
| `InferredEndPadding` | Padding added to the last transcript timestamp, or meeting start when no transcript arrived, for safeguard-triggered end times. |
| `CheckInterval` | Polling interval for checking the active recording inactivity state. |
`Hotkey:Abort` configures a global discard shortcut. The default is `Ctrl+Alt+Z`. Aborting an active recording stops capture/transcription, deletes the meeting note, transcript, assistant context, summary file if present, and linked screenshot attachments for that run, and skips automatic summary generation.
Launch profile toggle hotkeys start a meeting when idle. When a different launch profile toggle is pressed during an active meeting, Meeting Assistant keeps the same meeting note, transcript, assistant context, and metadata; drains the current speech recognition pipeline without summarizing; appends a transcript marker; clears run-local speaker mappings; and continues transcription through the target profile.
@@ -0,0 +1,15 @@
# Add Recording Inactivity Safeguard
## Why
Meeting recordings can be left running after a conversation has effectively ended. The app should protect against forgotten active recordings without treating them as aborted meetings.
## What
- Track transcript inactivity during active recordings.
- Prompt the user to stop at configurable inactivity thresholds.
- Automatically stop the recording after a longer configurable inactivity threshold.
- When the safeguard stops a recording, process it normally and infer the meeting end time from the last transcript timestamp plus configurable padding.
## Impact
- Adds recording configuration for inactivity prompts and automatic stop.
- Adds a Windows prompt surface for the safeguard.
- Updates recording coordinator behavior and tests.
@@ -0,0 +1,52 @@
## ADDED Requirements
### Requirement: Recording inactivity safeguard stops forgotten meetings
Meeting Assistant SHALL track transcript inactivity during an active recording from the later of meeting start or the most recent live transcript segment that contains text.
Meeting Assistant SHALL show a stop prompt when transcript inactivity reaches configured prompt thresholds. The default thresholds SHALL be 2 minutes, 5 minutes, and 10 minutes.
On Windows, the stop prompt SHALL use a native Windows app notification with affirmative and negative action buttons.
The stop prompt SHALL ask whether to stop the meeting and SHALL provide affirmative and negative actions.
Showing or ignoring the stop prompt SHALL NOT block later inactivity checks, reminder prompts, or automatic stop.
If the user accepts the stop prompt, Meeting Assistant SHALL stop the recording normally, allowing transcription, speaker recognition, and summary generation to continue as for a normal stop.
Meeting Assistant SHALL automatically stop the recording normally when transcript inactivity reaches the configured auto-stop threshold, defaulting to 30 minutes.
When the inactivity safeguard stops a recording, Meeting Assistant SHALL infer the meeting end time from the most recent transcript segment timestamp plus configured padding, defaulting to 1 minute. If no transcript segment has arrived, Meeting Assistant SHALL infer the end time from the meeting start time plus the same padding.
#### Scenario: Inactive recording prompts the user
- **GIVEN** a recording is active
- **AND** no transcript text has arrived for the first configured inactivity prompt threshold
- **WHEN** the inactivity safeguard checks the active recording
- **THEN** Meeting Assistant prompts the user whether to stop the meeting with a native Windows app notification when running on Windows
- **AND** does not abort or discard meeting artifacts
#### Scenario: Ignored inactivity prompt does not block auto-stop
- **GIVEN** a recording is active
- **AND** the inactivity safeguard prompt was shown
- **WHEN** the user ignores the prompt until the automatic stop threshold is reached
- **THEN** Meeting Assistant stops the recording normally without waiting for a prompt response
#### Scenario: User accepts inactivity stop prompt
- **GIVEN** a recording is active
- **AND** the inactivity safeguard prompt is shown
- **WHEN** the user chooses to stop the meeting
- **THEN** Meeting Assistant stops the recording normally
- **AND** continues normal transcription and summary processing
- **AND** writes the inferred meeting end time to meeting artifacts
#### Scenario: Inactive recording automatically stops
- **GIVEN** a recording is active
- **AND** no transcript text has arrived through the configured automatic stop threshold
- **WHEN** the inactivity safeguard checks the active recording
- **THEN** Meeting Assistant stops the recording normally without aborting artifacts
- **AND** writes the inferred meeting end time to meeting artifacts
#### Scenario: New transcript text resets inactivity prompts
- **GIVEN** a recording is active
- **AND** the first inactivity prompt was shown
- **WHEN** a new transcript segment with text arrives
- **THEN** Meeting Assistant resets the inactivity prompt schedule from that transcript arrival
@@ -0,0 +1,9 @@
## Implementation
- [x] Add inactivity safeguard configuration.
- [x] Add prompt service abstraction with Windows and no-op implementations.
- [x] Use native Windows app notifications with action buttons for Windows prompts.
- [x] Track last text transcript arrival and last transcript timestamp on active recording runs.
- [x] Prompt at configured inactivity thresholds and auto-stop at the configured threshold.
- [x] Use inferred end time for safeguard-triggered normal stops.
- [x] Add focused behavior tests.
- [x] Validate with `dotnet test MeetingAssistant.slnx` and `openspec validate add-recording-inactivity-safeguard --strict`.