Archive completed meeting assistant changes
PR and Push Build/Test / build-and-test (push) Successful in 9m19s

This commit is contained in:
2026-06-26 13:15:47 +02:00
parent 7d16b0cad7
commit aecef30627
72 changed files with 2914 additions and 406 deletions
@@ -1,4 +1,5 @@
using MeetingAssistant.Recording;
using MeetingAssistant.MeetingNotes;
using Microsoft.Extensions.Options;
namespace MeetingAssistant.Calendar;
@@ -84,7 +85,8 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
foreach (var meeting in cachedMeetings.OrderBy(meeting => meeting.Start))
{
var promptKey = GetPromptKey(meeting);
if (promptedMeetingKeys.Contains(promptKey) ||
if (meeting.IsCanceled ||
promptedMeetingKeys.Contains(promptKey) ||
!IsInPromptWindow(meeting, now, promptOptions))
{
continue;
@@ -97,7 +99,7 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
meeting.Start);
await promptService.ShowPromptAsync(
new MeetingStartPromptRequest(meeting),
HandlePromptResponseAsync,
(response, token) => HandlePromptResponseAsync(meeting, response, token),
cancellationToken);
}
}
@@ -133,6 +135,7 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
}
private async Task HandlePromptResponseAsync(
CalendarMeeting meeting,
MeetingStartPromptResponse response,
CancellationToken cancellationToken)
{
@@ -146,7 +149,7 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
await recordingController.StopAsync(cancellationToken);
}
await recordingController.StartAsync(cancellationToken);
await recordingController.StartFromPromptAsync(meeting.Metadata, cancellationToken);
}
private void ResetPromptedMeetingsIfDayChanged(DateOnly today)
@@ -183,6 +186,7 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
var syncDelay = nextSyncAt - now;
var nextMeetingStart = cachedMeetings
.Where(meeting => !meeting.IsCanceled)
.Where(meeting => !promptedMeetingKeys.Contains(GetPromptKey(meeting)))
.Where(meeting => meeting.End > now)
.Select(meeting => meeting.Start <= now ? now : meeting.Start)
@@ -228,7 +232,9 @@ public sealed record CalendarMeeting(
string Id,
string Subject,
DateTimeOffset Start,
DateTimeOffset End);
DateTimeOffset End,
MeetingMetadata? Metadata = null,
bool IsCanceled = false);
public interface IMeetingStartPromptService
{
@@ -252,6 +258,10 @@ public interface IMeetingPromptRecordingController
Task<RecordingStatus> StartAsync(CancellationToken cancellationToken);
Task<RecordingStatus> StartFromPromptAsync(
MeetingMetadata? metadata,
CancellationToken cancellationToken);
Task<RecordingStatus> StopAsync(CancellationToken cancellationToken);
}
@@ -271,6 +281,13 @@ public sealed class MeetingPromptRecordingController : IMeetingPromptRecordingCo
return coordinator.StartAsync(cancellationToken);
}
public Task<RecordingStatus> StartFromPromptAsync(
MeetingMetadata? metadata,
CancellationToken cancellationToken)
{
return coordinator.StartFromPromptAsync(metadata, cancellationToken);
}
public Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
{
return coordinator.StopAsync(cancellationToken);
@@ -60,18 +60,21 @@ public sealed class OutlookClassicCalendarMeetingProvider : ICalendarMeetingProv
var end = Convert.ToDateTime(OutlookClassicCom.GetProperty(item, "End"));
if (end <= dayStart ||
start < dayStart ||
!IsTeamsAppointment(item))
OutlookClassicCom.IsCanceledAppointment(item) ||
!OutlookClassicAppointmentMetadata.IsTeamsAppointment(item))
{
continue;
}
var rawSubject = Convert.ToString(OutlookClassicCom.GetProperty(item, "Subject"))?.Trim();
var subject = string.IsNullOrWhiteSpace(rawSubject) ? "Teams meeting" : rawSubject;
var subject = OutlookClassicAppointmentMetadata.ReadSubject(item);
var localEnd = OutlookClassicCom.ToLocalOffset(end);
meetings.Add(new CalendarMeeting(
GetAppointmentId(item, start, end, subject),
subject,
OutlookClassicCom.ToLocalOffset(start),
OutlookClassicCom.ToLocalOffset(end)));
localEnd,
OutlookClassicAppointmentMetadata.CreateMetadata(item, end),
IsCanceled: false));
}
finally
{
@@ -98,13 +101,5 @@ public sealed class OutlookClassicCalendarMeetingProvider : ICalendarMeetingProv
? $"{start:O}|{end:O}|{subject}"
: $"{id}|{start:O}";
}
private static bool IsTeamsAppointment(object appointment)
{
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? "";
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body);
}
}
#endif
+5 -2
View File
@@ -59,6 +59,7 @@ public sealed class ScreenshotOcrOptions
Identify who appears to be talking, who appears to be presenting, and what is being presented when visible.
If people or participant tiles are visible, state whether it is clear from the screenshot exactly who is in the meeting or whether the visible people are only a partial participant result.
Include any attendee names that are clearly visible or strongly deduced from the screenshot in the meeting-assistant metadata. The attendee list may be partial.
If the screenshot contains slides, capture the visible text in clean markdown.
If the screenshot contains a diagram, convert it to Mermaid when possible and preserve labels accurately.
If it contains another kind of shared screen or scene, describe the scene and the relevant visible information.
@@ -67,9 +68,9 @@ public sealed class ScreenshotOcrOptions
Do not return crop coordinates for a person, webcam tile, whole app chrome, or content you cannot isolate confidently.
End your response with exactly one fenced json block containing meeting-assistant metadata, for example:
```json
{ "crop": { "x": 0, "y": 0, "width": 1920, "height": 1080 } }
{ "crop": { "x": 0, "y": 0, "width": 1920, "height": 1080 }, "attendees": ["Ada Lovelace"] }
```
Use `{ "crop": null }` when no confident presentation/shared-screen crop exists.
Use `{ "crop": null, "attendees": [] }` when no confident presentation/shared-screen crop exists and no attendees are visible or deducible.
Do not invent information that is not visible in the screenshot.
""";
@@ -125,6 +126,8 @@ public sealed class RecordingOptions
public int Channels { get; set; } = 1;
public string? MicrophoneDeviceId { get; set; }
public double MicrophoneMixGain { get; set; } = 1;
public double SystemAudioMixGain { get; set; } = 1;
@@ -8,4 +8,13 @@ internal static class MeetingAttendeeNames
var emailStart = trimmed.IndexOf('<', StringComparison.Ordinal);
return emailStart > 0 ? trimmed[..emailStart].Trim() : trimmed;
}
public static List<string> NormalizeDistinct(IEnumerable<string> attendees)
{
return attendees
.Select(attendee => attendee.Trim())
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
}
}
@@ -14,4 +14,24 @@ public static class MeetingNoteActionLinks
var url = $"{baseUrl}/meetings/summary/retry?summaryPath={Uri.EscapeDataString(summaryFileName)}";
return $"[{label}]({url})";
}
public static string CreateScreenshotOcrRetryLink(
string publicBaseUrl,
MeetingSessionArtifacts artifacts,
string screenshotPath,
string screenshotId,
string label = "Retry screenshot OCR")
{
var baseUrl = string.IsNullOrWhiteSpace(publicBaseUrl)
? "http://localhost:5090"
: publicBaseUrl.TrimEnd('/');
var url = $"{baseUrl}/meetings/screenshot-ocr/retry" +
$"?meetingNotePath={Uri.EscapeDataString(artifacts.MeetingNotePath)}" +
$"&transcriptPath={Uri.EscapeDataString(artifacts.TranscriptPath)}" +
$"&assistantContextPath={Uri.EscapeDataString(artifacts.AssistantContextPath)}" +
$"&summaryPath={Uri.EscapeDataString(artifacts.SummaryPath)}" +
$"&screenshotPath={Uri.EscapeDataString(screenshotPath)}" +
$"&screenshotId={Uri.EscapeDataString(screenshotId)}";
return $"[{label}]({url})";
}
}
@@ -0,0 +1,176 @@
#if WINDOWS
using MeetingAssistant.Calendar;
namespace MeetingAssistant.MeetingNotes;
internal static class OutlookClassicAppointmentMetadata
{
public static MeetingMetadata CreateMetadata(object appointment, DateTime end)
{
var title = ReadSubject(appointment);
return new MeetingMetadata(
title,
ReadAttendees(appointment),
ExtractAgenda(Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? ""),
OutlookClassicCom.ToLocalOffset(end));
}
public static string ReadSubject(object appointment)
{
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject"))?.Trim();
return string.IsNullOrWhiteSpace(subject) ? "Teams meeting" : subject;
}
public static bool IsTeamsAppointment(object appointment)
{
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? "";
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body);
}
public static string ExtractAgenda(string body)
{
if (string.IsNullOrWhiteSpace(body))
{
return "";
}
var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n');
var agendaLines = new List<string>();
foreach (var line in lines)
{
if (IsTeamsSeparator(line) || IsTeamsJoinLine(line))
{
break;
}
agendaLines.Add(line);
}
return string.Join(Environment.NewLine, agendaLines).Trim();
}
public static IReadOnlyList<string> ReadAttendees(object appointment)
{
var attendees = new List<string>();
var organizer = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Organizer"));
if (!string.IsNullOrWhiteSpace(organizer))
{
attendees.Add(organizer.Trim());
}
object? recipients = null;
try
{
recipients = OutlookClassicCom.GetProperty(appointment, "Recipients");
var count = Convert.ToInt32(OutlookClassicCom.GetProperty(recipients!, "Count"));
for (var index = 1; index <= count; index++)
{
object? recipient = null;
try
{
recipient = OutlookClassicCom.Invoke(recipients!, "Item", index);
var formatted = FormatRecipient(recipient!);
if (!string.IsNullOrWhiteSpace(formatted))
{
attendees.Add(formatted);
}
}
finally
{
OutlookClassicCom.ReleaseComObject(recipient);
}
}
}
finally
{
OutlookClassicCom.ReleaseComObject(recipients);
}
return NormalizeAttendees(attendees);
}
public static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
{
return attendees
.Select(NormalizeAttendee)
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase)
.Select(group => group
.OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal))
.ThenBy(attendee => attendee.Length)
.First())
.ToList();
}
private static bool IsTeamsSeparator(string line)
{
var trimmed = line.Trim();
return trimmed.Length >= 8 &&
trimmed.All(character => character is '_' or '-' or '*' or ' ');
}
private static bool IsTeamsJoinLine(string line)
{
return TeamsMeetingMarkerDetector.ContainsTeamsMarker(line);
}
private static string NormalizeAttendee(string attendee)
{
var normalized = attendee.Trim();
var mailtoPrefix = "mailto:";
normalized = normalized.Replace(mailtoPrefix, "", StringComparison.OrdinalIgnoreCase);
while (normalized.Contains(" ", StringComparison.Ordinal))
{
normalized = normalized.Replace(" ", " ", StringComparison.Ordinal);
}
var emailStart = normalized.IndexOf('<', StringComparison.Ordinal);
var emailEnd = normalized.LastIndexOf('>');
if (emailStart > 0 && emailEnd > emailStart)
{
var name = normalized[..emailStart].Trim();
var email = normalized[(emailStart + 1)..emailEnd].Trim();
if (name.Equals(email, StringComparison.OrdinalIgnoreCase))
{
return email;
}
return $"{name} <{email}>";
}
return normalized;
}
private static string GetAttendeeDeduplicationKey(string attendee)
{
var emailStart = attendee.IndexOf('<', StringComparison.Ordinal);
var emailEnd = attendee.LastIndexOf('>');
if (emailStart > 0 && emailEnd > emailStart)
{
return attendee[(emailStart + 1)..emailEnd].Trim();
}
return attendee.Trim();
}
private static string FormatRecipient(object recipient)
{
var name = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Name"))?.Trim() ?? "";
var email = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Address"))?.Trim() ?? "";
if (string.IsNullOrWhiteSpace(name))
{
return email;
}
if (string.IsNullOrWhiteSpace(email) ||
email.Contains('/'))
{
return name;
}
return $"{name} <{email}>";
}
}
#endif
@@ -108,6 +108,37 @@ internal static class OutlookClassicCom
}
}
public static bool IsCanceledAppointment(object appointment)
{
var meetingStatus = TryGetProperty(appointment, "MeetingStatus");
if (meetingStatus is not null)
{
try
{
var status = Convert.ToInt32(meetingStatus);
if (status is 5 or 7)
{
return true;
}
}
catch (Exception exception) when (exception is FormatException or InvalidCastException or OverflowException)
{
// Fall back to subject prefixes when Outlook exposes a non-numeric status value.
}
}
var subject = Convert.ToString(TryGetProperty(appointment, "Subject")) ?? "";
return SubjectIndicatesCancellation(subject);
}
internal static bool SubjectIndicatesCancellation(string subject)
{
var normalized = subject.TrimStart();
return normalized.StartsWith("Canceled:", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Cancelled:", StringComparison.OrdinalIgnoreCase) ||
normalized.StartsWith("Abgesagt:", StringComparison.OrdinalIgnoreCase);
}
public static DateTimeOffset ToLocalOffset(DateTime value)
{
return new DateTimeOffset(DateTime.SpecifyKind(value, DateTimeKind.Local));
@@ -121,7 +121,9 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv
}
var end = Convert.ToDateTime(OutlookClassicCom.GetProperty(item, "End"));
if (end < windowStart || !IsTeamsAppointment(item))
if (end < windowStart ||
OutlookClassicCom.IsCanceledAppointment(item) ||
!OutlookClassicAppointmentMetadata.IsTeamsAppointment(item))
{
continue;
}
@@ -150,15 +152,7 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv
return null;
}
var appointment = selected.Appointment;
var title = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
var attendees = ReadAttendees(appointment);
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
return new MeetingMetadata(
title.Trim(),
attendees,
ExtractAgenda(body),
OutlookClassicCom.ToLocalOffset(selected.End));
return OutlookClassicAppointmentMetadata.CreateMetadata(selected.Appointment, selected.End);
}
finally
{
@@ -169,158 +163,6 @@ public sealed class OutlookClassicMeetingMetadataProvider : IMeetingMetadataProv
}
}
internal static string ExtractAgenda(string body)
{
if (string.IsNullOrWhiteSpace(body))
{
return "";
}
var lines = body.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n');
var agendaLines = new List<string>();
foreach (var line in lines)
{
if (IsTeamsSeparator(line) || IsTeamsJoinLine(line))
{
break;
}
agendaLines.Add(line);
}
return string.Join(Environment.NewLine, agendaLines).Trim();
}
private static bool IsTeamsSeparator(string line)
{
var trimmed = line.Trim();
return trimmed.Length >= 8 &&
trimmed.All(character => character is '_' or '-' or '*' or ' ');
}
private static bool IsTeamsJoinLine(string line)
{
return TeamsMeetingMarkerDetector.ContainsTeamsMarker(line);
}
private static bool IsTeamsAppointment(object appointment)
{
var subject = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Subject")) ?? "";
var location = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Location")) ?? "";
var body = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Body")) ?? "";
return TeamsMeetingMarkerDetector.IsTeamsAppointment(subject, location, body);
}
private static IReadOnlyList<string> ReadAttendees(object appointment)
{
var attendees = new List<string>();
var organizer = Convert.ToString(OutlookClassicCom.GetProperty(appointment, "Organizer"));
if (!string.IsNullOrWhiteSpace(organizer))
{
attendees.Add(organizer.Trim());
}
object? recipients = null;
try
{
recipients = OutlookClassicCom.GetProperty(appointment, "Recipients");
var count = Convert.ToInt32(OutlookClassicCom.GetProperty(recipients!, "Count"));
for (var index = 1; index <= count; index++)
{
object? recipient = null;
try
{
recipient = OutlookClassicCom.Invoke(recipients!, "Item", index);
var formatted = FormatRecipient(recipient!);
if (!string.IsNullOrWhiteSpace(formatted))
{
attendees.Add(formatted);
}
}
finally
{
OutlookClassicCom.ReleaseComObject(recipient);
}
}
}
finally
{
OutlookClassicCom.ReleaseComObject(recipients);
}
return NormalizeAttendees(attendees);
}
internal static IReadOnlyList<string> NormalizeAttendees(IEnumerable<string> attendees)
{
return attendees
.Select(NormalizeAttendee)
.Where(attendee => !string.IsNullOrWhiteSpace(attendee))
.GroupBy(GetAttendeeDeduplicationKey, StringComparer.OrdinalIgnoreCase)
.Select(group => group
.OrderByDescending(attendee => attendee.Contains('<', StringComparison.Ordinal))
.ThenBy(attendee => attendee.Length)
.First())
.ToList();
}
private static string NormalizeAttendee(string attendee)
{
var normalized = attendee.Trim();
var mailtoPrefix = "mailto:";
normalized = normalized.Replace(mailtoPrefix, "", StringComparison.OrdinalIgnoreCase);
while (normalized.Contains(" ", StringComparison.Ordinal))
{
normalized = normalized.Replace(" ", " ", StringComparison.Ordinal);
}
var emailStart = normalized.IndexOf('<', StringComparison.Ordinal);
var emailEnd = normalized.LastIndexOf('>');
if (emailStart > 0 && emailEnd > emailStart)
{
var name = normalized[..emailStart].Trim();
var email = normalized[(emailStart + 1)..emailEnd].Trim();
if (name.Equals(email, StringComparison.OrdinalIgnoreCase))
{
return email;
}
return $"{name} <{email}>";
}
return normalized;
}
private static string GetAttendeeDeduplicationKey(string attendee)
{
var emailStart = attendee.IndexOf('<', StringComparison.Ordinal);
var emailEnd = attendee.LastIndexOf('>');
if (emailStart > 0 && emailEnd > emailStart)
{
return attendee[(emailStart + 1)..emailEnd].Trim();
}
return attendee.Trim();
}
private static string FormatRecipient(object recipient)
{
var name = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Name"))?.Trim() ?? "";
var email = Convert.ToString(OutlookClassicCom.GetProperty(recipient, "Address"))?.Trim() ?? "";
if (string.IsNullOrWhiteSpace(name))
{
return email;
}
if (string.IsNullOrWhiteSpace(email) ||
email.Contains('/'))
{
return name;
}
return $"{name} <{email}>";
}
private sealed record OutlookAppointmentCandidate(
object Appointment,
DateTime Start,
+133 -1
View File
@@ -19,6 +19,8 @@ builder.Logging.AddProvider(new MeetingAssistantFileLoggerProvider());
builder.Services.Configure<MeetingAssistantOptions>(builder.Configuration.GetSection("MeetingAssistant"));
builder.Services.AddSingleton<ILaunchProfileOptionsProvider, ConfigurationLaunchProfileOptionsProvider>();
#if WINDOWS
builder.Services.AddSingleton<MicrophoneDeviceSelection>();
builder.Services.AddSingleton<IMicrophoneDeviceProvider, WindowsMicrophoneDeviceProvider>();
builder.Services.AddSingleton<MicrophoneAudioSource>();
builder.Services.AddSingleton<SystemAudioSource>();
builder.Services.AddSingleton<IAcousticEchoCancellerFactory, AdaptiveFilterAcousticEchoCancellerFactory>();
@@ -70,7 +72,10 @@ builder.Services.AddSingleton<IActiveWindowScreenshotCapture, ActiveWindowScreen
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, UnavailableActiveWindowScreenshotCapture>();
#endif
builder.Services.AddSingleton<IScreenshotOcrClient, LiteLlmScreenshotOcrClient>();
builder.Services.AddSingleton<IMeetingScreenshotService, MeetingScreenshotService>();
builder.Services.AddSingleton<MeetingScreenshotService>();
builder.Services.AddSingleton<IMeetingScreenshotService>(sp => sp.GetRequiredService<MeetingScreenshotService>());
builder.Services.AddSingleton<IScreenshotOcrRetryRunner>(sp => sp.GetRequiredService<MeetingScreenshotService>());
builder.Services.AddSingleton<IMeetingNoteImageOcrService>(sp => sp.GetRequiredService<MeetingScreenshotService>());
builder.Services.AddDbContextFactory<SpeakerIdentityDbContext>((services, dbOptions) =>
{
var appOptions = services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value;
@@ -410,6 +415,81 @@ app.MapGet("/profiles/{launchProfile}/meetings/summary/retry", async (
launchProfiles,
cancellationToken));
app.MapPost("/meetings/screenshot-ocr/retry", async (
ScreenshotOcrRetryRequest request,
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
IOptions<MeetingAssistantOptions> options,
CancellationToken cancellationToken) =>
await RetryScreenshotOcrAsync(
null,
request,
screenshotOcrRetryRunner,
options.Value,
null,
cancellationToken));
app.MapPost("/profiles/{launchProfile}/meetings/screenshot-ocr/retry", async (
string launchProfile,
ScreenshotOcrRetryRequest request,
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
IOptions<MeetingAssistantOptions> options,
ILaunchProfileOptionsProvider launchProfiles,
CancellationToken cancellationToken) =>
await RetryScreenshotOcrAsync(
launchProfile,
request,
screenshotOcrRetryRunner,
options.Value,
launchProfiles,
cancellationToken));
app.MapGet("/meetings/screenshot-ocr/retry", async (
string meetingNotePath,
string transcriptPath,
string assistantContextPath,
string summaryPath,
string screenshotPath,
string screenshotId,
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
IOptions<MeetingAssistantOptions> options,
CancellationToken cancellationToken) =>
await RetryScreenshotOcrAsync(
null,
new ScreenshotOcrRetryRequest(
meetingNotePath,
transcriptPath,
assistantContextPath,
summaryPath,
screenshotPath,
screenshotId),
screenshotOcrRetryRunner,
options.Value,
null,
cancellationToken));
app.MapGet("/profiles/{launchProfile}/meetings/screenshot-ocr/retry", async (
string launchProfile,
string meetingNotePath,
string transcriptPath,
string assistantContextPath,
string summaryPath,
string screenshotPath,
string screenshotId,
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
IOptions<MeetingAssistantOptions> options,
ILaunchProfileOptionsProvider launchProfiles,
CancellationToken cancellationToken) =>
await RetryScreenshotOcrAsync(
launchProfile,
new ScreenshotOcrRetryRequest(
meetingNotePath,
transcriptPath,
assistantContextPath,
summaryPath,
screenshotPath,
screenshotId),
screenshotOcrRetryRunner,
options.Value,
launchProfiles,
cancellationToken));
static async Task<IResult> RetrySummaryAsync(
string? launchProfile,
string? summaryPath,
@@ -438,6 +518,50 @@ static async Task<IResult> RetrySummaryAsync(
});
}
static async Task<IResult> RetryScreenshotOcrAsync(
string? launchProfile,
ScreenshotOcrRetryRequest request,
IScreenshotOcrRetryRunner screenshotOcrRetryRunner,
MeetingAssistantOptions defaultOptions,
ILaunchProfileOptionsProvider? launchProfiles,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(request.MeetingNotePath) ||
string.IsNullOrWhiteSpace(request.TranscriptPath) ||
string.IsNullOrWhiteSpace(request.AssistantContextPath) ||
string.IsNullOrWhiteSpace(request.SummaryPath) ||
string.IsNullOrWhiteSpace(request.ScreenshotPath) ||
string.IsNullOrWhiteSpace(request.ScreenshotId))
{
return Results.BadRequest(new { error = "Meeting artifact paths, screenshot path, and screenshot id are required." });
}
var profileOptions = ResolveProfileOptions(launchProfile, defaultOptions, launchProfiles);
var artifacts = new MeetingSessionArtifacts(
request.MeetingNotePath,
request.TranscriptPath,
request.AssistantContextPath,
request.SummaryPath);
var trigger = await screenshotOcrRetryRunner.TriggerOcrRetryAsync(
artifacts,
request.ScreenshotPath,
request.ScreenshotId,
profileOptions,
cancellationToken);
if (trigger is null)
{
return Results.NotFound(new { error = "No retryable screenshot OCR block was found for the requested screenshot." });
}
return Results.Accepted(value: new
{
assistantContextPath = trigger.AssistantContextPath,
screenshotPath = trigger.ScreenshotPath,
screenshotId = trigger.ScreenshotId,
state = "ocr retrying"
});
}
app.MapPost("/asr/transcribe-file", async (
AsrDiagnosticRequest request,
AsrDiagnosticService diagnostics,
@@ -544,6 +668,14 @@ public sealed record AsrDiagnosticRequest(string Path, int? NumSpeakers = null);
public sealed record SummaryRetryRequest(string SummaryPath);
public sealed record ScreenshotOcrRetryRequest(
string MeetingNotePath,
string TranscriptPath,
string AssistantContextPath,
string SummaryPath,
string ScreenshotPath,
string ScreenshotId);
public sealed record SpeakerIdentityMergeDiagnosticRequest(double? RecentDays = null);
public sealed record WorkflowConfigurationReloadResponse(string? RulesPath);
@@ -66,32 +66,22 @@ public sealed class FileOfflineTranscriptionBacklog : IOfflineTranscriptionBackl
return items;
}
public async Task CompleteAsync(string id, CancellationToken cancellationToken)
public Task CompleteAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken)
{
var folder = GetBacklogFolder(options);
var path = GetItemPath(folder, id);
var path = GetItemPath(folder, item.Id);
if (!File.Exists(path))
{
return;
}
OfflineTranscriptionBacklogItem? item = null;
try
{
item = JsonSerializer.Deserialize<OfflineTranscriptionBacklogItem>(
await File.ReadAllTextAsync(path, cancellationToken),
JsonOptions);
}
catch (JsonException exception)
{
logger.LogWarning(exception, "Could not read completed offline transcription backlog item {BacklogItemPath}", path);
return Task.CompletedTask;
}
File.Delete(path);
if (item is not null && File.Exists(item.AudioPath))
if (File.Exists(item.AudioPath))
{
DeleteCompletedAudio(item.AudioPath);
}
return Task.CompletedTask;
}
private static string GetBacklogFolder(MeetingAssistantOptions options)
@@ -0,0 +1,12 @@
using NAudio.Wave;
namespace MeetingAssistant.Recording;
public interface IMicrophoneDeviceProvider
{
IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones();
MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options);
IWaveIn CreateCapture(MeetingAssistantOptions options);
}
@@ -28,6 +28,7 @@ public sealed class MeetingRecordingCoordinator
private readonly ILaunchProfileOptionsProvider? launchProfiles;
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
private readonly IMeetingScreenshotService screenshotService;
private readonly IMeetingNoteImageOcrService meetingNoteImageOcrService;
private readonly IMeetingRunArtifactCleaner artifactCleaner;
private readonly IMeetingInactivityPromptService inactivityPromptService;
private readonly IMeetingInactivityClock inactivityClock;
@@ -58,6 +59,7 @@ public sealed class MeetingRecordingCoordinator
ILaunchProfileOptionsProvider? launchProfiles = null,
IMeetingWorkflowEngine? meetingWorkflowEngine = null,
IMeetingScreenshotService? screenshotService = null,
IMeetingNoteImageOcrService? meetingNoteImageOcrService = null,
IMeetingRunArtifactCleaner? artifactCleaner = null,
IMeetingInactivityPromptService? inactivityPromptService = null,
IMeetingInactivityClock? inactivityClock = null,
@@ -78,6 +80,7 @@ public sealed class MeetingRecordingCoordinator
this.launchProfiles = launchProfiles;
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
this.screenshotService = screenshotService ?? NoopMeetingScreenshotService.Instance;
this.meetingNoteImageOcrService = meetingNoteImageOcrService ?? NoopMeetingNoteImageOcrService.Instance;
this.artifactCleaner = artifactCleaner ?? new MeetingRunArtifactCleaner();
this.inactivityPromptService = inactivityPromptService ?? new NoopMeetingInactivityPromptService();
this.inactivityClock = inactivityClock ?? new SystemMeetingInactivityClock();
@@ -142,12 +145,28 @@ public sealed class MeetingRecordingCoordinator
public async Task<RecordingStatus> StartAsync(CancellationToken cancellationToken)
{
return await StartAsync(null, cancellationToken);
return await StartAsync(null, null, suppressMetadataLookup: false, cancellationToken);
}
public async Task<RecordingStatus> StartFromPromptAsync(
MeetingMetadata? metadata,
CancellationToken cancellationToken)
{
return await StartAsync(null, metadata, suppressMetadataLookup: true, cancellationToken);
}
public async Task<RecordingStatus> StartAsync(
string? launchProfileName,
CancellationToken cancellationToken)
{
return await StartAsync(launchProfileName, null, suppressMetadataLookup: false, cancellationToken);
}
private async Task<RecordingStatus> StartAsync(
string? launchProfileName,
MeetingMetadata? suppliedMetadata,
bool suppressMetadataLookup,
CancellationToken cancellationToken)
{
await gate.WaitAsync(cancellationToken);
try
@@ -164,14 +183,16 @@ public sealed class MeetingRecordingCoordinator
var recordedAudio = await recordedAudioStore.CreateSessionAsync(runOptions, cancellationToken);
var assistantContextPath = GetAssistantContextPath(startedAt, runOptions);
var summaryPath = GetSummaryPath(startedAt, runOptions);
var meetingNote = MeetingNoteTemplate.Create(
title: $"Meeting {startedAt:yyyy-MM-dd HH:mm}",
startTime: startedAt,
attendees: [],
transcriptPath: currentSession.TranscriptPath,
assistantContextPath: assistantContextPath,
summaryPath: summaryPath);
currentMeetingNote = await meetingNoteStore.SaveAsync(
MeetingNoteTemplate.Create(
title: $"Meeting {startedAt:yyyy-MM-dd HH:mm}",
startTime: startedAt,
attendees: [],
transcriptPath: currentSession.TranscriptPath,
assistantContextPath: assistantContextPath,
summaryPath: summaryPath),
meetingNote,
runOptions,
cancellationToken);
currentArtifacts = new MeetingSessionArtifacts(
@@ -179,12 +200,32 @@ public sealed class MeetingRecordingCoordinator
currentSession.TranscriptPath,
assistantContextPath,
summaryPath);
await meetingArtifactStore.CreateAssistantContextAsync(currentArtifacts, currentMeetingNote, "", null, cancellationToken);
await meetingArtifactStore.CreateAssistantContextAsync(
currentArtifacts,
currentMeetingNote,
"",
null,
cancellationToken);
await meetingWorkflowEngine.RunAsync(
MeetingWorkflowEvent.Created(currentArtifacts),
runOptions,
cancellationToken);
currentMeetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
if (suppliedMetadata is not null)
{
await ApplyMeetingMetadataAsync(currentMeetingNote, suppliedMetadata, runOptions, cancellationToken);
currentMeetingNote = await meetingNoteStore.SaveAsync(
currentMeetingNote,
runOptions,
cancellationToken);
await meetingArtifactStore.UpdateAssistantContextMetadataAsync(
currentArtifacts,
currentMeetingNote,
suppliedMetadata.Agenda,
suppliedMetadata.ScheduledEnd,
cancellationToken);
}
await meetingArtifactStore.UpdateAssistantContextMeetingAsync(
currentArtifacts,
currentMeetingNote,
@@ -226,9 +267,20 @@ public sealed class MeetingRecordingCoordinator
}
currentRun = run;
_ = Task.Run(
() => ApplyMeetingMetadataWhenAvailableAsync(startedAt, run),
CancellationToken.None);
if (suppliedMetadata is null && !suppressMetadataLookup)
{
_ = Task.Run(
() => ApplyMeetingMetadataWhenAvailableAsync(startedAt, run),
CancellationToken.None);
}
else
{
await TransitionMeetingAsync(
run,
AssistantContextState.Transcribing,
cancellationToken);
}
logger.LogInformation("Meeting recording started");
return CurrentStatus;
@@ -445,7 +497,7 @@ public sealed class MeetingRecordingCoordinator
TimeSpan.Zero,
TimeSpan.Zero,
"System",
$"Transcription profile changed to {targetProfile.Name}.");
$"Transcription profile changed to {targetProfile.Name}. Speaker recognition and identities reset.");
run.AddLiveSegment(marker);
await AppendTranscriptSegmentAsync(run, marker, cancellationToken);
@@ -642,14 +694,23 @@ public sealed class MeetingRecordingCoordinator
CancellationToken cancellationToken)
{
var formatted = TranscriptLineFormatter.Format(segment);
var lineReference = segment.ReplacesMarkerId is { } replacesMarkerId &&
run.TryTakeTranscriptMarker(replacesMarkerId, out var markerReference)
? await ReplaceTranscriptMarkerAsync(run, markerReference, formatted, cancellationToken)
: segment.Kind == TranscriptionSegmentKind.Marker &&
segment.MarkerId is { } existingMarkerId &&
run.TryTakeTranscriptMarker(existingMarkerId, out var existingMarkerReference)
? await ReplaceTranscriptMarkerAsync(run, existingMarkerReference, formatted, cancellationToken)
: await transcriptStore.AppendLineAsync(run.Session, formatted.Line, cancellationToken);
TranscriptLineReference lineReference;
if (segment.ReplacesMarkerId is { } replacesMarkerId &&
run.TryTakeTranscriptMarker(replacesMarkerId, out var markerReference))
{
lineReference = await ReplaceTranscriptMarkerAsync(run, markerReference, formatted, cancellationToken);
}
else if (segment.Kind == TranscriptionSegmentKind.Marker &&
segment.MarkerId is { } existingMarkerId &&
run.TryTakeTranscriptMarker(existingMarkerId, out var existingMarkerReference))
{
lineReference = await ReplaceTranscriptMarkerAsync(run, existingMarkerReference, formatted, cancellationToken);
}
else
{
lineReference = await transcriptStore.AppendLineAsync(run.Session, formatted.Line, cancellationToken);
}
if (segment.Kind == TranscriptionSegmentKind.Marker && segment.MarkerId is { } markerId)
{
run.SetTranscriptMarker(markerId, lineReference);
@@ -1505,6 +1566,10 @@ public sealed class MeetingRecordingCoordinator
try
{
await meetingNoteImageOcrService.ProcessMeetingNoteImageEmbedsAsync(
run.Artifacts,
run.Options,
cancellationToken);
await screenshotService.WaitForPendingOcrAsync(
run.Artifacts,
run.Options.Screenshots.Ocr.GetEffectiveTimeout(),
@@ -0,0 +1,9 @@
namespace MeetingAssistant.Recording;
public sealed record MicrophoneDevice(
string Id,
string Name);
public sealed record MicrophoneDeviceSnapshot(
IReadOnlyList<MicrophoneDevice> Available,
MicrophoneDevice? Current);
@@ -0,0 +1,54 @@
namespace MeetingAssistant.Recording;
public sealed class MicrophoneDeviceSelection
{
private readonly object sync = new();
private string? selectedDeviceId;
public string? SelectedDeviceId
{
get
{
lock (sync)
{
return selectedDeviceId;
}
}
}
public void Select(string deviceId)
{
lock (sync)
{
selectedDeviceId = string.IsNullOrWhiteSpace(deviceId)
? null
: deviceId.Trim();
}
}
public MicrophoneDevice? Resolve(
string? configuredDeviceId,
MicrophoneDevice? defaultDevice,
IReadOnlyList<MicrophoneDevice> availableDevices)
{
var selected = SelectedDeviceId;
return FindById(availableDevices, selected) ??
FindById(availableDevices, configuredDeviceId) ??
defaultDevice;
}
private static MicrophoneDevice? FindById(
IReadOnlyList<MicrophoneDevice> devices,
string? id)
{
if (string.IsNullOrWhiteSpace(id))
{
return null;
}
return devices.FirstOrDefault(device => string.Equals(
device.Id,
id.Trim(),
StringComparison.OrdinalIgnoreCase));
}
}
@@ -6,10 +6,14 @@ namespace MeetingAssistant.Recording;
public sealed class MicrophoneAudioSource : IMeetingAudioSource
{
private readonly IMicrophoneDeviceProvider microphones;
private readonly ILogger<MicrophoneAudioSource> logger;
public MicrophoneAudioSource(ILogger<MicrophoneAudioSource> logger)
public MicrophoneAudioSource(
IMicrophoneDeviceProvider microphones,
ILogger<MicrophoneAudioSource> logger)
{
this.microphones = microphones;
this.logger = logger;
}
@@ -22,7 +26,7 @@ public sealed class MicrophoneAudioSource : IMeetingAudioSource
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return CaptureAsync(new WasapiCapture(), options, cancellationToken);
return CaptureAsync(microphones.CreateCapture(options), options, cancellationToken);
}
private IAsyncEnumerable<AudioChunk> CaptureAsync(
@@ -6,7 +6,7 @@ public interface IOfflineTranscriptionBacklog
Task<IReadOnlyList<OfflineTranscriptionBacklogItem>> ListAsync(CancellationToken cancellationToken);
Task CompleteAsync(string id, CancellationToken cancellationToken);
Task CompleteAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken);
}
public sealed record OfflineTranscriptionBacklogItem(
@@ -38,7 +38,7 @@ public sealed class NoopOfflineTranscriptionBacklog : IOfflineTranscriptionBackl
return Task.FromResult<IReadOnlyList<OfflineTranscriptionBacklogItem>>([]);
}
public Task CompleteAsync(string id, CancellationToken cancellationToken)
public Task CompleteAsync(OfflineTranscriptionBacklogItem item, CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
@@ -4,14 +4,11 @@ using MeetingAssistant.Summary;
using MeetingAssistant.Transcription;
using MeetingAssistant.Workflow;
using Microsoft.Extensions.Options;
using NAudio.Wave;
namespace MeetingAssistant.Recording;
public sealed class OfflineTranscriptionBacklogProcessor
{
private const int MaxChunkDurationMilliseconds = 1000;
private readonly IOfflineTranscriptionBacklog backlog;
private readonly ISpeechRecognitionPipelineFactory pipelineFactory;
private readonly ITranscriptStore transcriptStore;
@@ -72,7 +69,7 @@ public sealed class OfflineTranscriptionBacklogProcessor
try
{
await ProcessAsync(item, cancellationToken);
await backlog.CompleteAsync(item.Id, cancellationToken);
await backlog.CompleteAsync(item, cancellationToken);
logger.LogInformation(
"Completed offline transcription backlog item {BacklogItemId} for recording {RecordingPath}",
item.Id,
@@ -215,29 +212,9 @@ public sealed class OfflineTranscriptionBacklogProcessor
ISpeechRecognitionPipeline pipeline,
CancellationToken cancellationToken)
{
using var reader = new WaveFileReader(audioPath);
var bytesPerSecond = reader.WaveFormat.AverageBytesPerSecond;
var chunkSize = Math.Max(
reader.WaveFormat.BlockAlign,
bytesPerSecond * MaxChunkDurationMilliseconds / 1000);
chunkSize -= chunkSize % reader.WaveFormat.BlockAlign;
var buffer = new byte[chunkSize];
while (true)
await foreach (var chunk in PcmWavAudioChunkReader.ReadChunksAsync(audioPath, cancellationToken: cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
var read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken);
if (read == 0)
{
break;
}
await pipeline.WriteAsync(
new AudioChunk(
buffer[..read],
reader.WaveFormat.SampleRate,
reader.WaveFormat.Channels),
cancellationToken);
await pipeline.WriteAsync(chunk, cancellationToken);
}
}
}
@@ -0,0 +1,48 @@
using System.Runtime.CompilerServices;
using NAudio.Wave;
namespace MeetingAssistant.Recording;
public static class PcmWavAudioChunkReader
{
private const int MaxChunkDurationMilliseconds = 1000;
public static async IAsyncEnumerable<AudioChunk> ReadChunksAsync(
string path,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var reader = new WaveFileReader(path);
try
{
var format = reader.WaveFormat;
if (format.Encoding != WaveFormatEncoding.Pcm || format.BitsPerSample != 16)
{
throw new InvalidDataException(
$"Only 16-bit PCM WAV files are supported. Found {format.Encoding} with {format.BitsPerSample} bits per sample.");
}
var chunkSize = Math.Max(
format.BlockAlign,
format.AverageBytesPerSecond * MaxChunkDurationMilliseconds / 1000);
chunkSize -= chunkSize % format.BlockAlign;
var buffer = new byte[chunkSize];
int read;
while ((read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)) > 0)
{
yield return new AudioChunk(buffer[..read], format.SampleRate, format.Channels);
}
}
finally
{
try
{
reader.Dispose();
}
catch (NotSupportedException)
{
// NAudio can throw while disposing reader streams from async iterators after all chunks were read.
}
}
}
}
@@ -0,0 +1,79 @@
using NAudio.CoreAudioApi;
using NAudio.Wave;
namespace MeetingAssistant.Recording;
public sealed class WindowsMicrophoneDeviceProvider : IMicrophoneDeviceProvider
{
private readonly MicrophoneDeviceSelection selection;
private readonly ILogger<WindowsMicrophoneDeviceProvider> logger;
public WindowsMicrophoneDeviceProvider(
MicrophoneDeviceSelection selection,
ILogger<WindowsMicrophoneDeviceProvider> logger)
{
this.selection = selection;
this.logger = logger;
}
public IReadOnlyList<MicrophoneDevice> GetAvailableMicrophones()
{
try
{
using var enumerator = new MMDeviceEnumerator();
return enumerator
.EnumerateAudioEndPoints(DataFlow.Capture, DeviceState.Active)
.Select(ToMicrophoneDevice)
.OrderBy(device => device.Name, StringComparer.CurrentCultureIgnoreCase)
.ToArray();
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not enumerate microphone capture endpoints");
return [];
}
}
public MicrophoneDeviceSnapshot GetMicrophoneSnapshot(MeetingAssistantOptions options)
{
var devices = GetAvailableMicrophones();
return new MicrophoneDeviceSnapshot(
devices,
selection.Resolve(options.Recording.MicrophoneDeviceId, GetDefaultMicrophone(), devices));
}
public IWaveIn CreateCapture(MeetingAssistantOptions options)
{
var current = GetMicrophoneSnapshot(options).Current;
if (current is null)
{
logger.LogInformation("Starting microphone capture from Windows default capture endpoint");
return new WasapiCapture();
}
logger.LogInformation(
"Starting microphone capture from {MicrophoneName} ({MicrophoneDeviceId})",
current.Name,
current.Id);
using var enumerator = new MMDeviceEnumerator();
return new WasapiCapture(enumerator.GetDevice(current.Id));
}
private static MicrophoneDevice? GetDefaultMicrophone()
{
try
{
using var enumerator = new MMDeviceEnumerator();
return ToMicrophoneDevice(enumerator.GetDefaultAudioEndpoint(DataFlow.Capture, Role.Console));
}
catch
{
return null;
}
}
private static MicrophoneDevice ToMicrophoneDevice(MMDevice device)
{
return new MicrophoneDevice(device.ID, device.FriendlyName);
}
}
@@ -3,6 +3,7 @@ using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.RegularExpressions;
using MeetingAssistant.MeetingNotes;
namespace MeetingAssistant.Screenshots;
@@ -104,41 +105,69 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
private static ScreenshotOcrResult ParseOcrResult(string text)
{
ScreenshotCropCoordinates? crop = null;
IReadOnlyList<string> attendees = [];
var cleaned = JsonCodeBlockRegex().Replace(text, match =>
{
if (TryParseCrop(match.Groups["json"].Value, out var parsedCrop))
if (TryParseMetadata(match.Groups["json"].Value, out var parsedCrop, out var parsedAttendees))
{
crop = parsedCrop;
attendees = parsedAttendees;
return "";
}
return match.Value;
}).Trim();
return new ScreenshotOcrResult(cleaned, crop);
return new ScreenshotOcrResult(cleaned, crop, attendees);
}
private static bool TryParseCrop(string json, out ScreenshotCropCoordinates? crop)
private static bool TryParseMetadata(
string json,
out ScreenshotCropCoordinates? crop,
out IReadOnlyList<string> attendees)
{
crop = null;
attendees = [];
try
{
using var document = JsonDocument.Parse(json);
if (!document.RootElement.TryGetProperty("crop", out var cropElement) ||
cropElement.ValueKind == JsonValueKind.Null)
{
return true;
}
if (cropElement.ValueKind != JsonValueKind.Object ||
!TryGetInt(cropElement, "x", out var x) ||
!TryGetInt(cropElement, "y", out var y) ||
!TryGetInt(cropElement, "width", out var width) ||
!TryGetInt(cropElement, "height", out var height))
if (document.RootElement.ValueKind != JsonValueKind.Object)
{
return false;
}
crop = new ScreenshotCropCoordinates(x, y, width, height);
var hasMetadata = false;
if (document.RootElement.TryGetProperty("attendees", out var attendeesElement))
{
hasMetadata = true;
if (attendeesElement.ValueKind == JsonValueKind.Array)
{
attendees = MeetingAttendeeNames.NormalizeDistinct(attendeesElement
.EnumerateArray()
.Where(element => element.ValueKind == JsonValueKind.String)
.Select(element => element.GetString() ?? ""));
}
}
if (!document.RootElement.TryGetProperty("crop", out var cropElement))
{
return hasMetadata;
}
hasMetadata = true;
if (cropElement.ValueKind == JsonValueKind.Null)
{
return true;
}
if (cropElement.ValueKind == JsonValueKind.Object &&
TryGetInt(cropElement, "x", out var x) &&
TryGetInt(cropElement, "y", out var y) &&
TryGetInt(cropElement, "width", out var width) &&
TryGetInt(cropElement, "height", out var height))
{
crop = new ScreenshotCropCoordinates(x, y, width, height);
}
return true;
}
catch (JsonException)
@@ -2,7 +2,9 @@ using System.Collections.Concurrent;
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Speakers;
namespace MeetingAssistant.Screenshots;
@@ -22,7 +24,24 @@ public interface IScreenshotOcrClient
CancellationToken cancellationToken);
}
public sealed record ScreenshotOcrResult(string Text, ScreenshotCropCoordinates? Crop);
public sealed record ScreenshotOcrResult
{
public ScreenshotOcrResult(
string text,
ScreenshotCropCoordinates? crop,
IReadOnlyList<string>? attendees = null)
{
Text = text;
Crop = crop;
Attendees = attendees ?? [];
}
public string Text { get; init; }
public ScreenshotCropCoordinates? Crop { get; init; }
public IReadOnlyList<string> Attendees { get; init; }
}
public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height);
@@ -46,11 +65,36 @@ public interface IMeetingScreenshotService
CancellationToken cancellationToken);
}
public interface IMeetingNoteImageOcrService
{
Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken);
}
public interface IScreenshotOcrRetryRunner
{
Task<MeetingScreenshotOcrRetryTriggerResult?> TriggerOcrRetryAsync(
MeetingSessionArtifacts artifacts,
string screenshotPath,
string screenshotId,
MeetingAssistantOptions options,
CancellationToken cancellationToken);
}
public sealed record MeetingScreenshotCaptureResult(
string ScreenshotPath,
TimeSpan MeetingTimestamp,
bool OcrStarted);
public sealed record MeetingScreenshotOcrRetryTriggerResult(
string AssistantContextPath,
string ScreenshotPath,
string ScreenshotId);
public sealed record MeetingNoteImageOcrQueueResult(int QueuedCount);
public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
{
public static NoopMeetingScreenshotService Instance { get; } = new();
@@ -80,12 +124,31 @@ public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
{
return Task.CompletedTask;
}
}
public sealed class MeetingScreenshotService : IMeetingScreenshotService
public sealed class NoopMeetingNoteImageOcrService : IMeetingNoteImageOcrService
{
public static NoopMeetingNoteImageOcrService Instance { get; } = new();
public Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return Task.FromResult(new MeetingNoteImageOcrQueueResult(0));
}
}
public sealed partial class MeetingScreenshotService :
IMeetingScreenshotService,
IScreenshotOcrRetryRunner,
IMeetingNoteImageOcrService
{
private readonly IActiveWindowScreenshotCapture screenshotCapture;
private readonly IMeetingArtifactStore artifactStore;
private readonly IMeetingNoteStore meetingNoteStore;
private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer;
private readonly IScreenshotOcrClient ocrClient;
private readonly ILogger<MeetingScreenshotService> logger;
private readonly ConcurrentDictionary<string, List<PendingOcrTask>> pendingOcrByContext = new(StringComparer.OrdinalIgnoreCase);
@@ -94,11 +157,15 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
public MeetingScreenshotService(
IActiveWindowScreenshotCapture screenshotCapture,
IMeetingArtifactStore artifactStore,
IMeetingNoteStore meetingNoteStore,
ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer,
IScreenshotOcrClient ocrClient,
ILogger<MeetingScreenshotService> logger)
{
this.screenshotCapture = screenshotCapture;
this.artifactStore = artifactStore;
this.meetingNoteStore = meetingNoteStore;
this.attendeeCanonicalizer = attendeeCanonicalizer;
this.ocrClient = ocrClient;
this.logger = logger;
}
@@ -135,7 +202,13 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
TrackOcr(
artifacts,
ocrCancellation,
RunOcrAsync(artifacts, screenshotPath, screenshotId, options, ocrCancellation.Token));
RunOcrAsync(
artifacts,
screenshotPath,
screenshotId,
options,
ScreenshotOcrPolicy.CapturedScreenshot,
ocrCancellation.Token));
}
logger.LogInformation(
@@ -212,6 +285,100 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
await WaitForPendingOcrAsync(artifacts, timeout, cancellationToken);
}
public async Task<MeetingScreenshotOcrRetryTriggerResult?> TriggerOcrRetryAsync(
MeetingSessionArtifacts artifacts,
string screenshotPath,
string screenshotId,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(screenshotId) ||
string.IsNullOrWhiteSpace(screenshotPath) ||
!File.Exists(screenshotPath) ||
!File.Exists(artifacts.AssistantContextPath))
{
return null;
}
var replaced = await ReplaceOcrPlaceholderAsync(
artifacts.AssistantContextPath,
screenshotId,
"_OCR retry pending..._",
cancellationToken,
preserveMarkers: true,
appendWhenMissing: false);
if (!replaced)
{
return null;
}
var ocrCancellation = new CancellationTokenSource();
TrackOcr(
artifacts,
ocrCancellation,
RunOcrAsync(
artifacts,
screenshotPath,
screenshotId,
options,
ScreenshotOcrPolicy.CapturedScreenshot,
ocrCancellation.Token));
return new MeetingScreenshotOcrRetryTriggerResult(
artifacts.AssistantContextPath,
screenshotPath,
screenshotId);
}
public async Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
if (!options.Screenshots.Ocr.Enabled || !File.Exists(artifacts.MeetingNotePath))
{
return new MeetingNoteImageOcrQueueResult(0);
}
var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken);
var embeds = FindMeetingNoteImageEmbeds(
meetingNote.UserNotes,
artifacts.MeetingNotePath,
options);
var queuedEmbeds = new List<(MeetingNoteImageEmbed Embed, string OcrId)>();
foreach (var embed in embeds)
{
cancellationToken.ThrowIfCancellationRequested();
var ocrId = Guid.NewGuid().ToString("N");
await artifactStore.AppendAssistantContextAsync(
artifacts,
CreateMeetingNoteImageMarkdown(
ocrId,
embed.OriginalEmbed,
embed.ImagePath,
artifacts.AssistantContextPath),
cancellationToken);
queuedEmbeds.Add((embed, ocrId));
}
foreach (var (embed, ocrId) in queuedEmbeds)
{
cancellationToken.ThrowIfCancellationRequested();
var ocrCancellation = new CancellationTokenSource();
TrackOcr(
artifacts,
ocrCancellation,
RunOcrAsync(
artifacts,
embed.ImagePath,
ocrId,
options,
ScreenshotOcrPolicy.MeetingNoteImage,
ocrCancellation.Token));
}
return new MeetingNoteImageOcrQueueResult(queuedEmbeds.Count);
}
private async Task<string> SaveScreenshotAsync(
MeetingSessionArtifacts artifacts,
byte[] pngBytes,
@@ -234,6 +401,7 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
string screenshotPath,
string screenshotId,
MeetingAssistantOptions options,
ScreenshotOcrPolicy policy,
CancellationToken cancellationToken)
{
var timeout = options.Screenshots.Ocr.GetEffectiveTimeout();
@@ -248,17 +416,26 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
: options.Screenshots.Ocr.Prompt,
options,
timeoutSource.Token);
var cropMarkdown = await TryCreateCropMarkdownAsync(
artifacts.AssistantContextPath,
screenshotPath,
result.Crop,
timeoutSource.Token);
var cropMarkdown = policy.AllowCrop
? await TryCreateCropMarkdownAsync(
artifacts.AssistantContextPath,
screenshotPath,
result.Crop,
timeoutSource.Token)
: "";
await ReplaceOcrPlaceholderAsync(
artifacts.AssistantContextPath,
screenshotId,
cropMarkdown +
"### OCR" + Environment.NewLine + Environment.NewLine + result.Text.Trim(),
CancellationToken.None);
if (policy.AddAttendees)
{
await AddOcrAttendeesAsync(
artifacts,
result.Attendees,
CancellationToken.None);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
@@ -269,8 +446,15 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
await ReplaceOcrPlaceholderAsync(
artifacts.AssistantContextPath,
screenshotId,
$"_OCR timed out after {timeout:g}._",
CancellationToken.None);
CreateOcrFailureMarkdown(
artifacts,
screenshotPath,
screenshotId,
options,
$"_OCR timed out after {timeout:g}._",
policy.IncludeRetryLink),
CancellationToken.None,
preserveMarkers: true);
}
catch (Exception exception)
{
@@ -278,16 +462,63 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
await ReplaceOcrPlaceholderAsync(
artifacts.AssistantContextPath,
screenshotId,
$"_OCR failed: {exception.Message}_",
CancellationToken.None);
CreateOcrFailureMarkdown(
artifacts,
screenshotPath,
screenshotId,
options,
$"_OCR failed: {exception.Message}_",
policy.IncludeRetryLink),
CancellationToken.None,
preserveMarkers: true);
}
}
private async Task ReplaceOcrPlaceholderAsync(
private async Task AddOcrAttendeesAsync(
MeetingSessionArtifacts artifacts,
IReadOnlyList<string> attendees,
CancellationToken cancellationToken)
{
var additions = MeetingAttendeeNames.NormalizeDistinct(attendees);
if (additions.Count == 0)
{
return;
}
try
{
var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken);
var canonicalized = await attendeeCanonicalizer.CanonicalizeAsync(
meetingNote.Frontmatter.Attendees.Concat(additions).ToList(),
cancellationToken);
if (meetingNote.Frontmatter.Attendees.SequenceEqual(canonicalized, StringComparer.Ordinal))
{
return;
}
meetingNote.Frontmatter.Attendees = canonicalized.ToList();
await meetingNoteStore.SaveAsync(meetingNote, cancellationToken);
logger.LogInformation(
"Added {AttendeeCount} screenshot OCR attendee candidate(s) to meeting note {MeetingNotePath}",
additions.Count,
artifacts.MeetingNotePath);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogWarning(
exception,
"Could not add screenshot OCR attendees to meeting note {MeetingNotePath}",
artifacts.MeetingNotePath);
}
}
private async Task<bool> ReplaceOcrPlaceholderAsync(
string assistantContextPath,
string screenshotId,
string replacement,
CancellationToken cancellationToken)
CancellationToken cancellationToken,
bool preserveMarkers = false,
bool appendWhenMissing = true)
{
var fileLock = contextFileLocks.GetOrAdd(
NormalizeContextKey(assistantContextPath),
@@ -297,28 +528,41 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
{
if (!File.Exists(assistantContextPath))
{
return;
return false;
}
var content = await File.ReadAllTextAsync(assistantContextPath, cancellationToken);
var startMarker = $"<!-- screenshot-ocr:{screenshotId} -->";
var endMarker = $"<!-- /screenshot-ocr:{screenshotId} -->";
var startMarker = CreateOcrStartMarker(screenshotId);
var endMarker = CreateOcrEndMarker(screenshotId);
var startIndex = content.IndexOf(startMarker, StringComparison.Ordinal);
var endIndex = content.IndexOf(endMarker, StringComparison.Ordinal);
if (startIndex < 0 || endIndex < startIndex)
{
await File.AppendAllTextAsync(
assistantContextPath,
Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
cancellationToken);
return;
if (appendWhenMissing)
{
await File.AppendAllTextAsync(
assistantContextPath,
Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
cancellationToken);
}
return false;
}
endIndex += endMarker.Length;
var updated = content[..startIndex] +
replacement.TrimEnd() +
content[endIndex..];
var updated = preserveMarkers
? content[..startIndex] +
startMarker +
Environment.NewLine +
replacement.TrimEnd() +
Environment.NewLine +
endMarker +
content[endIndex..]
: content[..startIndex] +
replacement.TrimEnd() +
content[endIndex..];
await File.WriteAllTextAsync(assistantContextPath, updated, cancellationToken);
return true;
}
finally
{
@@ -326,6 +570,29 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
}
}
private static string CreateOcrFailureMarkdown(
MeetingSessionArtifacts artifacts,
string screenshotPath,
string screenshotId,
MeetingAssistantOptions options,
string status,
bool includeRetryLink = true)
{
var markdown = status.TrimEnd();
if (includeRetryLink)
{
markdown += Environment.NewLine +
Environment.NewLine +
MeetingNoteActionLinks.CreateScreenshotOcrRetryLink(
options.Api.PublicBaseUrl,
artifacts,
screenshotPath,
screenshotId);
}
return markdown;
}
private async Task<string> TryCreateCropMarkdownAsync(
string assistantContextPath,
string screenshotPath,
@@ -435,13 +702,189 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
return content +
Environment.NewLine +
Environment.NewLine +
$"<!-- screenshot-ocr:{screenshotId} -->" +
Environment.NewLine +
"_OCR pending..._" +
Environment.NewLine +
$"<!-- /screenshot-ocr:{screenshotId} -->";
CreateOcrBlock(screenshotId, "_OCR pending..._");
}
private static string CreateMeetingNoteImageMarkdown(
string ocrId,
string originalEmbed,
string imagePath,
string assistantContextPath)
{
var relativePath = ToMarkdownPath(Path.GetRelativePath(
Path.GetDirectoryName(assistantContextPath)!,
imagePath));
return "## Meeting Note Image" +
Environment.NewLine +
"Image from meeting note." +
Environment.NewLine +
$"Original embed: `{originalEmbed.Replace("`", "\\`", StringComparison.Ordinal)}`" +
Environment.NewLine +
$"![Meeting note image]({relativePath})" +
Environment.NewLine +
Environment.NewLine +
CreateOcrBlock(ocrId, "_OCR pending..._");
}
private static string CreateOcrBlock(string screenshotId, string content)
{
return CreateOcrStartMarker(screenshotId) +
Environment.NewLine +
content.TrimEnd() +
Environment.NewLine +
CreateOcrEndMarker(screenshotId);
}
private static string CreateOcrStartMarker(string screenshotId)
{
return $"<!-- screenshot-ocr:{screenshotId} -->";
}
private static string CreateOcrEndMarker(string screenshotId)
{
return $"<!-- /screenshot-ocr:{screenshotId} -->";
}
private static IReadOnlyList<MeetingNoteImageEmbed> FindMeetingNoteImageEmbeds(
string userNotes,
string meetingNotePath,
MeetingAssistantOptions options)
{
if (string.IsNullOrWhiteSpace(userNotes))
{
return [];
}
var result = new List<MeetingNoteImageEmbed>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (Match match in ObsidianImageEmbedRegex().Matches(userNotes))
{
AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options);
}
foreach (Match match in MarkdownImageEmbedRegex().Matches(userNotes))
{
AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options);
}
return result;
}
private static void AddResolvedEmbed(
List<MeetingNoteImageEmbed> embeds,
HashSet<string> seen,
string originalEmbed,
string target,
string meetingNotePath,
MeetingAssistantOptions options)
{
var imagePath = ResolveMeetingNoteImagePath(target, meetingNotePath, options);
if (imagePath is null || !seen.Add(imagePath))
{
return;
}
embeds.Add(new MeetingNoteImageEmbed(originalEmbed, imagePath));
}
private static string? ResolveMeetingNoteImagePath(
string target,
string meetingNotePath,
MeetingAssistantOptions options)
{
var cleaned = CleanEmbedTarget(target);
if (string.IsNullOrWhiteSpace(cleaned) ||
cleaned.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
cleaned.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ||
cleaned.StartsWith("data:", StringComparison.OrdinalIgnoreCase) ||
!IsSupportedImagePath(cleaned))
{
return null;
}
var candidates = new List<string>();
if (Path.IsPathRooted(cleaned))
{
candidates.Add(VaultPath.Resolve(cleaned));
}
else
{
candidates.Add(Path.GetFullPath(Path.Combine(Path.GetDirectoryName(meetingNotePath)!, cleaned)));
candidates.Add(VaultPath.Resolve(options.Vault, cleaned));
}
foreach (var candidate in candidates)
{
if (File.Exists(candidate))
{
return candidate;
}
}
if (Path.GetFileName(cleaned).Equals(cleaned, StringComparison.Ordinal))
{
try
{
var vaultRoot = VaultPath.Resolve(options.Vault.BaseFolder);
return Directory.EnumerateFiles(vaultRoot, cleaned, SearchOption.AllDirectories)
.FirstOrDefault(IsSupportedImagePath);
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or DirectoryNotFoundException)
{
return null;
}
}
return null;
}
private static string CleanEmbedTarget(string target)
{
var cleaned = target.Trim();
if (cleaned.StartsWith("<", StringComparison.Ordinal) &&
cleaned.EndsWith(">", StringComparison.Ordinal) &&
cleaned.Length > 1)
{
cleaned = cleaned[1..^1].Trim();
}
var pipeIndex = cleaned.IndexOf('|', StringComparison.Ordinal);
if (pipeIndex >= 0)
{
cleaned = cleaned[..pipeIndex].Trim();
}
var hashIndex = cleaned.IndexOf('#', StringComparison.Ordinal);
if (hashIndex >= 0)
{
cleaned = cleaned[..hashIndex].Trim();
}
try
{
return Uri.UnescapeDataString(cleaned.Replace('/', Path.DirectorySeparatorChar));
}
catch (UriFormatException)
{
return cleaned.Replace('/', Path.DirectorySeparatorChar);
}
}
private static bool IsSupportedImagePath(string path)
{
return Path.GetExtension(path).ToLowerInvariant() switch
{
".png" or ".jpg" or ".jpeg" or ".gif" or ".webp" or ".bmp" or ".tif" or ".tiff" => true,
_ => false
};
}
[GeneratedRegex(@"!\[\[(?<target>[^\]\r\n]+)\]\]", RegexOptions.CultureInvariant)]
private static partial Regex ObsidianImageEmbedRegex();
[GeneratedRegex(@"!\[[^\]\r\n]*\]\((?<target>[^)\r\n]+)\)", RegexOptions.CultureInvariant)]
private static partial Regex MarkdownImageEmbedRegex();
private static string ResolveAttachmentsFolder(string assistantContextPath, string configuredFolder)
{
var expanded = Environment.ExpandEnvironmentVariables(
@@ -479,4 +922,19 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
}
private sealed record PendingOcrTask(Task Task, CancellationTokenSource Cancellation);
private sealed record ScreenshotOcrPolicy(bool AllowCrop, bool AddAttendees, bool IncludeRetryLink)
{
public static ScreenshotOcrPolicy CapturedScreenshot { get; } = new(
AllowCrop: true,
AddAttendees: true,
IncludeRetryLink: true);
public static ScreenshotOcrPolicy MeetingNoteImage { get; } = new(
AllowCrop: false,
AddAttendees: false,
IncludeRetryLink: false);
}
private sealed record MeetingNoteImageEmbed(string OriginalEmbed, string ImagePath);
}
@@ -114,11 +114,7 @@ public sealed class PassthroughSpeakerIdentityAttendeeCanonicalizer : ISpeakerId
IReadOnlyList<string> attendees,
CancellationToken cancellationToken)
{
var distinctAttendees = attendees
.Select(attendee => attendee.Trim())
.Where(name => !string.IsNullOrWhiteSpace(name))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
var distinctAttendees = MeetingAttendeeNames.NormalizeDistinct(attendees);
return Task.FromResult<IReadOnlyList<string>>(distinctAttendees);
}
}
+34 -4
View File
@@ -6,10 +6,12 @@ namespace MeetingAssistant.Taskbar;
public enum MeetingTaskbarAction
{
EditRules,
OpenSubmenu,
StartRecording,
StopRecording,
AbortRecording,
SwitchProfile,
SelectMicrophone,
Exit
}
@@ -21,19 +23,29 @@ public sealed record MeetingTaskbarMenu(
public sealed record MeetingTaskbarMenuItem(
string Text,
MeetingTaskbarAction Action,
string? ProfileName = null);
string? ProfileName = null,
string? MicrophoneDeviceId = null,
bool IsChecked = false,
IReadOnlyList<MeetingTaskbarMenuItem>? Items = null);
public static class MeetingTaskbarMenuBuilder
{
public static MeetingTaskbarMenu Build(
RecordingStatus status,
IReadOnlyList<LaunchProfile> launchProfiles)
IReadOnlyList<LaunchProfile> launchProfiles,
IReadOnlyList<MicrophoneDevice>? microphones = null,
string? currentMicrophoneDeviceId = null)
{
var items = new List<MeetingTaskbarMenuItem>
{
new("Settings and logs", MeetingTaskbarAction.EditRules)
};
if (microphones is { Count: > 0 })
{
items.Add(BuildMicrophoneMenu(microphones, currentMicrophoneDeviceId));
}
if (status.IsRecording)
{
items.Add(new MeetingTaskbarMenuItem(
@@ -72,6 +84,24 @@ public static class MeetingTaskbarMenuBuilder
items);
}
private static MeetingTaskbarMenuItem BuildMicrophoneMenu(
IReadOnlyList<MicrophoneDevice> microphones,
string? currentMicrophoneDeviceId)
{
var microphoneItems = microphones
.Select(microphone => new MeetingTaskbarMenuItem(
microphone.Name,
MeetingTaskbarAction.SelectMicrophone,
MicrophoneDeviceId: microphone.Id,
IsChecked: string.Equals(microphone.Id, currentMicrophoneDeviceId, StringComparison.Ordinal)))
.ToArray();
return new MeetingTaskbarMenuItem(
"Microphone",
MeetingTaskbarAction.OpenSubmenu,
Items: microphoneItems);
}
private static string BuildTooltip(RecordingStatus status)
{
return status.State switch
@@ -104,8 +134,8 @@ public static class MeetingTaskbarMenuBuilder
public static class MeetingTaskbarExitPolicy
{
public static bool RequiresConfirmation(RecordingStatus status)
public static bool RequiresConfirmation(RecordingProcessState state)
{
return status.State != RecordingProcessState.Idle;
return state != RecordingProcessState.Idle;
}
}
@@ -14,6 +14,8 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
private readonly MeetingRecordingCoordinator coordinator;
private readonly ILaunchProfileOptionsProvider launchProfiles;
private readonly IMicrophoneDeviceProvider microphones;
private readonly MicrophoneDeviceSelection microphoneSelection;
private readonly IWorkflowRulesEditorWindowService rulesEditorWindow;
private readonly IHostApplicationLifetime applicationLifetime;
private readonly ILogger<UnoTaskbarIconService> logger;
@@ -29,12 +31,16 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
public UnoTaskbarIconService(
MeetingRecordingCoordinator coordinator,
ILaunchProfileOptionsProvider launchProfiles,
IMicrophoneDeviceProvider microphones,
MicrophoneDeviceSelection microphoneSelection,
IWorkflowRulesEditorWindowService rulesEditorWindow,
IHostApplicationLifetime applicationLifetime,
ILogger<UnoTaskbarIconService> logger)
{
this.coordinator = coordinator;
this.launchProfiles = launchProfiles;
this.microphones = microphones;
this.microphoneSelection = microphoneSelection;
this.rulesEditorWindow = rulesEditorWindow;
this.applicationLifetime = applicationLifetime;
this.logger = logger;
@@ -167,6 +173,8 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
private MeetingTaskbarMenu BuildMenu()
{
var profiles = launchProfiles.GetProfiles();
var activeOptions = GetActiveProfileOptions(profiles);
var microphoneSnapshot = microphones.GetMicrophoneSnapshot(activeOptions);
var profilesSignature = string.Join(
", ",
profiles.Select(profile => $"{profile.Name}:{profile.Options.Hotkey.Toggle}"));
@@ -178,7 +186,9 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
return MeetingTaskbarMenuBuilder.Build(
coordinator.CurrentStatus,
profiles);
profiles,
microphoneSnapshot.Available,
microphoneSnapshot.Current?.Id);
}
private PopupMenu BuildContextMenu(MeetingTaskbarMenu menu)
@@ -194,14 +204,33 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
}
var menuItem = menu.Items[index];
popupMenu.Items.Add(new PopupMenuItem(
menuItem.Text,
(_, _) => _ = ExecuteMenuItemAsync(menuItem)));
popupMenu.Items.Add(BuildPopupItem(menuItem));
}
return popupMenu;
}
private PopupItem BuildPopupItem(MeetingTaskbarMenuItem menuItem)
{
if (menuItem.Items is { Count: > 0 })
{
var submenu = new PopupSubMenu(menuItem.Text);
foreach (var child in menuItem.Items)
{
submenu.Items.Add(BuildPopupItem(child));
}
return submenu;
}
return new PopupMenuItem(
menuItem.Text,
(_, _) => _ = ExecuteMenuItemAsync(menuItem))
{
Checked = menuItem.IsChecked
};
}
private async Task ExecuteMenuItemAsync(MeetingTaskbarMenuItem item)
{
try
@@ -227,6 +256,13 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
break;
case MeetingTaskbarAction.SwitchProfile:
await coordinator.ToggleAsync(item.ProfileName, CancellationToken.None);
break;
case MeetingTaskbarAction.SelectMicrophone:
if (!string.IsNullOrWhiteSpace(item.MicrophoneDeviceId))
{
microphoneSelection.Select(item.MicrophoneDeviceId);
}
break;
case MeetingTaskbarAction.Exit:
ExitApplication();
@@ -253,13 +289,47 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
{
return string.Join(
"|",
menu.Items.Select(item => $"{item.Action}:{item.ProfileName}:{item.Text}"));
FlattenMenuItems(menu.Items).Select(item =>
$"{item.Action}:{item.ProfileName}:{item.MicrophoneDeviceId}:{item.IsChecked}:{item.Text}"));
}
private static IEnumerable<MeetingTaskbarMenuItem> FlattenMenuItems(
IEnumerable<MeetingTaskbarMenuItem> items)
{
foreach (var item in items)
{
yield return item;
if (item.Items is null)
{
continue;
}
foreach (var child in FlattenMenuItems(item.Items))
{
yield return child;
}
}
}
private MeetingAssistantOptions GetActiveProfileOptions(IReadOnlyList<LaunchProfile> profiles)
{
var activeProfile = coordinator.CurrentStatus.LaunchProfile;
return profiles.FirstOrDefault(profile => string.Equals(
profile.Name,
activeProfile,
StringComparison.OrdinalIgnoreCase))?.Options ??
profiles.FirstOrDefault(profile => string.Equals(
profile.Name,
"default",
StringComparison.OrdinalIgnoreCase))?.Options ??
profiles.FirstOrDefault()?.Options ??
new MeetingAssistantOptions();
}
private void ExitApplication()
{
var status = coordinator.CurrentStatus;
if (MeetingTaskbarExitPolicy.RequiresConfirmation(status) && !ConfirmExitDuringProcessing(status.State))
if (MeetingTaskbarExitPolicy.RequiresConfirmation(status.State) && !ConfirmExitDuringProcessing(status.State))
{
logger.LogInformation("Taskbar exit canceled while meeting state was {State}", status.State);
return;
@@ -1,5 +1,4 @@
using MeetingAssistant.Recording;
using NAudio.Wave;
namespace MeetingAssistant.Transcription;
@@ -77,7 +76,7 @@ public sealed class AsrDiagnosticService
bool paceAudio,
CancellationToken cancellationToken)
{
await foreach (var chunk in ReadPcmWavChunks(fullPath, cancellationToken))
await foreach (var chunk in PcmWavAudioChunkReader.ReadChunksAsync(fullPath, cancellationToken: cancellationToken))
{
await pipeline.WriteAsync(chunk, cancellationToken);
if (paceAudio && chunk.Duration > TimeSpan.Zero)
@@ -102,40 +101,6 @@ public sealed class AsrDiagnosticService
return fullPath;
}
private static async IAsyncEnumerable<AudioChunk> ReadPcmWavChunks(
string path,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var reader = new WaveFileReader(path);
try
{
var format = reader.WaveFormat;
if (format.Encoding != WaveFormatEncoding.Pcm || format.BitsPerSample != 16)
{
throw new InvalidDataException(
$"Only 16-bit PCM WAV files are supported. Found {format.Encoding} with {format.BitsPerSample} bits per sample.");
}
var buffer = new byte[format.AverageBytesPerSecond];
int read;
while ((read = await reader.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)) > 0)
{
yield return new AudioChunk(buffer[..read], format.SampleRate, format.Channels);
}
}
finally
{
try
{
reader.Dispose();
}
catch (NotSupportedException)
{
// NAudio can throw while disposing reader streams from async iterators; diagnostics should not fail after reading all chunks.
}
}
}
}
public sealed record AsrDiagnosticResult(string Path, IReadOnlyList<TranscriptionSegment> Segments);
+1
View File
@@ -17,6 +17,7 @@
"TranscriptionProvider": "azure-speech",
"SampleRate": 16000,
"Channels": 1,
"MicrophoneDeviceId": "",
"MicrophoneMixGain": 1,
"SystemAudioMixGain": 1,
"StopProcessingTimeout": "00:10:00",