Archive meeting workflow and screenshot changes

This commit is contained in:
2026-05-27 12:55:19 +02:00
parent 2422236ef7
commit 12832dde84
48 changed files with 3041 additions and 43 deletions
+63 -13
View File
@@ -13,7 +13,7 @@ public sealed class GlobalHotkeyService : BackgroundService
private readonly ILaunchProfileOptionsProvider launchProfiles;
private readonly MeetingRecordingCoordinator coordinator;
private readonly ILogger<GlobalHotkeyService> logger;
private readonly Dictionary<int, string> hotkeyProfiles = [];
private readonly Dictionary<int, LaunchProfileHotkey> registeredHotkeys = [];
private TaskCompletionSource? messageLoopCompletion;
private uint messageThreadId;
@@ -66,42 +66,44 @@ public sealed class GlobalHotkeyService : BackgroundService
for (var index = 0; index < hotkeys.Count; index++)
{
var profileHotkey = hotkeys[index];
var hotkey = HotkeyDefinition.Parse(profileHotkey.Toggle);
var hotkey = HotkeyDefinition.Parse(profileHotkey.Hotkey);
var hotkeyId = HotkeyIdBase + index;
if (!RegisterHotKey(IntPtr.Zero, hotkeyId, (uint)hotkey.Modifiers, hotkey.VirtualKey))
{
logger.LogWarning(
"Could not register global hotkey {Hotkey} for launch profile {LaunchProfile}",
profileHotkey.Toggle,
profileHotkey.ProfileName);
"Could not register global hotkey {Hotkey} for launch profile {LaunchProfile} action {Action}",
profileHotkey.Hotkey,
profileHotkey.ProfileName,
profileHotkey.Action);
continue;
}
hotkeyProfiles[hotkeyId] = profileHotkey.ProfileName;
registeredHotkeys[hotkeyId] = profileHotkey;
logger.LogInformation(
"Registered global hotkey {Hotkey} for launch profile {LaunchProfile}",
profileHotkey.Toggle,
profileHotkey.ProfileName);
"Registered global hotkey {Hotkey} for launch profile {LaunchProfile} action {Action}",
profileHotkey.Hotkey,
profileHotkey.ProfileName,
profileHotkey.Action);
}
try
{
while (!stoppingToken.IsCancellationRequested && GetMessage(out var message, IntPtr.Zero, 0, 0) > 0)
{
if (message.Message == WmHotkey && hotkeyProfiles.TryGetValue((int)message.WParam, out var profileName))
if (message.Message == WmHotkey && registeredHotkeys.TryGetValue((int)message.WParam, out var hotkey))
{
_ = Task.Run(() => ToggleRecordingAsync(profileName), CancellationToken.None);
_ = Task.Run(() => HandleHotkeyAsync(hotkey), CancellationToken.None);
}
}
}
finally
{
foreach (var hotkeyId in hotkeyProfiles.Keys)
foreach (var hotkeyId in registeredHotkeys.Keys)
{
UnregisterHotKey(IntPtr.Zero, hotkeyId);
}
hotkeyProfiles.Clear();
registeredHotkeys.Clear();
messageThreadId = 0;
}
}
@@ -112,6 +114,22 @@ public sealed class GlobalHotkeyService : BackgroundService
return base.StopAsync(cancellationToken);
}
private async Task HandleHotkeyAsync(LaunchProfileHotkey hotkey)
{
switch (hotkey.Action)
{
case LaunchProfileHotkeyAction.ToggleRecording:
await ToggleRecordingAsync(hotkey.ProfileName);
break;
case LaunchProfileHotkeyAction.AbortRecording:
await AbortRecordingAsync(hotkey.ProfileName);
break;
case LaunchProfileHotkeyAction.CaptureScreenshot:
await CaptureScreenshotAsync(hotkey.ProfileName);
break;
}
}
private async Task ToggleRecordingAsync(string profileName)
{
try
@@ -128,6 +146,38 @@ public sealed class GlobalHotkeyService : BackgroundService
}
}
private async Task AbortRecordingAsync(string profileName)
{
try
{
var status = await coordinator.AbortAsync(CancellationToken.None);
logger.LogInformation(
"Hotkey aborted recording for launch profile {LaunchProfile}. IsRecording={IsRecording}",
profileName,
status.IsRecording);
}
catch (Exception exception)
{
logger.LogError(exception, "Hotkey abort failed");
}
}
private async Task CaptureScreenshotAsync(string profileName)
{
try
{
var result = await coordinator.CaptureScreenshotAsync(profileName, CancellationToken.None);
logger.LogInformation(
"Hotkey captured screenshot for launch profile {LaunchProfile}: {ScreenshotPath}",
profileName,
result.ScreenshotPath);
}
catch (Exception exception)
{
logger.LogError(exception, "Hotkey screenshot capture failed");
}
}
private void PostQuitToMessageThread()
{
var threadId = messageThreadId;
@@ -11,11 +11,27 @@ public interface ILaunchProfileOptionsProvider
public sealed record LaunchProfile(string Name, MeetingAssistantOptions Options);
public sealed record LaunchProfileHotkey(string ProfileName, string Toggle);
public enum LaunchProfileHotkeyAction
{
ToggleRecording,
AbortRecording,
CaptureScreenshot
}
public sealed record LaunchProfileHotkey(
string ProfileName,
string Hotkey,
LaunchProfileHotkeyAction Action);
public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOptionsProvider
{
public const string DefaultProfileName = "default";
private static readonly HotkeyDescriptor[] HotkeyDescriptors =
[
new("Hotkey:Toggle", options => options.Hotkey.Toggle, LaunchProfileHotkeyAction.ToggleRecording),
new("Hotkey:Abort", options => options.Hotkey.Abort, LaunchProfileHotkeyAction.AbortRecording),
new("Screenshots:Hotkey", options => options.Screenshots.Hotkey, LaunchProfileHotkeyAction.CaptureScreenshot)
];
private readonly IConfiguration configuration;
@@ -47,25 +63,47 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
public IReadOnlyList<LaunchProfileHotkey> GetHotkeys()
{
var hotkeys = GetProfileNames()
.Select(profileName =>
{
var profile = GetRequiredProfile(profileName);
return new LaunchProfileHotkey(profile.Name, profile.Options.Hotkey.Toggle);
})
.Where(hotkey => !string.IsNullOrWhiteSpace(hotkey.Toggle))
.SelectMany(CreateHotkeys)
.ToList();
var duplicate = hotkeys
.GroupBy(hotkey => hotkey.Toggle.Trim(), StringComparer.OrdinalIgnoreCase)
.GroupBy(hotkey => hotkey.Hotkey.Trim(), StringComparer.OrdinalIgnoreCase)
.FirstOrDefault(group => group.Count() > 1);
if (duplicate is not null)
{
throw new InvalidOperationException(
$"Launch profile hotkey '{duplicate.Key}' is configured more than once for profiles {string.Join(", ", duplicate.Select(hotkey => hotkey.ProfileName))}.");
$"Launch profile hotkey '{duplicate.Key}' is configured more than once for profiles {string.Join(", ", duplicate.Select(hotkey => $"{hotkey.ProfileName}:{hotkey.Action}"))}.");
}
return hotkeys;
}
private IEnumerable<LaunchProfileHotkey> CreateHotkeys(string profileName)
{
var profile = GetRequiredProfile(profileName);
var isDefaultProfile = profileName.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase);
var profileSection = isDefaultProfile
? null
: GetProfilesSection().GetSection(profileName);
foreach (var descriptor in HotkeyDescriptors)
{
var hotkey = descriptor.GetValue(profile.Options);
if (ShouldRegisterHotkey(isDefaultProfile, profileSection?.GetSection(descriptor.SectionPath)) &&
!string.IsNullOrWhiteSpace(hotkey))
{
yield return new LaunchProfileHotkey(
profile.Name,
hotkey,
descriptor.Action);
}
}
}
private static bool ShouldRegisterHotkey(bool isDefaultProfile, IConfigurationSection? profileHotkeySection)
{
return isDefaultProfile || profileHotkeySection?.Exists() == true;
}
private IReadOnlyList<string> GetProfileNames()
{
return new[] { DefaultProfileName }
@@ -150,4 +188,9 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
.Select(value => value!.Value)
.ToArray();
}
private sealed record HotkeyDescriptor(
string SectionPath,
Func<MeetingAssistantOptions, string> GetValue,
LaunchProfileHotkeyAction Action);
}
+3
View File
@@ -4,6 +4,7 @@
<TargetFrameworks>net10.0;net10.0-windows</TargetFrameworks>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<PreserveCompilationContext>true</PreserveCompilationContext>
</PropertyGroup>
<ItemGroup>
@@ -18,6 +19,7 @@
<PackageReference Include="NAudio" Version="2.3.0" />
<PackageReference Include="NCalcSync" Version="5.12.0" />
<PackageReference Include="RazorLight" Version="2.3.1" />
<PackageReference Include="System.Drawing.Common" Version="10.0.0" />
<PackageReference Include="Whisper.net" Version="1.9.0" />
<PackageReference Include="Whisper.net.Runtime" Version="1.9.0" />
<PackageReference Include="YamlDotNet" Version="17.1.0" />
@@ -27,6 +29,7 @@
<Compile Remove="Hotkeys\GlobalHotkeyService.cs" />
<Compile Remove="Recording\NaudioCaptureSource.cs" />
<Compile Remove="MeetingNotes\OutlookClassicMeetingMetadataProvider.Windows.cs" />
<Compile Remove="Screenshots\ActiveWindowScreenshotCapture.Windows.cs" />
</ItemGroup>
</Project>
@@ -18,6 +18,8 @@ public sealed class MeetingAssistantOptions
public AutomationOptions Automation { get; set; } = new();
public ScreenshotOptions Screenshots { get; set; } = new();
public AgentOptions Agent { get; set; } = new();
public ApiOptions Api { get; set; } = new();
@@ -28,9 +30,60 @@ public sealed class AutomationOptions
public string? RulesPath { get; set; } = "meeting-rules.local.yaml";
}
public sealed class ScreenshotOptions
{
public string Hotkey { get; set; } = "Ctrl+Alt+S";
public string AttachmentsFolder { get; set; } = "Attachments";
public ScreenshotOcrOptions Ocr { get; set; } = new();
}
public sealed class ScreenshotOcrOptions
{
public const string DefaultPrompt = """
This screenshot was captured during a meeting. Extract information that is useful for meeting notes.
Identify who appears to be talking, who appears to be presenting, and what is being presented when visible.
If the screenshot contains slides, capture the visible text in clean markdown.
If the screenshot contains a diagram, convert it to Mermaid when possible and preserve labels accurately.
If it contains another kind of shared screen or scene, describe the scene and the relevant visible information.
If you can isolate the actual presentation, shared screen, or similarly relevant meeting content, return crop coordinates for only that content.
Crop coordinates must be pixel coordinates relative to the original screenshot: x, y, width, height.
Do not return crop coordinates for a person, webcam tile, whole app chrome, or content you cannot isolate confidently.
End your response with exactly one fenced json block containing meeting-assistant metadata, for example:
```json
{ "crop": { "x": 0, "y": 0, "width": 1920, "height": 1080 } }
```
Use `{ "crop": null }` when no confident presentation/shared-screen crop exists.
Do not invent information that is not visible in the screenshot.
""";
public bool Enabled { get; set; }
public string? Endpoint { get; set; }
public string? Key { get; set; }
public string KeyEnv { get; set; } = "LITELLM_API_KEY";
public string? Model { get; set; }
public string Prompt { get; set; } = DefaultPrompt;
public TimeSpan Timeout { get; set; } = TimeSpan.FromMinutes(2);
public TimeSpan GetEffectiveTimeout()
{
return Timeout > TimeSpan.Zero ? Timeout : TimeSpan.FromMinutes(2);
}
}
public sealed class HotkeyOptions
{
public string Toggle { get; set; } = "Ctrl+Alt+M";
public string Abort { get; set; } = "Ctrl+Alt+D";
}
public sealed class VaultOptions
@@ -64,6 +117,8 @@ public sealed class RecordingOptions
public TimeSpan BackgroundMetadataLookupTimeout { get; set; } = TimeSpan.FromMinutes(1);
public int MaxMetadataAttendeeImportCount { get; set; } = 30;
public TimeSpan OpenMeetingNoteDelay { get; set; } = TimeSpan.FromSeconds(1);
public string TemporaryRecordingsFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Recordings";
+16
View File
@@ -3,6 +3,7 @@ using MeetingAssistant.Hotkeys;
using MeetingAssistant.LaunchProfiles;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Recording;
using MeetingAssistant.Screenshots;
using MeetingAssistant.Speakers;
using MeetingAssistant.Summary;
using MeetingAssistant.Transcription;
@@ -29,8 +30,16 @@ builder.Services.AddSingleton<ITranscriptStore, VaultTranscriptStore>();
builder.Services.AddSingleton<IMeetingNoteStore, MarkdownMeetingNoteStore>();
builder.Services.AddSingleton<IMeetingNoteOpener, ObsidianMeetingNoteOpener>();
builder.Services.AddSingleton<IMeetingArtifactStore, MarkdownMeetingArtifactStore>();
builder.Services.AddSingleton<IMeetingRunArtifactCleaner, MeetingRunArtifactCleaner>();
builder.Services.AddSingleton<IRecordedAudioStore, TemporaryRecordedAudioStore>();
builder.Services.AddSingleton<IDictationWordStore, MarkdownDictationWordStore>();
#if WINDOWS
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, ActiveWindowScreenshotCapture>();
#else
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, UnavailableActiveWindowScreenshotCapture>();
#endif
builder.Services.AddSingleton<IScreenshotOcrClient, LiteLlmScreenshotOcrClient>();
builder.Services.AddSingleton<IMeetingScreenshotService, MeetingScreenshotService>();
builder.Services.AddDbContextFactory<SpeakerIdentityDbContext>((services, dbOptions) =>
{
var appOptions = services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value;
@@ -236,6 +245,13 @@ app.MapPost("/profiles/{launchProfile}/recording/stop", async (
MeetingRecordingCoordinator coordinator,
CancellationToken cancellationToken) =>
Results.Ok(await coordinator.StopAsync(cancellationToken)));
app.MapPost("/recording/abort", async (MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) =>
Results.Ok(await coordinator.AbortAsync(cancellationToken)));
app.MapPost("/profiles/{launchProfile}/recording/abort", async (
string launchProfile,
MeetingRecordingCoordinator coordinator,
CancellationToken cancellationToken) =>
Results.Ok(await coordinator.AbortAsync(cancellationToken)));
app.MapPost("/meetings/current/summary/run", async (
MeetingRecordingCoordinator coordinator,
IMeetingSummaryPipeline summaryPipeline,
@@ -3,6 +3,7 @@ using System.Collections.Concurrent;
using MeetingAssistant.LaunchProfiles;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Speakers;
using MeetingAssistant.Screenshots;
using MeetingAssistant.Summary;
using MeetingAssistant.Transcription;
using MeetingAssistant.Workflow;
@@ -26,6 +27,8 @@ public sealed class MeetingRecordingCoordinator
private readonly IDictationWordStore dictationWordStore;
private readonly ILaunchProfileOptionsProvider? launchProfiles;
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
private readonly IMeetingScreenshotService screenshotService;
private readonly IMeetingRunArtifactCleaner artifactCleaner;
private readonly MeetingAssistantOptions options;
private readonly ILogger<MeetingRecordingCoordinator> logger;
private readonly SemaphoreSlim gate = new(1, 1);
@@ -50,7 +53,9 @@ public sealed class MeetingRecordingCoordinator
IDictationWordStore? dictationWordStore = null,
ISpeakerIdentityAttendeeCanonicalizer? attendeeCanonicalizer = null,
ILaunchProfileOptionsProvider? launchProfiles = null,
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
IMeetingWorkflowEngine? meetingWorkflowEngine = null,
IMeetingScreenshotService? screenshotService = null,
IMeetingRunArtifactCleaner? artifactCleaner = null)
{
this.audioSource = audioSource;
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
@@ -66,6 +71,8 @@ public sealed class MeetingRecordingCoordinator
this.attendeeCanonicalizer = attendeeCanonicalizer ?? PassthroughSpeakerIdentityAttendeeCanonicalizer.Instance;
this.launchProfiles = launchProfiles;
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
this.screenshotService = screenshotService ?? NoopMeetingScreenshotService.Instance;
this.artifactCleaner = artifactCleaner ?? new MeetingRunArtifactCleaner();
this.options = options.Value;
this.logger = logger;
}
@@ -230,6 +237,87 @@ public sealed class MeetingRecordingCoordinator
return CurrentStatus;
}
public async Task<RecordingStatus> AbortAsync(CancellationToken cancellationToken)
{
RecordingRun run;
MeetingSessionArtifacts artifacts;
await gate.WaitAsync(cancellationToken);
try
{
if (currentRun is null)
{
return CurrentStatus;
}
run = currentRun;
artifacts = run.Artifacts;
run.Abort();
}
finally
{
gate.Release();
}
try
{
await run.Task.WaitAsync(run.Options.Recording.StopProcessingTimeout, cancellationToken);
}
catch (TimeoutException)
{
logger.LogWarning("Timed out while aborting meeting recording; forcing transcription cancellation");
run.CancelTranscription();
await run.Task.WaitAsync(cancellationToken);
}
await screenshotService.CancelPendingOcrAsync(
artifacts,
run.Options.Screenshots.Ocr.GetEffectiveTimeout(),
cancellationToken);
await artifactCleaner.DeleteRunArtifactsAsync(artifacts, run.Options, cancellationToken);
logger.LogInformation("Meeting recording aborted and artifacts deleted");
return CurrentStatus;
}
public Task<MeetingScreenshotCaptureResult> CaptureScreenshotAsync(CancellationToken cancellationToken)
{
return CaptureScreenshotAsync(null, cancellationToken);
}
public async Task<MeetingScreenshotCaptureResult> CaptureScreenshotAsync(
string? launchProfileName,
CancellationToken cancellationToken)
{
MeetingSessionArtifacts artifacts;
DateTimeOffset? meetingStartedAt;
MeetingAssistantOptions runOptions;
await gate.WaitAsync(cancellationToken);
try
{
if (currentRun is not { IsCaptureStopping: false, IsAborted: false } run ||
currentMeetingNote is null)
{
throw new InvalidOperationException("No active meeting is available for screenshot capture.");
}
artifacts = run.Artifacts;
meetingStartedAt = currentMeetingNote.Frontmatter.StartTime;
runOptions = run.Options;
}
finally
{
gate.Release();
}
return await screenshotService.CaptureAsync(
artifacts,
meetingStartedAt,
DateTimeOffset.Now,
runOptions,
cancellationToken);
}
private async Task RecordAsync(RecordingRun run)
{
await run.Pipeline.InitializeAsync(run.PipelineOptions, run.TranscriptionCancellation);
@@ -457,11 +545,21 @@ public sealed class MeetingRecordingCoordinator
return;
}
if (run.IsAborted)
{
return;
}
await gate.WaitAsync(CancellationToken.None);
try
{
if (run.IsAborted)
{
return;
}
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, CancellationToken.None);
await ApplyMeetingMetadataAsync(meetingNote, metadata, CancellationToken.None);
await ApplyMeetingMetadataAsync(meetingNote, metadata, run.Options, CancellationToken.None);
meetingNote = await meetingNoteStore.SaveAsync(meetingNote, run.Options, CancellationToken.None);
if (currentMeetingNote?.Path == meetingNote.Path)
{
@@ -494,10 +592,13 @@ public sealed class MeetingRecordingCoordinator
{
try
{
await TransitionMeetingAsync(
run,
AssistantContextState.Transcribing,
CancellationToken.None);
if (!run.IsAborted)
{
await TransitionMeetingAsync(
run,
AssistantContextState.Transcribing,
CancellationToken.None);
}
}
catch (Exception exception)
{
@@ -512,6 +613,7 @@ public sealed class MeetingRecordingCoordinator
private async Task ApplyMeetingMetadataAsync(
MeetingNote meetingNote,
MeetingMetadata metadata,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
if (!string.IsNullOrWhiteSpace(metadata.Title))
@@ -521,6 +623,16 @@ public sealed class MeetingRecordingCoordinator
if (metadata.Attendees.Count > 0)
{
var attendeeImportLimit = Math.Max(0, options.Recording.MaxMetadataAttendeeImportCount);
if (metadata.Attendees.Count > attendeeImportLimit)
{
logger.LogInformation(
"Skipped importing {AttendeeCount} metadata attendees because it exceeds the configured limit {AttendeeImportLimit}",
metadata.Attendees.Count,
attendeeImportLimit);
return;
}
meetingNote.Frontmatter.Attendees = await CanonicalizeAttendeesAsync(metadata.Attendees, cancellationToken);
}
}
@@ -835,8 +947,17 @@ public sealed class MeetingRecordingCoordinator
RecordingRun run,
CancellationToken cancellationToken)
{
if (run.IsAborted)
{
return;
}
try
{
await screenshotService.WaitForPendingOcrAsync(
run.Artifacts,
run.Options.Screenshots.Ocr.GetEffectiveTimeout(),
cancellationToken);
await TransitionMeetingAsync(
run,
AssistantContextState.Summarizing,
@@ -866,6 +987,11 @@ public sealed class MeetingRecordingCoordinator
AssistantContextState state,
CancellationToken cancellationToken)
{
if (run.IsAborted)
{
return;
}
if (!run.TryTransitionTo(state, out var fromState))
{
return;
@@ -1038,6 +1164,8 @@ public sealed class MeetingRecordingCoordinator
public bool IsCaptureStopping => CaptureCancellationSource.IsCancellationRequested;
public bool IsAborted { get; private set; }
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
public void StopCapture()
@@ -1045,6 +1173,14 @@ public sealed class MeetingRecordingCoordinator
CaptureCancellationSource.Cancel();
}
public void Abort()
{
IsAborted = true;
CaptureCancellationSource.Cancel();
TranscriptionCancellationSource.Cancel();
LiveIdentificationCancellationSource.Cancel();
}
public void CancelTranscription()
{
TranscriptionCancellationSource.Cancel();
@@ -0,0 +1,139 @@
using System.Text.RegularExpressions;
using MeetingAssistant.MeetingNotes;
namespace MeetingAssistant.Recording;
public interface IMeetingRunArtifactCleaner
{
Task DeleteRunArtifactsAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken);
}
public sealed class MeetingRunArtifactCleaner : IMeetingRunArtifactCleaner
{
private static readonly Regex MarkdownImageLinkPattern = new("!\\[[^\\]]*\\]\\((?<path>[^)]+)\\)", RegexOptions.Compiled);
private static readonly Regex GeneratedScreenshotFileNamePattern = new(
@"^\d{8}-\d{6}-\d{3}-\d{6}(?:-cropped)?\.png$",
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private readonly ILogger<MeetingRunArtifactCleaner>? logger;
public MeetingRunArtifactCleaner(ILogger<MeetingRunArtifactCleaner>? logger = null)
{
this.logger = logger;
}
public async Task DeleteRunArtifactsAsync(
MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var linkedFiles = await GetLinkedScreenshotAttachmentFilesAsync(
artifacts.AssistantContextPath,
options.Screenshots.AttachmentsFolder,
cancellationToken);
foreach (var linkedFile in linkedFiles)
{
await DeleteFileIfExistsAsync(linkedFile, cancellationToken);
}
await DeleteFileIfExistsAsync(artifacts.SummaryPath, cancellationToken);
await DeleteFileIfExistsAsync(artifacts.AssistantContextPath, cancellationToken);
await DeleteFileIfExistsAsync(artifacts.TranscriptPath, cancellationToken);
await DeleteFileIfExistsAsync(artifacts.MeetingNotePath, cancellationToken);
}
private static async Task<IReadOnlyList<string>> GetLinkedScreenshotAttachmentFilesAsync(
string assistantContextPath,
string configuredAttachmentsFolder,
CancellationToken cancellationToken)
{
if (!File.Exists(assistantContextPath))
{
return [];
}
var contextDirectory = Path.GetDirectoryName(Path.GetFullPath(assistantContextPath));
if (string.IsNullOrWhiteSpace(contextDirectory))
{
return [];
}
var content = await File.ReadAllTextAsync(assistantContextPath, cancellationToken);
var attachmentsFolder = ResolveAttachmentsFolder(contextDirectory, configuredAttachmentsFolder);
return MarkdownImageLinkPattern
.Matches(content)
.Select(match => match.Groups["path"].Value.Trim())
.Where(path => !string.IsNullOrWhiteSpace(path))
.Select(path => ResolveScreenshotAttachmentLink(contextDirectory, attachmentsFolder, path))
.Where(path => path is not null)
.Select(path => path!)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
}
private static string? ResolveScreenshotAttachmentLink(
string contextDirectory,
string attachmentsFolder,
string linkPath)
{
if (Uri.TryCreate(linkPath, UriKind.Absolute, out var uri) && !uri.IsFile)
{
return null;
}
var withoutFragment = linkPath.Split('#')[0].Split('?')[0];
var decoded = Uri.UnescapeDataString(withoutFragment);
var fullPath = Path.IsPathRooted(decoded)
? Path.GetFullPath(decoded)
: Path.GetFullPath(Path.Combine(contextDirectory, decoded));
var attachmentsRoot = Path.GetFullPath(attachmentsFolder).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) +
Path.DirectorySeparatorChar;
var fileName = Path.GetFileName(fullPath);
return fullPath.StartsWith(attachmentsRoot, StringComparison.OrdinalIgnoreCase) &&
GeneratedScreenshotFileNamePattern.IsMatch(fileName)
? fullPath
: null;
}
private static string ResolveAttachmentsFolder(string contextDirectory, string configuredFolder)
{
var expanded = Environment.ExpandEnvironmentVariables(
string.IsNullOrWhiteSpace(configuredFolder) ? "Attachments" : configuredFolder);
return Path.IsPathRooted(expanded)
? Path.GetFullPath(expanded)
: Path.GetFullPath(Path.Combine(contextDirectory, expanded));
}
private async Task DeleteFileIfExistsAsync(string path, CancellationToken cancellationToken)
{
var deadline = DateTimeOffset.UtcNow.AddSeconds(2);
while (true)
{
try
{
if (File.Exists(path))
{
File.Delete(path);
logger?.LogInformation("Deleted aborted meeting artifact {ArtifactPath}", path);
}
return;
}
catch (IOException) when (DateTimeOffset.UtcNow < deadline)
{
await Task.Delay(25, cancellationToken);
}
catch (UnauthorizedAccessException) when (DateTimeOffset.UtcNow < deadline)
{
await Task.Delay(25, cancellationToken);
}
catch (Exception exception)
{
logger?.LogWarning(exception, "Could not delete aborted meeting artifact {ArtifactPath}", path);
return;
}
}
}
}
@@ -0,0 +1,121 @@
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
namespace MeetingAssistant.Screenshots;
public sealed class ActiveWindowScreenshotCapture : IActiveWindowScreenshotCapture
{
private static readonly IntPtr DpiAwarenessContextPerMonitorAwareV2 = new(-4);
private const int DwmWindowAttributeExtendedFrameBounds = 9;
public Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken)
{
var previousDpiContext = TrySetThreadDpiAwarenessContext(DpiAwarenessContextPerMonitorAwareV2);
try
{
var foregroundWindow = GetForegroundWindow();
if (foregroundWindow == IntPtr.Zero ||
!TryGetWindowBounds(foregroundWindow, out var rect))
{
throw new InvalidOperationException("Could not determine the active window bounds.");
}
using var bitmap = new Bitmap(rect.Width, rect.Height);
using (var graphics = Graphics.FromImage(bitmap))
{
graphics.CopyFromScreen(rect.Left, rect.Top, 0, 0, new Size(rect.Width, rect.Height));
}
using var stream = new MemoryStream();
bitmap.Save(stream, ImageFormat.Png);
return Task.FromResult(new ActiveWindowScreenshot(stream.ToArray(), GetWindowTitle(foregroundWindow)));
}
finally
{
if (previousDpiContext != IntPtr.Zero)
{
TrySetThreadDpiAwarenessContext(previousDpiContext);
}
}
}
private static bool TryGetWindowBounds(IntPtr window, out NativeRect rect)
{
if (DwmGetWindowAttribute(
window,
DwmWindowAttributeExtendedFrameBounds,
out rect,
Marshal.SizeOf<NativeRect>()) == 0 &&
rect.Width > 0 &&
rect.Height > 0)
{
return true;
}
return GetWindowRect(window, out rect) &&
rect.Width > 0 &&
rect.Height > 0;
}
private static IntPtr TrySetThreadDpiAwarenessContext(IntPtr dpiContext)
{
try
{
return SetThreadDpiAwarenessContext(dpiContext);
}
catch (EntryPointNotFoundException)
{
return IntPtr.Zero;
}
}
private static string? GetWindowTitle(IntPtr window)
{
var length = GetWindowTextLength(window);
if (length <= 0)
{
return null;
}
var buffer = new System.Text.StringBuilder(length + 1);
return GetWindowText(window, buffer, buffer.Capacity) > 0
? buffer.ToString()
: null;
}
[DllImport("user32.dll")]
private static extern IntPtr GetForegroundWindow();
[DllImport("user32.dll", SetLastError = true)]
private static extern bool GetWindowRect(IntPtr hWnd, out NativeRect lpRect);
[DllImport("dwmapi.dll", PreserveSig = true)]
private static extern int DwmGetWindowAttribute(
IntPtr hwnd,
int dwAttribute,
out NativeRect pvAttribute,
int cbAttribute);
[DllImport("user32.dll")]
private static extern IntPtr SetThreadDpiAwarenessContext(IntPtr dpiContext);
[DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
private static extern int GetWindowText(IntPtr hWnd, System.Text.StringBuilder lpString, int nMaxCount);
[DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
private static extern int GetWindowTextLength(IntPtr hWnd);
[StructLayout(LayoutKind.Sequential)]
private readonly struct NativeRect
{
public readonly int Left;
public readonly int Top;
public readonly int Right;
public readonly int Bottom;
public int Width => Right - Left;
public int Height => Bottom - Top;
}
}
@@ -0,0 +1,265 @@
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.RegularExpressions;
namespace MeetingAssistant.Screenshots;
public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
{
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
private readonly ILogger<LiteLlmScreenshotOcrClient> logger;
private readonly Func<HttpMessageHandler>? httpMessageHandlerFactory;
public LiteLlmScreenshotOcrClient(ILogger<LiteLlmScreenshotOcrClient> logger)
{
this.logger = logger;
}
internal LiteLlmScreenshotOcrClient(
Func<HttpMessageHandler> httpMessageHandlerFactory,
ILogger<LiteLlmScreenshotOcrClient> logger)
{
this.httpMessageHandlerFactory = httpMessageHandlerFactory;
this.logger = logger;
}
public async Task<ScreenshotOcrResult> ExtractAsync(
string screenshotPath,
string prompt,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var endpoint = !string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Endpoint)
? options.Screenshots.Ocr.Endpoint
: options.Agent.Endpoint;
var model = !string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Model)
? options.Screenshots.Ocr.Model
: options.Agent.Model;
var key = ResolveApiKey(options);
var imageBytes = await File.ReadAllBytesAsync(screenshotPath, cancellationToken);
using var httpClient = CreateHttpClient();
httpClient.BaseAddress = NormalizeEndpoint(new Uri(endpoint));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", key);
var payload = CreatePayload(model, CreatePrompt(prompt, imageBytes), imageBytes);
using var content = new StringContent(payload.ToJsonString(JsonOptions), Encoding.UTF8, "application/json");
using var response = await httpClient.PostAsync("responses", content, cancellationToken);
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken);
if (!response.IsSuccessStatusCode)
{
throw new InvalidOperationException(
$"Screenshot OCR request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}");
}
var text = ParseOutputText(responseJson);
logger.LogInformation("Screenshot OCR completed for {ScreenshotPath}", screenshotPath);
return ParseOcrResult(text);
}
private HttpClient CreateHttpClient()
{
return httpMessageHandlerFactory is null
? new HttpClient()
: new HttpClient(httpMessageHandlerFactory());
}
private static JsonObject CreatePayload(string model, string prompt, byte[] imageBytes)
{
return new JsonObject
{
["model"] = model,
["store"] = false,
["input"] = new JsonArray
{
new JsonObject
{
["role"] = "user",
["content"] = new JsonArray
{
new JsonObject
{
["type"] = "input_text",
["text"] = prompt
},
new JsonObject
{
["type"] = "input_image",
["image_url"] = "data:image/png;base64," + Convert.ToBase64String(imageBytes)
}
}
}
}
};
}
private static string CreatePrompt(string prompt, byte[] imageBytes)
{
return TryReadPngDimensions(imageBytes, out var width, out var height)
? prompt + Environment.NewLine + Environment.NewLine +
$"Original screenshot dimensions: {width}x{height} pixels. Crop coordinates must fit within these bounds."
: prompt;
}
private static ScreenshotOcrResult ParseOcrResult(string text)
{
ScreenshotCropCoordinates? crop = null;
var cleaned = JsonCodeBlockRegex().Replace(text, match =>
{
if (TryParseCrop(match.Groups["json"].Value, out var parsedCrop))
{
crop = parsedCrop;
return "";
}
return match.Value;
}).Trim();
return new ScreenshotOcrResult(cleaned, crop);
}
private static bool TryParseCrop(string json, out ScreenshotCropCoordinates? crop)
{
crop = null;
try
{
using var document = JsonDocument.Parse(json);
if (!document.RootElement.TryGetProperty("crop", out var cropElement) ||
cropElement.ValueKind == JsonValueKind.Null)
{
return true;
}
if (cropElement.ValueKind != JsonValueKind.Object ||
!TryGetInt(cropElement, "x", out var x) ||
!TryGetInt(cropElement, "y", out var y) ||
!TryGetInt(cropElement, "width", out var width) ||
!TryGetInt(cropElement, "height", out var height))
{
return false;
}
crop = new ScreenshotCropCoordinates(x, y, width, height);
return true;
}
catch (JsonException)
{
return false;
}
}
private static bool TryGetInt(JsonElement element, string propertyName, out int value)
{
value = 0;
return element.TryGetProperty(propertyName, out var property) &&
property.ValueKind == JsonValueKind.Number &&
property.TryGetInt32(out value);
}
private static bool TryReadPngDimensions(byte[] bytes, out int width, out int height)
{
width = 0;
height = 0;
if (bytes.Length < 24 ||
bytes[0] != 0x89 ||
bytes[1] != 0x50 ||
bytes[2] != 0x4E ||
bytes[3] != 0x47)
{
return false;
}
width = ReadBigEndianInt32(bytes, 16);
height = ReadBigEndianInt32(bytes, 20);
return width > 0 && height > 0;
}
private static int ReadBigEndianInt32(byte[] bytes, int offset)
{
return (bytes[offset] << 24) |
(bytes[offset + 1] << 16) |
(bytes[offset + 2] << 8) |
bytes[offset + 3];
}
private static string ParseOutputText(string responseJson)
{
using var document = JsonDocument.Parse(responseJson);
var root = document.RootElement;
var parts = new List<string>();
if (root.TryGetProperty("output_text", out var outputText) &&
outputText.ValueKind == JsonValueKind.String &&
!string.IsNullOrWhiteSpace(outputText.GetString()))
{
parts.Add(outputText.GetString()!);
}
if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array)
{
foreach (var item in output.EnumerateArray())
{
if (!item.TryGetProperty("content", out var content) || content.ValueKind != JsonValueKind.Array)
{
continue;
}
foreach (var block in content.EnumerateArray())
{
if (block.TryGetProperty("type", out var type) &&
type.GetString() == "output_text" &&
block.TryGetProperty("text", out var text) &&
text.ValueKind == JsonValueKind.String &&
!string.IsNullOrWhiteSpace(text.GetString()))
{
parts.Add(text.GetString()!);
}
}
}
}
return parts.Count == 0
? ""
: string.Join(Environment.NewLine + Environment.NewLine, parts);
}
private static string ResolveApiKey(MeetingAssistantOptions options)
{
if (!string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Key))
{
return options.Screenshots.Ocr.Key;
}
var ocrKey = Environment.GetEnvironmentVariable(options.Screenshots.Ocr.KeyEnv);
if (!string.IsNullOrWhiteSpace(ocrKey))
{
return ocrKey;
}
if (!string.IsNullOrWhiteSpace(options.Agent.Key))
{
return options.Agent.Key;
}
var agentKey = Environment.GetEnvironmentVariable(options.Agent.KeyEnv);
if (!string.IsNullOrWhiteSpace(agentKey))
{
return agentKey;
}
throw new InvalidOperationException(
$"No screenshot OCR API key configured. Set MeetingAssistant:Screenshots:Ocr:Key or environment variable '{options.Screenshots.Ocr.KeyEnv}'.");
}
private static Uri NormalizeEndpoint(Uri endpoint)
{
var value = endpoint.ToString().TrimEnd('/');
if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase))
{
value += "/v1";
}
return new Uri(value + "/");
}
[GeneratedRegex("```json\\s*(?<json>.*?)\\s*```", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
private static partial Regex JsonCodeBlockRegex();
}
@@ -0,0 +1,482 @@
using System.Collections.Concurrent;
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using MeetingAssistant.MeetingNotes;
namespace MeetingAssistant.Screenshots;
public interface IActiveWindowScreenshotCapture
{
Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken);
}
public sealed record ActiveWindowScreenshot(byte[] PngBytes, string? WindowTitle);
public interface IScreenshotOcrClient
{
Task<ScreenshotOcrResult> ExtractAsync(
string screenshotPath,
string prompt,
MeetingAssistantOptions options,
CancellationToken cancellationToken);
}
public sealed record ScreenshotOcrResult(string Text, ScreenshotCropCoordinates? Crop);
public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height);
public interface IMeetingScreenshotService
{
Task<MeetingScreenshotCaptureResult> CaptureAsync(
MeetingSessionArtifacts artifacts,
DateTimeOffset? meetingStartedAt,
DateTimeOffset capturedAt,
MeetingAssistantOptions options,
CancellationToken cancellationToken);
Task WaitForPendingOcrAsync(
MeetingSessionArtifacts artifacts,
TimeSpan timeout,
CancellationToken cancellationToken);
Task CancelPendingOcrAsync(
MeetingSessionArtifacts artifacts,
TimeSpan timeout,
CancellationToken cancellationToken);
}
public sealed record MeetingScreenshotCaptureResult(
string ScreenshotPath,
TimeSpan MeetingTimestamp,
bool OcrStarted);
public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
{
public static NoopMeetingScreenshotService Instance { get; } = new();
public Task<MeetingScreenshotCaptureResult> CaptureAsync(
MeetingSessionArtifacts artifacts,
DateTimeOffset? meetingStartedAt,
DateTimeOffset capturedAt,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return Task.FromResult(new MeetingScreenshotCaptureResult("", TimeSpan.Zero, OcrStarted: false));
}
public Task WaitForPendingOcrAsync(
MeetingSessionArtifacts artifacts,
TimeSpan timeout,
CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
public Task CancelPendingOcrAsync(
MeetingSessionArtifacts artifacts,
TimeSpan timeout,
CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
public sealed class MeetingScreenshotService : IMeetingScreenshotService
{
private readonly IActiveWindowScreenshotCapture screenshotCapture;
private readonly IMeetingArtifactStore artifactStore;
private readonly IScreenshotOcrClient ocrClient;
private readonly ILogger<MeetingScreenshotService> logger;
private readonly ConcurrentDictionary<string, List<PendingOcrTask>> pendingOcrByContext = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, SemaphoreSlim> contextFileLocks = new(StringComparer.OrdinalIgnoreCase);
public MeetingScreenshotService(
IActiveWindowScreenshotCapture screenshotCapture,
IMeetingArtifactStore artifactStore,
IScreenshotOcrClient ocrClient,
ILogger<MeetingScreenshotService> logger)
{
this.screenshotCapture = screenshotCapture;
this.artifactStore = artifactStore;
this.ocrClient = ocrClient;
this.logger = logger;
}
public async Task<MeetingScreenshotCaptureResult> CaptureAsync(
MeetingSessionArtifacts artifacts,
DateTimeOffset? meetingStartedAt,
DateTimeOffset capturedAt,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var screenshot = await screenshotCapture.CaptureAsync(cancellationToken);
var timestamp = CalculateMeetingTimestamp(meetingStartedAt, capturedAt);
var screenshotPath = await SaveScreenshotAsync(
artifacts,
screenshot.PngBytes,
timestamp,
capturedAt,
options,
cancellationToken);
var relativePath = ToMarkdownPath(Path.GetRelativePath(
Path.GetDirectoryName(artifacts.AssistantContextPath)!,
screenshotPath));
var screenshotId = Guid.NewGuid().ToString("N");
var ocrEnabled = options.Screenshots.Ocr.Enabled;
await artifactStore.AppendAssistantContextAsync(
artifacts,
CreateScreenshotMarkdown(screenshotId, timestamp, relativePath, screenshot.WindowTitle, ocrEnabled),
cancellationToken);
if (ocrEnabled)
{
var ocrCancellation = new CancellationTokenSource();
TrackOcr(
artifacts,
ocrCancellation,
RunOcrAsync(artifacts, screenshotPath, screenshotId, options, ocrCancellation.Token));
}
logger.LogInformation(
"Captured meeting screenshot {ScreenshotPath} at {MeetingTimestamp}",
screenshotPath,
FormatTimestamp(timestamp));
return new MeetingScreenshotCaptureResult(screenshotPath, timestamp, ocrEnabled);
}
public async Task WaitForPendingOcrAsync(
MeetingSessionArtifacts artifacts,
TimeSpan timeout,
CancellationToken cancellationToken)
{
var contextKey = NormalizeContextKey(artifacts.AssistantContextPath);
if (!pendingOcrByContext.TryGetValue(contextKey, out var tasks))
{
return;
}
PendingOcrTask[] snapshot;
lock (tasks)
{
tasks.RemoveAll(task => task.Task.IsCompleted);
snapshot = tasks.ToArray();
}
if (snapshot.Length == 0)
{
return;
}
try
{
await Task.WhenAll(snapshot.Select(task => task.Task)).WaitAsync(timeout, cancellationToken);
}
catch (TimeoutException)
{
logger.LogWarning(
"Timed out waiting for {Count} screenshot OCR task(s) for {AssistantContextPath}",
snapshot.Length,
artifacts.AssistantContextPath);
}
}
public async Task CancelPendingOcrAsync(
MeetingSessionArtifacts artifacts,
TimeSpan timeout,
CancellationToken cancellationToken)
{
var contextKey = NormalizeContextKey(artifacts.AssistantContextPath);
if (!pendingOcrByContext.TryGetValue(contextKey, out var tasks))
{
return;
}
PendingOcrTask[] snapshot;
lock (tasks)
{
tasks.RemoveAll(task => task.Task.IsCompleted);
snapshot = tasks.ToArray();
}
if (snapshot.Length == 0)
{
return;
}
foreach (var pending in snapshot)
{
await pending.Cancellation.CancelAsync();
}
await WaitForPendingOcrAsync(artifacts, timeout, cancellationToken);
}
private async Task<string> SaveScreenshotAsync(
MeetingSessionArtifacts artifacts,
byte[] pngBytes,
TimeSpan timestamp,
DateTimeOffset capturedAt,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var folder = ResolveAttachmentsFolder(artifacts.AssistantContextPath, options.Screenshots.AttachmentsFolder);
Directory.CreateDirectory(folder);
var path = Path.Combine(
folder,
$"{capturedAt:yyyyMMdd-HHmmss-fff}-{FormatTimestamp(timestamp).Replace(":", "", StringComparison.Ordinal)}.png");
await File.WriteAllBytesAsync(path, pngBytes, cancellationToken);
return path;
}
private async Task RunOcrAsync(
MeetingSessionArtifacts artifacts,
string screenshotPath,
string screenshotId,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var timeout = options.Screenshots.Ocr.GetEffectiveTimeout();
using var timeoutSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeoutSource.CancelAfter(timeout);
try
{
var result = await ocrClient.ExtractAsync(
screenshotPath,
string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Prompt)
? ScreenshotOcrOptions.DefaultPrompt
: options.Screenshots.Ocr.Prompt,
options,
timeoutSource.Token);
var cropMarkdown = await TryCreateCropMarkdownAsync(
artifacts.AssistantContextPath,
screenshotPath,
result.Crop,
timeoutSource.Token);
await ReplaceOcrPlaceholderAsync(
artifacts.AssistantContextPath,
screenshotId,
cropMarkdown +
"### OCR" + Environment.NewLine + Environment.NewLine + result.Text.Trim(),
CancellationToken.None);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
logger.LogInformation("Cancelled screenshot OCR for {ScreenshotPath}", screenshotPath);
}
catch (OperationCanceledException)
{
await ReplaceOcrPlaceholderAsync(
artifacts.AssistantContextPath,
screenshotId,
$"_OCR timed out after {timeout:g}._",
CancellationToken.None);
}
catch (Exception exception)
{
logger.LogWarning(exception, "Screenshot OCR failed for {ScreenshotPath}", screenshotPath);
await ReplaceOcrPlaceholderAsync(
artifacts.AssistantContextPath,
screenshotId,
$"_OCR failed: {exception.Message}_",
CancellationToken.None);
}
}
private async Task ReplaceOcrPlaceholderAsync(
string assistantContextPath,
string screenshotId,
string replacement,
CancellationToken cancellationToken)
{
var fileLock = contextFileLocks.GetOrAdd(
NormalizeContextKey(assistantContextPath),
_ => new SemaphoreSlim(1, 1));
await fileLock.WaitAsync(cancellationToken);
try
{
if (!File.Exists(assistantContextPath))
{
return;
}
var content = await File.ReadAllTextAsync(assistantContextPath, cancellationToken);
var startMarker = $"<!-- screenshot-ocr:{screenshotId} -->";
var endMarker = $"<!-- /screenshot-ocr:{screenshotId} -->";
var startIndex = content.IndexOf(startMarker, StringComparison.Ordinal);
var endIndex = content.IndexOf(endMarker, StringComparison.Ordinal);
if (startIndex < 0 || endIndex < startIndex)
{
await File.AppendAllTextAsync(
assistantContextPath,
Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
cancellationToken);
return;
}
endIndex += endMarker.Length;
var updated = content[..startIndex] +
replacement.TrimEnd() +
content[endIndex..];
await File.WriteAllTextAsync(assistantContextPath, updated, cancellationToken);
}
finally
{
fileLock.Release();
}
}
private async Task<string> TryCreateCropMarkdownAsync(
string assistantContextPath,
string screenshotPath,
ScreenshotCropCoordinates? crop,
CancellationToken cancellationToken)
{
if (crop is null)
{
return "";
}
if (!OperatingSystem.IsWindows())
{
logger.LogWarning(
"Ignoring screenshot crop coordinates for {ScreenshotPath} because image cropping is only supported on Windows",
screenshotPath);
return "";
}
try
{
var croppedPath = await SaveCroppedScreenshotAsync(screenshotPath, crop, cancellationToken);
var relativePath = ToMarkdownPath(Path.GetRelativePath(
Path.GetDirectoryName(assistantContextPath)!,
croppedPath));
return $"![Cropped screenshot]({relativePath})" +
Environment.NewLine +
Environment.NewLine;
}
catch (Exception exception) when (exception is InvalidDataException or ArgumentException or ExternalException)
{
logger.LogWarning(
exception,
"Ignoring invalid screenshot crop coordinates for {ScreenshotPath}",
screenshotPath);
return "";
}
}
#pragma warning disable CA1416
private static Task<string> SaveCroppedScreenshotAsync(
string screenshotPath,
ScreenshotCropCoordinates crop,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
using var original = new Bitmap(screenshotPath);
ValidateCrop(crop, original.Width, original.Height);
var cropRectangle = new Rectangle(crop.X, crop.Y, crop.Width, crop.Height);
using var cropped = original.Clone(cropRectangle, original.PixelFormat);
var croppedPath = Path.Combine(
Path.GetDirectoryName(screenshotPath)!,
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
cropped.Save(croppedPath, ImageFormat.Png);
return Task.FromResult(croppedPath);
}
#pragma warning restore CA1416
private static void ValidateCrop(ScreenshotCropCoordinates crop, int imageWidth, int imageHeight)
{
if (crop.X < 0 ||
crop.Y < 0 ||
crop.Width <= 0 ||
crop.Height <= 0 ||
crop.X + crop.Width > imageWidth ||
crop.Y + crop.Height > imageHeight)
{
throw new InvalidDataException(
$"Screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} is outside image bounds {imageWidth}x{imageHeight}.");
}
}
private void TrackOcr(
MeetingSessionArtifacts artifacts,
CancellationTokenSource cancellation,
Task task)
{
var contextKey = NormalizeContextKey(artifacts.AssistantContextPath);
var tasks = pendingOcrByContext.GetOrAdd(contextKey, _ => []);
lock (tasks)
{
tasks.RemoveAll(existing => existing.Task.IsCompleted);
tasks.Add(new PendingOcrTask(task, cancellation));
}
}
private static string CreateScreenshotMarkdown(
string screenshotId,
TimeSpan timestamp,
string relativePath,
string? windowTitle,
bool ocrEnabled)
{
var title = string.IsNullOrWhiteSpace(windowTitle)
? ""
: Environment.NewLine + $"Window: {windowTitle.Trim()}" + Environment.NewLine;
var timestampText = FormatTimestamp(timestamp);
var content = $"## Screenshot [{timestampText}]" +
title +
Environment.NewLine +
$"![Screenshot {timestampText}]({relativePath})";
if (!ocrEnabled)
{
return content;
}
return content +
Environment.NewLine +
Environment.NewLine +
$"<!-- screenshot-ocr:{screenshotId} -->" +
Environment.NewLine +
"_OCR pending..._" +
Environment.NewLine +
$"<!-- /screenshot-ocr:{screenshotId} -->";
}
private static string ResolveAttachmentsFolder(string assistantContextPath, string configuredFolder)
{
var expanded = Environment.ExpandEnvironmentVariables(
string.IsNullOrWhiteSpace(configuredFolder) ? "Attachments" : configuredFolder);
return Path.IsPathRooted(expanded)
? Path.GetFullPath(expanded)
: Path.GetFullPath(Path.Combine(Path.GetDirectoryName(assistantContextPath)!, expanded));
}
private static TimeSpan CalculateMeetingTimestamp(DateTimeOffset? meetingStartedAt, DateTimeOffset capturedAt)
{
if (meetingStartedAt is null)
{
return TimeSpan.Zero;
}
var elapsed = capturedAt - meetingStartedAt.Value;
return elapsed < TimeSpan.Zero ? TimeSpan.Zero : elapsed;
}
private static string FormatTimestamp(TimeSpan timestamp)
{
return $"{(int)timestamp.TotalHours:00}:{timestamp.Minutes:00}:{timestamp.Seconds:00}";
}
private static string ToMarkdownPath(string path)
{
return path.Replace(Path.DirectorySeparatorChar, '/')
.Replace(Path.AltDirectorySeparatorChar, '/');
}
private static string NormalizeContextKey(string assistantContextPath)
{
return Path.GetFullPath(assistantContextPath);
}
private sealed record PendingOcrTask(Task Task, CancellationTokenSource Cancellation);
}
@@ -0,0 +1,9 @@
namespace MeetingAssistant.Screenshots;
public sealed class UnavailableActiveWindowScreenshotCapture : IActiveWindowScreenshotCapture
{
public Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken)
{
throw new PlatformNotSupportedException("Active-window screenshot capture is only available on Windows.");
}
}
@@ -17,6 +17,7 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
After writing the summary, update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context.
Use list_projects first to see which projects are bound to this meeting. Use search and read_projectfile before changing existing project files.
The summary note should contain concise sections for summary, decisions, open questions, and next steps.
If the assistant context contains cropped screenshot markdown links, include only the most relevant cropped screenshots in the summary by markdown-linking them near the related summary text. Do not include every cropped screenshot by default, and do not link uncropped screenshots unless no cropped version exists and the image is important.
Keep the output grounded in the source material and explicitly say when a section has no known items.
""";
@@ -130,7 +130,7 @@ public sealed class MeetingSummaryTools
? await File.ReadAllTextAsync(artifacts.AssistantContextPath)
: "";
var (frontmatter, body) = MeetingArtifactFrontmatterRenderer.Split(existing);
var updatedBody = ApplyLineEdit(body, content, editMode);
var updatedBody = ApplyLineEdit(body, StripAccidentalFrontmatter(content), editMode);
var updated = string.IsNullOrWhiteSpace(frontmatter)
? updatedBody
: "---" + Environment.NewLine + frontmatter + Environment.NewLine + "---" + Environment.NewLine + Environment.NewLine + updatedBody;
@@ -139,6 +139,12 @@ public sealed class MeetingSummaryTools
return artifacts.AssistantContextPath;
}
private static string StripAccidentalFrontmatter(string content)
{
var document = MarkdownDocumentParser.SplitOptional(content);
return document.HasFrontmatter ? document.Body : content;
}
public async Task<string> ListProjects()
{
var projects = await GetBoundProjectsAsync();
@@ -30,6 +30,10 @@ public sealed class NoopMeetingWorkflowEngine : IMeetingWorkflowEngine
public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
{
private static readonly Regex EmailAddressPattern = new(
@"(?<![A-Za-z0-9.!#$%&'*+/=?^_`{|}~-])(?<local>[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+)@(?<domain>(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,63})(?![A-Za-z0-9-])",
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly string[] ParameterNames =
[
"meeting.attendees.count",
@@ -252,17 +256,65 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
string template,
MeetingWorkflowTemplateModel model)
{
if (!template.Contains('@', StringComparison.Ordinal))
if (!ContainsRazorTemplateSyntax(template))
{
return template;
}
return await razorEngine.CompileRenderStringAsync(
Guid.NewGuid().ToString("N"),
template,
EscapeEmailAddressAtSigns(template),
model);
}
private static bool ContainsRazorTemplateSyntax(string value)
{
if (!value.Contains('@', StringComparison.Ordinal))
{
return false;
}
var emailAtSigns = EmailAddressPattern
.Matches(value)
.Select(match => match.Index + match.Value.IndexOf('@', StringComparison.Ordinal))
.ToHashSet();
for (var index = 0; index < value.Length; index++)
{
if (value[index] != '@')
{
continue;
}
if (index + 1 < value.Length && value[index + 1] == '@')
{
index++;
continue;
}
if (emailAtSigns.Contains(index))
{
continue;
}
return true;
}
return false;
}
private static string EscapeEmailAddressAtSigns(string value)
{
if (!value.Contains('@', StringComparison.Ordinal))
{
return value;
}
return EmailAddressPattern.Replace(
value,
match => match.Value.Replace("@", "@@", StringComparison.Ordinal));
}
private static bool SetProperty(
MeetingNote meeting,
string? property,
+16 -4
View File
@@ -1,7 +1,8 @@
{
"MeetingAssistant": {
"Hotkey": {
"Toggle": "Ctrl+Alt+M"
"Toggle": "Ctrl+Alt+M",
"Abort": "Ctrl+Alt+D"
},
"Vault": {
"BaseFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Exxeta",
@@ -19,6 +20,7 @@
"StopProcessingTimeout": "00:10:00",
"MetadataLookupTimeout": "00:00:10",
"BackgroundMetadataLookupTimeout": "00:01:00",
"MaxMetadataAttendeeImportCount": 30,
"OpenMeetingNoteDelay": "00:00:01",
"TemporaryRecordingsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Recordings"
},
@@ -107,7 +109,18 @@
"MatchTimeout": "00:03:00"
},
"Automation": {
"RulesPath": "meeting-rules.local.yaml"
"RulesPath": "C:\\Manuel\\meeting-assistant\\meeting-rules.local.yaml"
},
"Screenshots": {
"Hotkey": "Ctrl+Alt+S",
"AttachmentsFolder": "Attachments",
"Ocr": {
"Enabled": true,
"Endpoint": "",
"KeyEnv": "LITELLM_API_KEY",
"Model": "",
"Timeout": "00:02:00"
}
},
"Agent": {
"Endpoint": "https://litellm.schweigert.cloud",
@@ -121,8 +134,7 @@
"MaxOutputTokens": 8192,
"EnableCompaction": true,
"CompactionRemainingRatio": 0.1,
"ResponsesCompactPath": "responses/compact",
"InitialPrompt": "You are the Meeting Assistant summary agent.\n\nUse the provided tools to read the meeting transcript, assistant context, user notes, glossary, and project files.\nAll read tools can return the whole file or a clamped inclusive line range when from and to are supplied; use ranges for large inputs before asking for more lines.\nThen write the finished meeting summary as markdown by calling write_summary. If the meeting note has no title, provide a concise title parameter to write_summary.\nUse read_meetingnote to inspect frontmatter such as title, attendees, projects, start_time, and end_time.\nUse read_context and write_context as your own meeting notebook. Its frontmatter may include agenda from the calendar appointment. Record useful internal notes, missing context, requests for future tools, suggested improvements, and relevant context discovered from other sources. Keep user-facing summary content in the summary note.\nUse add_dictation_word when project context, user notes, or transcript evidence show that a domain term, acronym, name, or unusual word is likely to be repeatedly mistranscribed. Add only the canonical spelling, one term at a time.\nAfter writing the summary, update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context.\nUse list_projects first to see which projects are bound to this meeting. Use search and read_projectfile before changing existing project files.\nThe summary note should contain concise sections for summary, decisions, open questions, and next steps.\nKeep the output grounded in the source material and explicitly say when a section has no known items."
"ResponsesCompactPath": "responses/compact"
},
"Api": {
"PublicBaseUrl": "http://localhost:5090"