Public Access
976 lines
33 KiB
C#
976 lines
33 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Drawing;
|
|
using System.Drawing.Imaging;
|
|
using System.Runtime.InteropServices;
|
|
using System.Text.RegularExpressions;
|
|
using MeetingAssistant.MeetingNotes;
|
|
using MeetingAssistant.Speakers;
|
|
using MeetingAssistant.Workflow;
|
|
|
|
namespace MeetingAssistant.Screenshots;
|
|
|
|
public interface IActiveWindowScreenshotCapture
|
|
{
|
|
Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken);
|
|
}
|
|
|
|
public sealed record ActiveWindowScreenshot(byte[] PngBytes, string? WindowTitle);
|
|
|
|
public interface IScreenshotOcrClient
|
|
{
|
|
Task<ScreenshotOcrResult> ExtractAsync(
|
|
string screenshotPath,
|
|
string prompt,
|
|
MeetingAssistantOptions options,
|
|
CancellationToken cancellationToken);
|
|
}
|
|
|
|
public sealed record ScreenshotOcrResult
|
|
{
|
|
public ScreenshotOcrResult(
|
|
string text,
|
|
ScreenshotCropCoordinates? crop,
|
|
IReadOnlyList<string>? attendees = null)
|
|
{
|
|
Text = text;
|
|
Crop = crop;
|
|
Attendees = attendees ?? [];
|
|
}
|
|
|
|
public string Text { get; init; }
|
|
|
|
public ScreenshotCropCoordinates? Crop { get; init; }
|
|
|
|
public IReadOnlyList<string> Attendees { get; init; }
|
|
}
|
|
|
|
public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height);
|
|
|
|
public interface IMeetingScreenshotService
|
|
{
|
|
Task<MeetingScreenshotCaptureResult> CaptureAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
DateTimeOffset? meetingStartedAt,
|
|
DateTimeOffset capturedAt,
|
|
MeetingAssistantOptions options,
|
|
CancellationToken cancellationToken);
|
|
|
|
Task WaitForPendingOcrAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
TimeSpan timeout,
|
|
CancellationToken cancellationToken);
|
|
|
|
Task CancelPendingOcrAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
TimeSpan timeout,
|
|
CancellationToken cancellationToken);
|
|
}
|
|
|
|
public interface IMeetingNoteImageOcrService
|
|
{
|
|
Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
MeetingAssistantOptions options,
|
|
CancellationToken cancellationToken);
|
|
}
|
|
|
|
public interface IScreenshotOcrRetryRunner
|
|
{
|
|
Task<MeetingScreenshotOcrRetryTriggerResult?> TriggerOcrRetryAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
string screenshotPath,
|
|
string screenshotId,
|
|
MeetingAssistantOptions options,
|
|
CancellationToken cancellationToken);
|
|
}
|
|
|
|
public sealed record MeetingScreenshotCaptureResult(
|
|
string ScreenshotPath,
|
|
TimeSpan MeetingTimestamp,
|
|
bool OcrStarted);
|
|
|
|
public sealed record MeetingScreenshotOcrRetryTriggerResult(
|
|
string AssistantContextPath,
|
|
string ScreenshotPath,
|
|
string ScreenshotId);
|
|
|
|
public sealed record MeetingNoteImageOcrQueueResult(int QueuedCount);
|
|
|
|
public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
|
|
{
|
|
public static NoopMeetingScreenshotService Instance { get; } = new();
|
|
|
|
public Task<MeetingScreenshotCaptureResult> CaptureAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
DateTimeOffset? meetingStartedAt,
|
|
DateTimeOffset capturedAt,
|
|
MeetingAssistantOptions options,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
return Task.FromResult(new MeetingScreenshotCaptureResult("", TimeSpan.Zero, OcrStarted: false));
|
|
}
|
|
|
|
public Task WaitForPendingOcrAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
TimeSpan timeout,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public Task CancelPendingOcrAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
TimeSpan timeout,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
}
|
|
|
|
public sealed class NoopMeetingNoteImageOcrService : IMeetingNoteImageOcrService
|
|
{
|
|
public static NoopMeetingNoteImageOcrService Instance { get; } = new();
|
|
|
|
public Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
MeetingAssistantOptions options,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
return Task.FromResult(new MeetingNoteImageOcrQueueResult(0));
|
|
}
|
|
}
|
|
|
|
public sealed partial class MeetingScreenshotService :
|
|
IMeetingScreenshotService,
|
|
IScreenshotOcrRetryRunner,
|
|
IMeetingNoteImageOcrService
|
|
{
|
|
private readonly IActiveWindowScreenshotCapture screenshotCapture;
|
|
private readonly IMeetingArtifactStore artifactStore;
|
|
private readonly IMeetingNoteStore meetingNoteStore;
|
|
private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer;
|
|
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
|
|
private readonly IScreenshotOcrClient ocrClient;
|
|
private readonly ILogger<MeetingScreenshotService> logger;
|
|
private readonly ConcurrentDictionary<string, List<PendingOcrTask>> pendingOcrByContext = new(StringComparer.OrdinalIgnoreCase);
|
|
private readonly ConcurrentDictionary<string, SemaphoreSlim> contextFileLocks = new(StringComparer.OrdinalIgnoreCase);
|
|
|
|
public MeetingScreenshotService(
|
|
IActiveWindowScreenshotCapture screenshotCapture,
|
|
IMeetingArtifactStore artifactStore,
|
|
IMeetingNoteStore meetingNoteStore,
|
|
ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer,
|
|
IScreenshotOcrClient ocrClient,
|
|
ILogger<MeetingScreenshotService> logger,
|
|
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
|
|
{
|
|
this.screenshotCapture = screenshotCapture;
|
|
this.artifactStore = artifactStore;
|
|
this.meetingNoteStore = meetingNoteStore;
|
|
this.attendeeCanonicalizer = attendeeCanonicalizer;
|
|
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
|
|
this.ocrClient = ocrClient;
|
|
this.logger = logger;
|
|
}
|
|
|
|
public async Task<MeetingScreenshotCaptureResult> CaptureAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
DateTimeOffset? meetingStartedAt,
|
|
DateTimeOffset capturedAt,
|
|
MeetingAssistantOptions options,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var screenshot = await screenshotCapture.CaptureAsync(cancellationToken);
|
|
var timestamp = CalculateMeetingTimestamp(meetingStartedAt, capturedAt);
|
|
var screenshotPath = await SaveScreenshotAsync(
|
|
artifacts,
|
|
screenshot.PngBytes,
|
|
timestamp,
|
|
capturedAt,
|
|
options,
|
|
cancellationToken);
|
|
var relativePath = ToMarkdownPath(Path.GetRelativePath(
|
|
Path.GetDirectoryName(artifacts.AssistantContextPath)!,
|
|
screenshotPath));
|
|
var screenshotId = Guid.NewGuid().ToString("N");
|
|
var ocrEnabled = options.Screenshots.Ocr.Enabled;
|
|
await artifactStore.AppendAssistantContextAsync(
|
|
artifacts,
|
|
CreateScreenshotMarkdown(screenshotId, timestamp, relativePath, screenshot.WindowTitle, ocrEnabled),
|
|
cancellationToken);
|
|
|
|
if (ocrEnabled)
|
|
{
|
|
var ocrCancellation = new CancellationTokenSource();
|
|
TrackOcr(
|
|
artifacts,
|
|
ocrCancellation,
|
|
RunOcrAsync(
|
|
artifacts,
|
|
screenshotPath,
|
|
screenshotId,
|
|
options,
|
|
ScreenshotOcrPolicy.CapturedScreenshot,
|
|
ocrCancellation.Token));
|
|
}
|
|
|
|
logger.LogInformation(
|
|
"Captured meeting screenshot {ScreenshotPath} at {MeetingTimestamp}",
|
|
screenshotPath,
|
|
FormatTimestamp(timestamp));
|
|
return new MeetingScreenshotCaptureResult(screenshotPath, timestamp, ocrEnabled);
|
|
}
|
|
|
|
public async Task WaitForPendingOcrAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
TimeSpan timeout,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var contextKey = NormalizeContextKey(artifacts.AssistantContextPath);
|
|
if (!pendingOcrByContext.TryGetValue(contextKey, out var tasks))
|
|
{
|
|
return;
|
|
}
|
|
|
|
PendingOcrTask[] snapshot;
|
|
lock (tasks)
|
|
{
|
|
tasks.RemoveAll(task => task.Task.IsCompleted);
|
|
snapshot = tasks.ToArray();
|
|
}
|
|
|
|
if (snapshot.Length == 0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
await Task.WhenAll(snapshot.Select(task => task.Task)).WaitAsync(timeout, cancellationToken);
|
|
}
|
|
catch (TimeoutException)
|
|
{
|
|
logger.LogWarning(
|
|
"Timed out waiting for {Count} screenshot OCR task(s) for {AssistantContextPath}",
|
|
snapshot.Length,
|
|
artifacts.AssistantContextPath);
|
|
}
|
|
}
|
|
|
|
public async Task CancelPendingOcrAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
TimeSpan timeout,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var contextKey = NormalizeContextKey(artifacts.AssistantContextPath);
|
|
if (!pendingOcrByContext.TryGetValue(contextKey, out var tasks))
|
|
{
|
|
return;
|
|
}
|
|
|
|
PendingOcrTask[] snapshot;
|
|
lock (tasks)
|
|
{
|
|
tasks.RemoveAll(task => task.Task.IsCompleted);
|
|
snapshot = tasks.ToArray();
|
|
}
|
|
|
|
if (snapshot.Length == 0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var pending in snapshot)
|
|
{
|
|
await pending.Cancellation.CancelAsync();
|
|
}
|
|
|
|
await WaitForPendingOcrAsync(artifacts, timeout, cancellationToken);
|
|
}
|
|
|
|
public async Task<MeetingScreenshotOcrRetryTriggerResult?> TriggerOcrRetryAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
string screenshotPath,
|
|
string screenshotId,
|
|
MeetingAssistantOptions options,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(screenshotId) ||
|
|
string.IsNullOrWhiteSpace(screenshotPath) ||
|
|
!File.Exists(screenshotPath) ||
|
|
!File.Exists(artifacts.AssistantContextPath))
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var replaced = await ReplaceOcrPlaceholderAsync(
|
|
artifacts.AssistantContextPath,
|
|
screenshotId,
|
|
"_OCR retry pending..._",
|
|
cancellationToken,
|
|
preserveMarkers: true,
|
|
appendWhenMissing: false);
|
|
if (!replaced)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var ocrCancellation = new CancellationTokenSource();
|
|
TrackOcr(
|
|
artifacts,
|
|
ocrCancellation,
|
|
RunOcrAsync(
|
|
artifacts,
|
|
screenshotPath,
|
|
screenshotId,
|
|
options,
|
|
ScreenshotOcrPolicy.CapturedScreenshot,
|
|
ocrCancellation.Token));
|
|
return new MeetingScreenshotOcrRetryTriggerResult(
|
|
artifacts.AssistantContextPath,
|
|
screenshotPath,
|
|
screenshotId);
|
|
}
|
|
|
|
public async Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
MeetingAssistantOptions options,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
if (!options.Screenshots.Ocr.Enabled || !File.Exists(artifacts.MeetingNotePath))
|
|
{
|
|
return new MeetingNoteImageOcrQueueResult(0);
|
|
}
|
|
|
|
var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken);
|
|
var embeds = FindMeetingNoteImageEmbeds(
|
|
meetingNote.UserNotes,
|
|
artifacts.MeetingNotePath,
|
|
options);
|
|
var queuedEmbeds = new List<(MeetingNoteImageEmbed Embed, string OcrId)>();
|
|
foreach (var embed in embeds)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
var ocrId = Guid.NewGuid().ToString("N");
|
|
await artifactStore.AppendAssistantContextAsync(
|
|
artifacts,
|
|
CreateMeetingNoteImageMarkdown(
|
|
ocrId,
|
|
embed.OriginalEmbed,
|
|
embed.ImagePath,
|
|
artifacts.AssistantContextPath),
|
|
cancellationToken);
|
|
queuedEmbeds.Add((embed, ocrId));
|
|
}
|
|
|
|
foreach (var (embed, ocrId) in queuedEmbeds)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
var ocrCancellation = new CancellationTokenSource();
|
|
TrackOcr(
|
|
artifacts,
|
|
ocrCancellation,
|
|
RunOcrAsync(
|
|
artifacts,
|
|
embed.ImagePath,
|
|
ocrId,
|
|
options,
|
|
ScreenshotOcrPolicy.MeetingNoteImage,
|
|
ocrCancellation.Token));
|
|
}
|
|
|
|
return new MeetingNoteImageOcrQueueResult(queuedEmbeds.Count);
|
|
}
|
|
|
|
private async Task<string> SaveScreenshotAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
byte[] pngBytes,
|
|
TimeSpan timestamp,
|
|
DateTimeOffset capturedAt,
|
|
MeetingAssistantOptions options,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var folder = ResolveAttachmentsFolder(artifacts.AssistantContextPath, options.Screenshots.AttachmentsFolder);
|
|
Directory.CreateDirectory(folder);
|
|
var path = Path.Combine(
|
|
folder,
|
|
$"{capturedAt:yyyyMMdd-HHmmss-fff}-{FormatTimestamp(timestamp).Replace(":", "", StringComparison.Ordinal)}.png");
|
|
await File.WriteAllBytesAsync(path, pngBytes, cancellationToken);
|
|
return path;
|
|
}
|
|
|
|
private async Task RunOcrAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
string screenshotPath,
|
|
string screenshotId,
|
|
MeetingAssistantOptions options,
|
|
ScreenshotOcrPolicy policy,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var timeout = options.Screenshots.Ocr.GetEffectiveTimeout();
|
|
using var timeoutSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
|
timeoutSource.CancelAfter(timeout);
|
|
try
|
|
{
|
|
var result = await ocrClient.ExtractAsync(
|
|
screenshotPath,
|
|
string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Prompt)
|
|
? ScreenshotOcrOptions.DefaultPrompt
|
|
: options.Screenshots.Ocr.Prompt,
|
|
options,
|
|
timeoutSource.Token);
|
|
var cropMarkdown = policy.AllowCrop
|
|
? await TryCreateCropMarkdownAsync(
|
|
artifacts.AssistantContextPath,
|
|
screenshotPath,
|
|
result.Crop,
|
|
timeoutSource.Token)
|
|
: "";
|
|
await ReplaceOcrPlaceholderAsync(
|
|
artifacts.AssistantContextPath,
|
|
screenshotId,
|
|
cropMarkdown +
|
|
"### OCR" + Environment.NewLine + Environment.NewLine + result.Text.Trim(),
|
|
CancellationToken.None);
|
|
if (policy.AddAttendees)
|
|
{
|
|
await AddOcrAttendeesAsync(
|
|
artifacts,
|
|
result.Attendees,
|
|
options,
|
|
CancellationToken.None);
|
|
}
|
|
}
|
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
|
{
|
|
logger.LogInformation("Cancelled screenshot OCR for {ScreenshotPath}", screenshotPath);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
await ReplaceOcrPlaceholderAsync(
|
|
artifacts.AssistantContextPath,
|
|
screenshotId,
|
|
CreateOcrFailureMarkdown(
|
|
artifacts,
|
|
screenshotPath,
|
|
screenshotId,
|
|
options,
|
|
$"_OCR timed out after {timeout:g}._",
|
|
policy.IncludeRetryLink),
|
|
CancellationToken.None,
|
|
preserveMarkers: true);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
logger.LogWarning(exception, "Screenshot OCR failed for {ScreenshotPath}", screenshotPath);
|
|
await ReplaceOcrPlaceholderAsync(
|
|
artifacts.AssistantContextPath,
|
|
screenshotId,
|
|
CreateOcrFailureMarkdown(
|
|
artifacts,
|
|
screenshotPath,
|
|
screenshotId,
|
|
options,
|
|
$"_OCR failed: {exception.Message}_",
|
|
policy.IncludeRetryLink),
|
|
CancellationToken.None,
|
|
preserveMarkers: true);
|
|
}
|
|
}
|
|
|
|
private async Task AddOcrAttendeesAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
IReadOnlyList<string> attendees,
|
|
MeetingAssistantOptions options,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var additions = MeetingAttendeeNames.NormalizeDistinct(attendees);
|
|
if (additions.Count == 0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken);
|
|
var transformedAdditions = await TransformAttendeesAsync(
|
|
artifacts,
|
|
additions,
|
|
options,
|
|
cancellationToken);
|
|
var canonicalized = await attendeeCanonicalizer.CanonicalizeAsync(
|
|
meetingNote.Frontmatter.Attendees.Concat(transformedAdditions).ToList(),
|
|
cancellationToken);
|
|
if (meetingNote.Frontmatter.Attendees.SequenceEqual(canonicalized, StringComparer.Ordinal))
|
|
{
|
|
return;
|
|
}
|
|
|
|
meetingNote.Frontmatter.Attendees = canonicalized.ToList();
|
|
await meetingNoteStore.SaveAsync(meetingNote, cancellationToken);
|
|
logger.LogInformation(
|
|
"Added {AttendeeCount} screenshot OCR attendee candidate(s) to meeting note {MeetingNotePath}",
|
|
additions.Count,
|
|
artifacts.MeetingNotePath);
|
|
}
|
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
|
{
|
|
logger.LogWarning(
|
|
exception,
|
|
"Could not add screenshot OCR attendees to meeting note {MeetingNotePath}",
|
|
artifacts.MeetingNotePath);
|
|
}
|
|
}
|
|
|
|
private async Task<IReadOnlyList<string>> TransformAttendeesAsync(
|
|
MeetingSessionArtifacts artifacts,
|
|
IReadOnlyList<string> attendees,
|
|
MeetingAssistantOptions options,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var transformed = new List<string>();
|
|
foreach (var attendee in attendees)
|
|
{
|
|
var value = await meetingWorkflowEngine.TransformAttendeeAsync(
|
|
MeetingWorkflowEvent.AttendeeAdded(artifacts, attendee),
|
|
options,
|
|
cancellationToken);
|
|
var normalized = MeetingAttendeeNames.NormalizeDisplayName(value);
|
|
if (!string.IsNullOrWhiteSpace(normalized) &&
|
|
!transformed.Contains(normalized, StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
transformed.Add(normalized);
|
|
}
|
|
}
|
|
|
|
return transformed;
|
|
}
|
|
|
|
private async Task<bool> ReplaceOcrPlaceholderAsync(
|
|
string assistantContextPath,
|
|
string screenshotId,
|
|
string replacement,
|
|
CancellationToken cancellationToken,
|
|
bool preserveMarkers = false,
|
|
bool appendWhenMissing = true)
|
|
{
|
|
var fileLock = contextFileLocks.GetOrAdd(
|
|
NormalizeContextKey(assistantContextPath),
|
|
_ => new SemaphoreSlim(1, 1));
|
|
await fileLock.WaitAsync(cancellationToken);
|
|
try
|
|
{
|
|
if (!File.Exists(assistantContextPath))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var content = await File.ReadAllTextAsync(assistantContextPath, cancellationToken);
|
|
var startMarker = CreateOcrStartMarker(screenshotId);
|
|
var endMarker = CreateOcrEndMarker(screenshotId);
|
|
var startIndex = content.IndexOf(startMarker, StringComparison.Ordinal);
|
|
var endIndex = content.IndexOf(endMarker, StringComparison.Ordinal);
|
|
if (startIndex < 0 || endIndex < startIndex)
|
|
{
|
|
if (appendWhenMissing)
|
|
{
|
|
await File.AppendAllTextAsync(
|
|
assistantContextPath,
|
|
Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
|
|
cancellationToken);
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
endIndex += endMarker.Length;
|
|
var updated = preserveMarkers
|
|
? content[..startIndex] +
|
|
startMarker +
|
|
Environment.NewLine +
|
|
replacement.TrimEnd() +
|
|
Environment.NewLine +
|
|
endMarker +
|
|
content[endIndex..]
|
|
: content[..startIndex] +
|
|
replacement.TrimEnd() +
|
|
content[endIndex..];
|
|
await File.WriteAllTextAsync(assistantContextPath, updated, cancellationToken);
|
|
return true;
|
|
}
|
|
finally
|
|
{
|
|
fileLock.Release();
|
|
}
|
|
}
|
|
|
|
private static string CreateOcrFailureMarkdown(
|
|
MeetingSessionArtifacts artifacts,
|
|
string screenshotPath,
|
|
string screenshotId,
|
|
MeetingAssistantOptions options,
|
|
string status,
|
|
bool includeRetryLink = true)
|
|
{
|
|
var markdown = status.TrimEnd();
|
|
if (includeRetryLink)
|
|
{
|
|
markdown += Environment.NewLine +
|
|
Environment.NewLine +
|
|
MeetingNoteActionLinks.CreateScreenshotOcrRetryLink(
|
|
options.Api.PublicBaseUrl,
|
|
artifacts,
|
|
screenshotPath,
|
|
screenshotId);
|
|
}
|
|
|
|
return markdown;
|
|
}
|
|
|
|
private async Task<string> TryCreateCropMarkdownAsync(
|
|
string assistantContextPath,
|
|
string screenshotPath,
|
|
ScreenshotCropCoordinates? crop,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
if (crop is null)
|
|
{
|
|
return "";
|
|
}
|
|
|
|
if (!OperatingSystem.IsWindows())
|
|
{
|
|
logger.LogWarning(
|
|
"Ignoring screenshot crop coordinates for {ScreenshotPath} because image cropping is only supported on Windows",
|
|
screenshotPath);
|
|
return "";
|
|
}
|
|
|
|
try
|
|
{
|
|
var croppedPath = await SaveCroppedScreenshotAsync(screenshotPath, crop, cancellationToken);
|
|
var relativePath = ToMarkdownPath(Path.GetRelativePath(
|
|
Path.GetDirectoryName(assistantContextPath)!,
|
|
croppedPath));
|
|
return $"" +
|
|
Environment.NewLine +
|
|
Environment.NewLine;
|
|
}
|
|
catch (Exception exception) when (exception is InvalidDataException or ArgumentException or ExternalException)
|
|
{
|
|
logger.LogWarning(
|
|
exception,
|
|
"Ignoring invalid screenshot crop coordinates for {ScreenshotPath}",
|
|
screenshotPath);
|
|
return "";
|
|
}
|
|
}
|
|
|
|
#pragma warning disable CA1416
|
|
private static Task<string> SaveCroppedScreenshotAsync(
|
|
string screenshotPath,
|
|
ScreenshotCropCoordinates crop,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
using var original = new Bitmap(screenshotPath);
|
|
ValidateCrop(crop, original.Width, original.Height);
|
|
var cropRectangle = new Rectangle(crop.X, crop.Y, crop.Width, crop.Height);
|
|
using var cropped = original.Clone(cropRectangle, original.PixelFormat);
|
|
var croppedPath = Path.Combine(
|
|
Path.GetDirectoryName(screenshotPath)!,
|
|
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
|
|
cropped.Save(croppedPath, ImageFormat.Png);
|
|
return Task.FromResult(croppedPath);
|
|
}
|
|
#pragma warning restore CA1416
|
|
|
|
private static void ValidateCrop(ScreenshotCropCoordinates crop, int imageWidth, int imageHeight)
|
|
{
|
|
if (crop.X < 0 ||
|
|
crop.Y < 0 ||
|
|
crop.Width <= 0 ||
|
|
crop.Height <= 0 ||
|
|
crop.X + crop.Width > imageWidth ||
|
|
crop.Y + crop.Height > imageHeight)
|
|
{
|
|
throw new InvalidDataException(
|
|
$"Screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} is outside image bounds {imageWidth}x{imageHeight}.");
|
|
}
|
|
}
|
|
|
|
private void TrackOcr(
|
|
MeetingSessionArtifacts artifacts,
|
|
CancellationTokenSource cancellation,
|
|
Task task)
|
|
{
|
|
var contextKey = NormalizeContextKey(artifacts.AssistantContextPath);
|
|
var tasks = pendingOcrByContext.GetOrAdd(contextKey, _ => []);
|
|
lock (tasks)
|
|
{
|
|
tasks.RemoveAll(existing => existing.Task.IsCompleted);
|
|
tasks.Add(new PendingOcrTask(task, cancellation));
|
|
}
|
|
}
|
|
|
|
private static string CreateScreenshotMarkdown(
|
|
string screenshotId,
|
|
TimeSpan timestamp,
|
|
string relativePath,
|
|
string? windowTitle,
|
|
bool ocrEnabled)
|
|
{
|
|
var title = string.IsNullOrWhiteSpace(windowTitle)
|
|
? ""
|
|
: Environment.NewLine + $"Window: {windowTitle.Trim()}" + Environment.NewLine;
|
|
var timestampText = FormatTimestamp(timestamp);
|
|
var content = $"## Screenshot [{timestampText}]" +
|
|
title +
|
|
Environment.NewLine +
|
|
$"";
|
|
if (!ocrEnabled)
|
|
{
|
|
return content;
|
|
}
|
|
|
|
return content +
|
|
Environment.NewLine +
|
|
Environment.NewLine +
|
|
CreateOcrBlock(screenshotId, "_OCR pending..._");
|
|
}
|
|
|
|
private static string CreateMeetingNoteImageMarkdown(
|
|
string ocrId,
|
|
string originalEmbed,
|
|
string imagePath,
|
|
string assistantContextPath)
|
|
{
|
|
var relativePath = ToMarkdownPath(Path.GetRelativePath(
|
|
Path.GetDirectoryName(assistantContextPath)!,
|
|
imagePath));
|
|
return "## Meeting Note Image" +
|
|
Environment.NewLine +
|
|
"Image from meeting note." +
|
|
Environment.NewLine +
|
|
$"Original embed: `{originalEmbed.Replace("`", "\\`", StringComparison.Ordinal)}`" +
|
|
Environment.NewLine +
|
|
$"" +
|
|
Environment.NewLine +
|
|
Environment.NewLine +
|
|
CreateOcrBlock(ocrId, "_OCR pending..._");
|
|
}
|
|
|
|
private static string CreateOcrBlock(string screenshotId, string content)
|
|
{
|
|
return CreateOcrStartMarker(screenshotId) +
|
|
Environment.NewLine +
|
|
content.TrimEnd() +
|
|
Environment.NewLine +
|
|
CreateOcrEndMarker(screenshotId);
|
|
}
|
|
|
|
private static string CreateOcrStartMarker(string screenshotId)
|
|
{
|
|
return $"<!-- screenshot-ocr:{screenshotId} -->";
|
|
}
|
|
|
|
private static string CreateOcrEndMarker(string screenshotId)
|
|
{
|
|
return $"<!-- /screenshot-ocr:{screenshotId} -->";
|
|
}
|
|
|
|
private static IReadOnlyList<MeetingNoteImageEmbed> FindMeetingNoteImageEmbeds(
|
|
string userNotes,
|
|
string meetingNotePath,
|
|
MeetingAssistantOptions options)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(userNotes))
|
|
{
|
|
return [];
|
|
}
|
|
|
|
var result = new List<MeetingNoteImageEmbed>();
|
|
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
foreach (Match match in ObsidianImageEmbedRegex().Matches(userNotes))
|
|
{
|
|
AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options);
|
|
}
|
|
|
|
foreach (Match match in MarkdownImageEmbedRegex().Matches(userNotes))
|
|
{
|
|
AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
private static void AddResolvedEmbed(
|
|
List<MeetingNoteImageEmbed> embeds,
|
|
HashSet<string> seen,
|
|
string originalEmbed,
|
|
string target,
|
|
string meetingNotePath,
|
|
MeetingAssistantOptions options)
|
|
{
|
|
var imagePath = ResolveMeetingNoteImagePath(target, meetingNotePath, options);
|
|
if (imagePath is null || !seen.Add(imagePath))
|
|
{
|
|
return;
|
|
}
|
|
|
|
embeds.Add(new MeetingNoteImageEmbed(originalEmbed, imagePath));
|
|
}
|
|
|
|
private static string? ResolveMeetingNoteImagePath(
|
|
string target,
|
|
string meetingNotePath,
|
|
MeetingAssistantOptions options)
|
|
{
|
|
var cleaned = CleanEmbedTarget(target);
|
|
if (string.IsNullOrWhiteSpace(cleaned) ||
|
|
cleaned.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
|
|
cleaned.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ||
|
|
cleaned.StartsWith("data:", StringComparison.OrdinalIgnoreCase) ||
|
|
!IsSupportedImagePath(cleaned))
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var candidates = new List<string>();
|
|
if (Path.IsPathRooted(cleaned))
|
|
{
|
|
candidates.Add(VaultPath.Resolve(cleaned));
|
|
}
|
|
else
|
|
{
|
|
candidates.Add(Path.GetFullPath(Path.Combine(Path.GetDirectoryName(meetingNotePath)!, cleaned)));
|
|
candidates.Add(VaultPath.Resolve(options.Vault, cleaned));
|
|
}
|
|
|
|
foreach (var candidate in candidates)
|
|
{
|
|
if (File.Exists(candidate))
|
|
{
|
|
return candidate;
|
|
}
|
|
}
|
|
|
|
if (Path.GetFileName(cleaned).Equals(cleaned, StringComparison.Ordinal))
|
|
{
|
|
try
|
|
{
|
|
var vaultRoot = VaultPath.Resolve(options.Vault.BaseFolder);
|
|
return Directory.EnumerateFiles(vaultRoot, cleaned, SearchOption.AllDirectories)
|
|
.FirstOrDefault(IsSupportedImagePath);
|
|
}
|
|
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or DirectoryNotFoundException)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private static string CleanEmbedTarget(string target)
|
|
{
|
|
var cleaned = target.Trim();
|
|
if (cleaned.StartsWith("<", StringComparison.Ordinal) &&
|
|
cleaned.EndsWith(">", StringComparison.Ordinal) &&
|
|
cleaned.Length > 1)
|
|
{
|
|
cleaned = cleaned[1..^1].Trim();
|
|
}
|
|
|
|
var pipeIndex = cleaned.IndexOf('|', StringComparison.Ordinal);
|
|
if (pipeIndex >= 0)
|
|
{
|
|
cleaned = cleaned[..pipeIndex].Trim();
|
|
}
|
|
|
|
var hashIndex = cleaned.IndexOf('#', StringComparison.Ordinal);
|
|
if (hashIndex >= 0)
|
|
{
|
|
cleaned = cleaned[..hashIndex].Trim();
|
|
}
|
|
|
|
try
|
|
{
|
|
return Uri.UnescapeDataString(cleaned.Replace('/', Path.DirectorySeparatorChar));
|
|
}
|
|
catch (UriFormatException)
|
|
{
|
|
return cleaned.Replace('/', Path.DirectorySeparatorChar);
|
|
}
|
|
}
|
|
|
|
private static bool IsSupportedImagePath(string path)
|
|
{
|
|
return Path.GetExtension(path).ToLowerInvariant() switch
|
|
{
|
|
".png" or ".jpg" or ".jpeg" or ".gif" or ".webp" or ".bmp" or ".tif" or ".tiff" => true,
|
|
_ => false
|
|
};
|
|
}
|
|
|
|
[GeneratedRegex(@"!\[\[(?<target>[^\]\r\n]+)\]\]", RegexOptions.CultureInvariant)]
|
|
private static partial Regex ObsidianImageEmbedRegex();
|
|
|
|
[GeneratedRegex(@"!\[[^\]\r\n]*\]\((?<target>[^)\r\n]+)\)", RegexOptions.CultureInvariant)]
|
|
private static partial Regex MarkdownImageEmbedRegex();
|
|
|
|
private static string ResolveAttachmentsFolder(string assistantContextPath, string configuredFolder)
|
|
{
|
|
var expanded = Environment.ExpandEnvironmentVariables(
|
|
string.IsNullOrWhiteSpace(configuredFolder) ? "Attachments" : configuredFolder);
|
|
return Path.IsPathRooted(expanded)
|
|
? Path.GetFullPath(expanded)
|
|
: Path.GetFullPath(Path.Combine(Path.GetDirectoryName(assistantContextPath)!, expanded));
|
|
}
|
|
|
|
private static TimeSpan CalculateMeetingTimestamp(DateTimeOffset? meetingStartedAt, DateTimeOffset capturedAt)
|
|
{
|
|
if (meetingStartedAt is null)
|
|
{
|
|
return TimeSpan.Zero;
|
|
}
|
|
|
|
var elapsed = capturedAt - meetingStartedAt.Value;
|
|
return elapsed < TimeSpan.Zero ? TimeSpan.Zero : elapsed;
|
|
}
|
|
|
|
private static string FormatTimestamp(TimeSpan timestamp)
|
|
{
|
|
return $"{(int)timestamp.TotalHours:00}:{timestamp.Minutes:00}:{timestamp.Seconds:00}";
|
|
}
|
|
|
|
private static string ToMarkdownPath(string path)
|
|
{
|
|
return path.Replace(Path.DirectorySeparatorChar, '/')
|
|
.Replace(Path.AltDirectorySeparatorChar, '/');
|
|
}
|
|
|
|
private static string NormalizeContextKey(string assistantContextPath)
|
|
{
|
|
return Path.GetFullPath(assistantContextPath);
|
|
}
|
|
|
|
private sealed record PendingOcrTask(Task Task, CancellationTokenSource Cancellation);
|
|
|
|
private sealed record ScreenshotOcrPolicy(bool AllowCrop, bool AddAttendees, bool IncludeRetryLink)
|
|
{
|
|
public static ScreenshotOcrPolicy CapturedScreenshot { get; } = new(
|
|
AllowCrop: true,
|
|
AddAttendees: true,
|
|
IncludeRetryLink: true);
|
|
|
|
public static ScreenshotOcrPolicy MeetingNoteImage { get; } = new(
|
|
AllowCrop: false,
|
|
AddAttendees: false,
|
|
IncludeRetryLink: false);
|
|
}
|
|
|
|
private sealed record MeetingNoteImageEmbed(string OriginalEmbed, string ImagePath);
|
|
}
|