Public Access
Archive meeting workflow and screenshot changes
This commit is contained in:
@@ -0,0 +1,121 @@
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
public sealed class ActiveWindowScreenshotCapture : IActiveWindowScreenshotCapture
|
||||
{
|
||||
private static readonly IntPtr DpiAwarenessContextPerMonitorAwareV2 = new(-4);
|
||||
private const int DwmWindowAttributeExtendedFrameBounds = 9;
|
||||
|
||||
public Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var previousDpiContext = TrySetThreadDpiAwarenessContext(DpiAwarenessContextPerMonitorAwareV2);
|
||||
try
|
||||
{
|
||||
var foregroundWindow = GetForegroundWindow();
|
||||
if (foregroundWindow == IntPtr.Zero ||
|
||||
!TryGetWindowBounds(foregroundWindow, out var rect))
|
||||
{
|
||||
throw new InvalidOperationException("Could not determine the active window bounds.");
|
||||
}
|
||||
|
||||
using var bitmap = new Bitmap(rect.Width, rect.Height);
|
||||
using (var graphics = Graphics.FromImage(bitmap))
|
||||
{
|
||||
graphics.CopyFromScreen(rect.Left, rect.Top, 0, 0, new Size(rect.Width, rect.Height));
|
||||
}
|
||||
|
||||
using var stream = new MemoryStream();
|
||||
bitmap.Save(stream, ImageFormat.Png);
|
||||
return Task.FromResult(new ActiveWindowScreenshot(stream.ToArray(), GetWindowTitle(foregroundWindow)));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (previousDpiContext != IntPtr.Zero)
|
||||
{
|
||||
TrySetThreadDpiAwarenessContext(previousDpiContext);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryGetWindowBounds(IntPtr window, out NativeRect rect)
|
||||
{
|
||||
if (DwmGetWindowAttribute(
|
||||
window,
|
||||
DwmWindowAttributeExtendedFrameBounds,
|
||||
out rect,
|
||||
Marshal.SizeOf<NativeRect>()) == 0 &&
|
||||
rect.Width > 0 &&
|
||||
rect.Height > 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return GetWindowRect(window, out rect) &&
|
||||
rect.Width > 0 &&
|
||||
rect.Height > 0;
|
||||
}
|
||||
|
||||
private static IntPtr TrySetThreadDpiAwarenessContext(IntPtr dpiContext)
|
||||
{
|
||||
try
|
||||
{
|
||||
return SetThreadDpiAwarenessContext(dpiContext);
|
||||
}
|
||||
catch (EntryPointNotFoundException)
|
||||
{
|
||||
return IntPtr.Zero;
|
||||
}
|
||||
}
|
||||
|
||||
private static string? GetWindowTitle(IntPtr window)
|
||||
{
|
||||
var length = GetWindowTextLength(window);
|
||||
if (length <= 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var buffer = new System.Text.StringBuilder(length + 1);
|
||||
return GetWindowText(window, buffer, buffer.Capacity) > 0
|
||||
? buffer.ToString()
|
||||
: null;
|
||||
}
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
private static extern IntPtr GetForegroundWindow();
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
private static extern bool GetWindowRect(IntPtr hWnd, out NativeRect lpRect);
|
||||
|
||||
[DllImport("dwmapi.dll", PreserveSig = true)]
|
||||
private static extern int DwmGetWindowAttribute(
|
||||
IntPtr hwnd,
|
||||
int dwAttribute,
|
||||
out NativeRect pvAttribute,
|
||||
int cbAttribute);
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
private static extern IntPtr SetThreadDpiAwarenessContext(IntPtr dpiContext);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
||||
private static extern int GetWindowText(IntPtr hWnd, System.Text.StringBuilder lpString, int nMaxCount);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
||||
private static extern int GetWindowTextLength(IntPtr hWnd);
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
private readonly struct NativeRect
|
||||
{
|
||||
public readonly int Left;
|
||||
public readonly int Top;
|
||||
public readonly int Right;
|
||||
public readonly int Bottom;
|
||||
|
||||
public int Width => Right - Left;
|
||||
|
||||
public int Height => Bottom - Top;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
private readonly ILogger<LiteLlmScreenshotOcrClient> logger;
|
||||
private readonly Func<HttpMessageHandler>? httpMessageHandlerFactory;
|
||||
|
||||
public LiteLlmScreenshotOcrClient(ILogger<LiteLlmScreenshotOcrClient> logger)
|
||||
{
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
internal LiteLlmScreenshotOcrClient(
|
||||
Func<HttpMessageHandler> httpMessageHandlerFactory,
|
||||
ILogger<LiteLlmScreenshotOcrClient> logger)
|
||||
{
|
||||
this.httpMessageHandlerFactory = httpMessageHandlerFactory;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<ScreenshotOcrResult> ExtractAsync(
|
||||
string screenshotPath,
|
||||
string prompt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var endpoint = !string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Endpoint)
|
||||
? options.Screenshots.Ocr.Endpoint
|
||||
: options.Agent.Endpoint;
|
||||
var model = !string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Model)
|
||||
? options.Screenshots.Ocr.Model
|
||||
: options.Agent.Model;
|
||||
var key = ResolveApiKey(options);
|
||||
var imageBytes = await File.ReadAllBytesAsync(screenshotPath, cancellationToken);
|
||||
using var httpClient = CreateHttpClient();
|
||||
httpClient.BaseAddress = NormalizeEndpoint(new Uri(endpoint));
|
||||
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", key);
|
||||
var payload = CreatePayload(model, CreatePrompt(prompt, imageBytes), imageBytes);
|
||||
using var content = new StringContent(payload.ToJsonString(JsonOptions), Encoding.UTF8, "application/json");
|
||||
using var response = await httpClient.PostAsync("responses", content, cancellationToken);
|
||||
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Screenshot OCR request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}");
|
||||
}
|
||||
|
||||
var text = ParseOutputText(responseJson);
|
||||
logger.LogInformation("Screenshot OCR completed for {ScreenshotPath}", screenshotPath);
|
||||
return ParseOcrResult(text);
|
||||
}
|
||||
|
||||
private HttpClient CreateHttpClient()
|
||||
{
|
||||
return httpMessageHandlerFactory is null
|
||||
? new HttpClient()
|
||||
: new HttpClient(httpMessageHandlerFactory());
|
||||
}
|
||||
|
||||
private static JsonObject CreatePayload(string model, string prompt, byte[] imageBytes)
|
||||
{
|
||||
return new JsonObject
|
||||
{
|
||||
["model"] = model,
|
||||
["store"] = false,
|
||||
["input"] = new JsonArray
|
||||
{
|
||||
new JsonObject
|
||||
{
|
||||
["role"] = "user",
|
||||
["content"] = new JsonArray
|
||||
{
|
||||
new JsonObject
|
||||
{
|
||||
["type"] = "input_text",
|
||||
["text"] = prompt
|
||||
},
|
||||
new JsonObject
|
||||
{
|
||||
["type"] = "input_image",
|
||||
["image_url"] = "data:image/png;base64," + Convert.ToBase64String(imageBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static string CreatePrompt(string prompt, byte[] imageBytes)
|
||||
{
|
||||
return TryReadPngDimensions(imageBytes, out var width, out var height)
|
||||
? prompt + Environment.NewLine + Environment.NewLine +
|
||||
$"Original screenshot dimensions: {width}x{height} pixels. Crop coordinates must fit within these bounds."
|
||||
: prompt;
|
||||
}
|
||||
|
||||
private static ScreenshotOcrResult ParseOcrResult(string text)
|
||||
{
|
||||
ScreenshotCropCoordinates? crop = null;
|
||||
var cleaned = JsonCodeBlockRegex().Replace(text, match =>
|
||||
{
|
||||
if (TryParseCrop(match.Groups["json"].Value, out var parsedCrop))
|
||||
{
|
||||
crop = parsedCrop;
|
||||
return "";
|
||||
}
|
||||
|
||||
return match.Value;
|
||||
}).Trim();
|
||||
return new ScreenshotOcrResult(cleaned, crop);
|
||||
}
|
||||
|
||||
private static bool TryParseCrop(string json, out ScreenshotCropCoordinates? crop)
|
||||
{
|
||||
crop = null;
|
||||
try
|
||||
{
|
||||
using var document = JsonDocument.Parse(json);
|
||||
if (!document.RootElement.TryGetProperty("crop", out var cropElement) ||
|
||||
cropElement.ValueKind == JsonValueKind.Null)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (cropElement.ValueKind != JsonValueKind.Object ||
|
||||
!TryGetInt(cropElement, "x", out var x) ||
|
||||
!TryGetInt(cropElement, "y", out var y) ||
|
||||
!TryGetInt(cropElement, "width", out var width) ||
|
||||
!TryGetInt(cropElement, "height", out var height))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
crop = new ScreenshotCropCoordinates(x, y, width, height);
|
||||
return true;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryGetInt(JsonElement element, string propertyName, out int value)
|
||||
{
|
||||
value = 0;
|
||||
return element.TryGetProperty(propertyName, out var property) &&
|
||||
property.ValueKind == JsonValueKind.Number &&
|
||||
property.TryGetInt32(out value);
|
||||
}
|
||||
|
||||
private static bool TryReadPngDimensions(byte[] bytes, out int width, out int height)
|
||||
{
|
||||
width = 0;
|
||||
height = 0;
|
||||
if (bytes.Length < 24 ||
|
||||
bytes[0] != 0x89 ||
|
||||
bytes[1] != 0x50 ||
|
||||
bytes[2] != 0x4E ||
|
||||
bytes[3] != 0x47)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
width = ReadBigEndianInt32(bytes, 16);
|
||||
height = ReadBigEndianInt32(bytes, 20);
|
||||
return width > 0 && height > 0;
|
||||
}
|
||||
|
||||
private static int ReadBigEndianInt32(byte[] bytes, int offset)
|
||||
{
|
||||
return (bytes[offset] << 24) |
|
||||
(bytes[offset + 1] << 16) |
|
||||
(bytes[offset + 2] << 8) |
|
||||
bytes[offset + 3];
|
||||
}
|
||||
|
||||
private static string ParseOutputText(string responseJson)
|
||||
{
|
||||
using var document = JsonDocument.Parse(responseJson);
|
||||
var root = document.RootElement;
|
||||
var parts = new List<string>();
|
||||
if (root.TryGetProperty("output_text", out var outputText) &&
|
||||
outputText.ValueKind == JsonValueKind.String &&
|
||||
!string.IsNullOrWhiteSpace(outputText.GetString()))
|
||||
{
|
||||
parts.Add(outputText.GetString()!);
|
||||
}
|
||||
|
||||
if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var item in output.EnumerateArray())
|
||||
{
|
||||
if (!item.TryGetProperty("content", out var content) || content.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var block in content.EnumerateArray())
|
||||
{
|
||||
if (block.TryGetProperty("type", out var type) &&
|
||||
type.GetString() == "output_text" &&
|
||||
block.TryGetProperty("text", out var text) &&
|
||||
text.ValueKind == JsonValueKind.String &&
|
||||
!string.IsNullOrWhiteSpace(text.GetString()))
|
||||
{
|
||||
parts.Add(text.GetString()!);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return parts.Count == 0
|
||||
? ""
|
||||
: string.Join(Environment.NewLine + Environment.NewLine, parts);
|
||||
}
|
||||
|
||||
private static string ResolveApiKey(MeetingAssistantOptions options)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Key))
|
||||
{
|
||||
return options.Screenshots.Ocr.Key;
|
||||
}
|
||||
|
||||
var ocrKey = Environment.GetEnvironmentVariable(options.Screenshots.Ocr.KeyEnv);
|
||||
if (!string.IsNullOrWhiteSpace(ocrKey))
|
||||
{
|
||||
return ocrKey;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.Agent.Key))
|
||||
{
|
||||
return options.Agent.Key;
|
||||
}
|
||||
|
||||
var agentKey = Environment.GetEnvironmentVariable(options.Agent.KeyEnv);
|
||||
if (!string.IsNullOrWhiteSpace(agentKey))
|
||||
{
|
||||
return agentKey;
|
||||
}
|
||||
|
||||
throw new InvalidOperationException(
|
||||
$"No screenshot OCR API key configured. Set MeetingAssistant:Screenshots:Ocr:Key or environment variable '{options.Screenshots.Ocr.KeyEnv}'.");
|
||||
}
|
||||
|
||||
private static Uri NormalizeEndpoint(Uri endpoint)
|
||||
{
|
||||
var value = endpoint.ToString().TrimEnd('/');
|
||||
if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
value += "/v1";
|
||||
}
|
||||
|
||||
return new Uri(value + "/");
|
||||
}
|
||||
|
||||
[GeneratedRegex("```json\\s*(?<json>.*?)\\s*```", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
||||
private static partial Regex JsonCodeBlockRegex();
|
||||
}
|
||||
@@ -0,0 +1,482 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using System.Runtime.InteropServices;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
public interface IActiveWindowScreenshotCapture
|
||||
{
|
||||
Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record ActiveWindowScreenshot(byte[] PngBytes, string? WindowTitle);
|
||||
|
||||
public interface IScreenshotOcrClient
|
||||
{
|
||||
Task<ScreenshotOcrResult> ExtractAsync(
|
||||
string screenshotPath,
|
||||
string prompt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record ScreenshotOcrResult(string Text, ScreenshotCropCoordinates? Crop);
|
||||
|
||||
public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height);
|
||||
|
||||
public interface IMeetingScreenshotService
|
||||
{
|
||||
Task<MeetingScreenshotCaptureResult> CaptureAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
DateTimeOffset? meetingStartedAt,
|
||||
DateTimeOffset capturedAt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task WaitForPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task CancelPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record MeetingScreenshotCaptureResult(
|
||||
string ScreenshotPath,
|
||||
TimeSpan MeetingTimestamp,
|
||||
bool OcrStarted);
|
||||
|
||||
public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
|
||||
{
|
||||
public static NoopMeetingScreenshotService Instance { get; } = new();
|
||||
|
||||
public Task<MeetingScreenshotCaptureResult> CaptureAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
DateTimeOffset? meetingStartedAt,
|
||||
DateTimeOffset capturedAt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new MeetingScreenshotCaptureResult("", TimeSpan.Zero, OcrStarted: false));
|
||||
}
|
||||
|
||||
public Task WaitForPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task CancelPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
{
|
||||
private readonly IActiveWindowScreenshotCapture screenshotCapture;
|
||||
private readonly IMeetingArtifactStore artifactStore;
|
||||
private readonly IScreenshotOcrClient ocrClient;
|
||||
private readonly ILogger<MeetingScreenshotService> logger;
|
||||
private readonly ConcurrentDictionary<string, List<PendingOcrTask>> pendingOcrByContext = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> contextFileLocks = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public MeetingScreenshotService(
|
||||
IActiveWindowScreenshotCapture screenshotCapture,
|
||||
IMeetingArtifactStore artifactStore,
|
||||
IScreenshotOcrClient ocrClient,
|
||||
ILogger<MeetingScreenshotService> logger)
|
||||
{
|
||||
this.screenshotCapture = screenshotCapture;
|
||||
this.artifactStore = artifactStore;
|
||||
this.ocrClient = ocrClient;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<MeetingScreenshotCaptureResult> CaptureAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
DateTimeOffset? meetingStartedAt,
|
||||
DateTimeOffset capturedAt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var screenshot = await screenshotCapture.CaptureAsync(cancellationToken);
|
||||
var timestamp = CalculateMeetingTimestamp(meetingStartedAt, capturedAt);
|
||||
var screenshotPath = await SaveScreenshotAsync(
|
||||
artifacts,
|
||||
screenshot.PngBytes,
|
||||
timestamp,
|
||||
capturedAt,
|
||||
options,
|
||||
cancellationToken);
|
||||
var relativePath = ToMarkdownPath(Path.GetRelativePath(
|
||||
Path.GetDirectoryName(artifacts.AssistantContextPath)!,
|
||||
screenshotPath));
|
||||
var screenshotId = Guid.NewGuid().ToString("N");
|
||||
var ocrEnabled = options.Screenshots.Ocr.Enabled;
|
||||
await artifactStore.AppendAssistantContextAsync(
|
||||
artifacts,
|
||||
CreateScreenshotMarkdown(screenshotId, timestamp, relativePath, screenshot.WindowTitle, ocrEnabled),
|
||||
cancellationToken);
|
||||
|
||||
if (ocrEnabled)
|
||||
{
|
||||
var ocrCancellation = new CancellationTokenSource();
|
||||
TrackOcr(
|
||||
artifacts,
|
||||
ocrCancellation,
|
||||
RunOcrAsync(artifacts, screenshotPath, screenshotId, options, ocrCancellation.Token));
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Captured meeting screenshot {ScreenshotPath} at {MeetingTimestamp}",
|
||||
screenshotPath,
|
||||
FormatTimestamp(timestamp));
|
||||
return new MeetingScreenshotCaptureResult(screenshotPath, timestamp, ocrEnabled);
|
||||
}
|
||||
|
||||
public async Task WaitForPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var contextKey = NormalizeContextKey(artifacts.AssistantContextPath);
|
||||
if (!pendingOcrByContext.TryGetValue(contextKey, out var tasks))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
PendingOcrTask[] snapshot;
|
||||
lock (tasks)
|
||||
{
|
||||
tasks.RemoveAll(task => task.Task.IsCompleted);
|
||||
snapshot = tasks.ToArray();
|
||||
}
|
||||
|
||||
if (snapshot.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(snapshot.Select(task => task.Task)).WaitAsync(timeout, cancellationToken);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Timed out waiting for {Count} screenshot OCR task(s) for {AssistantContextPath}",
|
||||
snapshot.Length,
|
||||
artifacts.AssistantContextPath);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task CancelPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var contextKey = NormalizeContextKey(artifacts.AssistantContextPath);
|
||||
if (!pendingOcrByContext.TryGetValue(contextKey, out var tasks))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
PendingOcrTask[] snapshot;
|
||||
lock (tasks)
|
||||
{
|
||||
tasks.RemoveAll(task => task.Task.IsCompleted);
|
||||
snapshot = tasks.ToArray();
|
||||
}
|
||||
|
||||
if (snapshot.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var pending in snapshot)
|
||||
{
|
||||
await pending.Cancellation.CancelAsync();
|
||||
}
|
||||
|
||||
await WaitForPendingOcrAsync(artifacts, timeout, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<string> SaveScreenshotAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
byte[] pngBytes,
|
||||
TimeSpan timestamp,
|
||||
DateTimeOffset capturedAt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = ResolveAttachmentsFolder(artifacts.AssistantContextPath, options.Screenshots.AttachmentsFolder);
|
||||
Directory.CreateDirectory(folder);
|
||||
var path = Path.Combine(
|
||||
folder,
|
||||
$"{capturedAt:yyyyMMdd-HHmmss-fff}-{FormatTimestamp(timestamp).Replace(":", "", StringComparison.Ordinal)}.png");
|
||||
await File.WriteAllBytesAsync(path, pngBytes, cancellationToken);
|
||||
return path;
|
||||
}
|
||||
|
||||
private async Task RunOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
string screenshotPath,
|
||||
string screenshotId,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var timeout = options.Screenshots.Ocr.GetEffectiveTimeout();
|
||||
using var timeoutSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeoutSource.CancelAfter(timeout);
|
||||
try
|
||||
{
|
||||
var result = await ocrClient.ExtractAsync(
|
||||
screenshotPath,
|
||||
string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Prompt)
|
||||
? ScreenshotOcrOptions.DefaultPrompt
|
||||
: options.Screenshots.Ocr.Prompt,
|
||||
options,
|
||||
timeoutSource.Token);
|
||||
var cropMarkdown = await TryCreateCropMarkdownAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotPath,
|
||||
result.Crop,
|
||||
timeoutSource.Token);
|
||||
await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
cropMarkdown +
|
||||
"### OCR" + Environment.NewLine + Environment.NewLine + result.Text.Trim(),
|
||||
CancellationToken.None);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogInformation("Cancelled screenshot OCR for {ScreenshotPath}", screenshotPath);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
$"_OCR timed out after {timeout:g}._",
|
||||
CancellationToken.None);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Screenshot OCR failed for {ScreenshotPath}", screenshotPath);
|
||||
await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
$"_OCR failed: {exception.Message}_",
|
||||
CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReplaceOcrPlaceholderAsync(
|
||||
string assistantContextPath,
|
||||
string screenshotId,
|
||||
string replacement,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var fileLock = contextFileLocks.GetOrAdd(
|
||||
NormalizeContextKey(assistantContextPath),
|
||||
_ => new SemaphoreSlim(1, 1));
|
||||
await fileLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (!File.Exists(assistantContextPath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var content = await File.ReadAllTextAsync(assistantContextPath, cancellationToken);
|
||||
var startMarker = $"<!-- screenshot-ocr:{screenshotId} -->";
|
||||
var endMarker = $"<!-- /screenshot-ocr:{screenshotId} -->";
|
||||
var startIndex = content.IndexOf(startMarker, StringComparison.Ordinal);
|
||||
var endIndex = content.IndexOf(endMarker, StringComparison.Ordinal);
|
||||
if (startIndex < 0 || endIndex < startIndex)
|
||||
{
|
||||
await File.AppendAllTextAsync(
|
||||
assistantContextPath,
|
||||
Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
|
||||
cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
endIndex += endMarker.Length;
|
||||
var updated = content[..startIndex] +
|
||||
replacement.TrimEnd() +
|
||||
content[endIndex..];
|
||||
await File.WriteAllTextAsync(assistantContextPath, updated, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
fileLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> TryCreateCropMarkdownAsync(
|
||||
string assistantContextPath,
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates? crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (crop is null)
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Ignoring screenshot crop coordinates for {ScreenshotPath} because image cropping is only supported on Windows",
|
||||
screenshotPath);
|
||||
return "";
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var croppedPath = await SaveCroppedScreenshotAsync(screenshotPath, crop, cancellationToken);
|
||||
var relativePath = ToMarkdownPath(Path.GetRelativePath(
|
||||
Path.GetDirectoryName(assistantContextPath)!,
|
||||
croppedPath));
|
||||
return $"" +
|
||||
Environment.NewLine +
|
||||
Environment.NewLine;
|
||||
}
|
||||
catch (Exception exception) when (exception is InvalidDataException or ArgumentException or ExternalException)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Ignoring invalid screenshot crop coordinates for {ScreenshotPath}",
|
||||
screenshotPath);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
#pragma warning disable CA1416
|
||||
private static Task<string> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
using var original = new Bitmap(screenshotPath);
|
||||
ValidateCrop(crop, original.Width, original.Height);
|
||||
var cropRectangle = new Rectangle(crop.X, crop.Y, crop.Width, crop.Height);
|
||||
using var cropped = original.Clone(cropRectangle, original.PixelFormat);
|
||||
var croppedPath = Path.Combine(
|
||||
Path.GetDirectoryName(screenshotPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
|
||||
cropped.Save(croppedPath, ImageFormat.Png);
|
||||
return Task.FromResult(croppedPath);
|
||||
}
|
||||
#pragma warning restore CA1416
|
||||
|
||||
private static void ValidateCrop(ScreenshotCropCoordinates crop, int imageWidth, int imageHeight)
|
||||
{
|
||||
if (crop.X < 0 ||
|
||||
crop.Y < 0 ||
|
||||
crop.Width <= 0 ||
|
||||
crop.Height <= 0 ||
|
||||
crop.X + crop.Width > imageWidth ||
|
||||
crop.Y + crop.Height > imageHeight)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} is outside image bounds {imageWidth}x{imageHeight}.");
|
||||
}
|
||||
}
|
||||
|
||||
private void TrackOcr(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
CancellationTokenSource cancellation,
|
||||
Task task)
|
||||
{
|
||||
var contextKey = NormalizeContextKey(artifacts.AssistantContextPath);
|
||||
var tasks = pendingOcrByContext.GetOrAdd(contextKey, _ => []);
|
||||
lock (tasks)
|
||||
{
|
||||
tasks.RemoveAll(existing => existing.Task.IsCompleted);
|
||||
tasks.Add(new PendingOcrTask(task, cancellation));
|
||||
}
|
||||
}
|
||||
|
||||
private static string CreateScreenshotMarkdown(
|
||||
string screenshotId,
|
||||
TimeSpan timestamp,
|
||||
string relativePath,
|
||||
string? windowTitle,
|
||||
bool ocrEnabled)
|
||||
{
|
||||
var title = string.IsNullOrWhiteSpace(windowTitle)
|
||||
? ""
|
||||
: Environment.NewLine + $"Window: {windowTitle.Trim()}" + Environment.NewLine;
|
||||
var timestampText = FormatTimestamp(timestamp);
|
||||
var content = $"## Screenshot [{timestampText}]" +
|
||||
title +
|
||||
Environment.NewLine +
|
||||
$"";
|
||||
if (!ocrEnabled)
|
||||
{
|
||||
return content;
|
||||
}
|
||||
|
||||
return content +
|
||||
Environment.NewLine +
|
||||
Environment.NewLine +
|
||||
$"<!-- screenshot-ocr:{screenshotId} -->" +
|
||||
Environment.NewLine +
|
||||
"_OCR pending..._" +
|
||||
Environment.NewLine +
|
||||
$"<!-- /screenshot-ocr:{screenshotId} -->";
|
||||
}
|
||||
|
||||
private static string ResolveAttachmentsFolder(string assistantContextPath, string configuredFolder)
|
||||
{
|
||||
var expanded = Environment.ExpandEnvironmentVariables(
|
||||
string.IsNullOrWhiteSpace(configuredFolder) ? "Attachments" : configuredFolder);
|
||||
return Path.IsPathRooted(expanded)
|
||||
? Path.GetFullPath(expanded)
|
||||
: Path.GetFullPath(Path.Combine(Path.GetDirectoryName(assistantContextPath)!, expanded));
|
||||
}
|
||||
|
||||
private static TimeSpan CalculateMeetingTimestamp(DateTimeOffset? meetingStartedAt, DateTimeOffset capturedAt)
|
||||
{
|
||||
if (meetingStartedAt is null)
|
||||
{
|
||||
return TimeSpan.Zero;
|
||||
}
|
||||
|
||||
var elapsed = capturedAt - meetingStartedAt.Value;
|
||||
return elapsed < TimeSpan.Zero ? TimeSpan.Zero : elapsed;
|
||||
}
|
||||
|
||||
private static string FormatTimestamp(TimeSpan timestamp)
|
||||
{
|
||||
return $"{(int)timestamp.TotalHours:00}:{timestamp.Minutes:00}:{timestamp.Seconds:00}";
|
||||
}
|
||||
|
||||
private static string ToMarkdownPath(string path)
|
||||
{
|
||||
return path.Replace(Path.DirectorySeparatorChar, '/')
|
||||
.Replace(Path.AltDirectorySeparatorChar, '/');
|
||||
}
|
||||
|
||||
private static string NormalizeContextKey(string assistantContextPath)
|
||||
{
|
||||
return Path.GetFullPath(assistantContextPath);
|
||||
}
|
||||
|
||||
private sealed record PendingOcrTask(Task Task, CancellationTokenSource Cancellation);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
public sealed class UnavailableActiveWindowScreenshotCapture : IActiveWindowScreenshotCapture
|
||||
{
|
||||
public Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
throw new PlatformNotSupportedException("Active-window screenshot capture is only available on Windows.");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user