Public Access
Archive completed meeting assistant changes
PR and Push Build/Test / build-and-test (push) Successful in 9m19s
PR and Push Build/Test / build-and-test (push) Successful in 9m19s
This commit is contained in:
@@ -3,6 +3,7 @@ using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.RegularExpressions;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
@@ -104,41 +105,69 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
||||
private static ScreenshotOcrResult ParseOcrResult(string text)
|
||||
{
|
||||
ScreenshotCropCoordinates? crop = null;
|
||||
IReadOnlyList<string> attendees = [];
|
||||
var cleaned = JsonCodeBlockRegex().Replace(text, match =>
|
||||
{
|
||||
if (TryParseCrop(match.Groups["json"].Value, out var parsedCrop))
|
||||
if (TryParseMetadata(match.Groups["json"].Value, out var parsedCrop, out var parsedAttendees))
|
||||
{
|
||||
crop = parsedCrop;
|
||||
attendees = parsedAttendees;
|
||||
return "";
|
||||
}
|
||||
|
||||
return match.Value;
|
||||
}).Trim();
|
||||
return new ScreenshotOcrResult(cleaned, crop);
|
||||
return new ScreenshotOcrResult(cleaned, crop, attendees);
|
||||
}
|
||||
|
||||
private static bool TryParseCrop(string json, out ScreenshotCropCoordinates? crop)
|
||||
private static bool TryParseMetadata(
|
||||
string json,
|
||||
out ScreenshotCropCoordinates? crop,
|
||||
out IReadOnlyList<string> attendees)
|
||||
{
|
||||
crop = null;
|
||||
attendees = [];
|
||||
try
|
||||
{
|
||||
using var document = JsonDocument.Parse(json);
|
||||
if (!document.RootElement.TryGetProperty("crop", out var cropElement) ||
|
||||
cropElement.ValueKind == JsonValueKind.Null)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (cropElement.ValueKind != JsonValueKind.Object ||
|
||||
!TryGetInt(cropElement, "x", out var x) ||
|
||||
!TryGetInt(cropElement, "y", out var y) ||
|
||||
!TryGetInt(cropElement, "width", out var width) ||
|
||||
!TryGetInt(cropElement, "height", out var height))
|
||||
if (document.RootElement.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
crop = new ScreenshotCropCoordinates(x, y, width, height);
|
||||
var hasMetadata = false;
|
||||
if (document.RootElement.TryGetProperty("attendees", out var attendeesElement))
|
||||
{
|
||||
hasMetadata = true;
|
||||
if (attendeesElement.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
attendees = MeetingAttendeeNames.NormalizeDistinct(attendeesElement
|
||||
.EnumerateArray()
|
||||
.Where(element => element.ValueKind == JsonValueKind.String)
|
||||
.Select(element => element.GetString() ?? ""));
|
||||
}
|
||||
}
|
||||
|
||||
if (!document.RootElement.TryGetProperty("crop", out var cropElement))
|
||||
{
|
||||
return hasMetadata;
|
||||
}
|
||||
|
||||
hasMetadata = true;
|
||||
if (cropElement.ValueKind == JsonValueKind.Null)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (cropElement.ValueKind == JsonValueKind.Object &&
|
||||
TryGetInt(cropElement, "x", out var x) &&
|
||||
TryGetInt(cropElement, "y", out var y) &&
|
||||
TryGetInt(cropElement, "width", out var width) &&
|
||||
TryGetInt(cropElement, "height", out var height))
|
||||
{
|
||||
crop = new ScreenshotCropCoordinates(x, y, width, height);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
catch (JsonException)
|
||||
|
||||
@@ -2,7 +2,9 @@ using System.Collections.Concurrent;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text.RegularExpressions;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Speakers;
|
||||
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
@@ -22,7 +24,24 @@ public interface IScreenshotOcrClient
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record ScreenshotOcrResult(string Text, ScreenshotCropCoordinates? Crop);
|
||||
public sealed record ScreenshotOcrResult
|
||||
{
|
||||
public ScreenshotOcrResult(
|
||||
string text,
|
||||
ScreenshotCropCoordinates? crop,
|
||||
IReadOnlyList<string>? attendees = null)
|
||||
{
|
||||
Text = text;
|
||||
Crop = crop;
|
||||
Attendees = attendees ?? [];
|
||||
}
|
||||
|
||||
public string Text { get; init; }
|
||||
|
||||
public ScreenshotCropCoordinates? Crop { get; init; }
|
||||
|
||||
public IReadOnlyList<string> Attendees { get; init; }
|
||||
}
|
||||
|
||||
public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height);
|
||||
|
||||
@@ -46,11 +65,36 @@ public interface IMeetingScreenshotService
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public interface IMeetingNoteImageOcrService
|
||||
{
|
||||
Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public interface IScreenshotOcrRetryRunner
|
||||
{
|
||||
Task<MeetingScreenshotOcrRetryTriggerResult?> TriggerOcrRetryAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
string screenshotPath,
|
||||
string screenshotId,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record MeetingScreenshotCaptureResult(
|
||||
string ScreenshotPath,
|
||||
TimeSpan MeetingTimestamp,
|
||||
bool OcrStarted);
|
||||
|
||||
public sealed record MeetingScreenshotOcrRetryTriggerResult(
|
||||
string AssistantContextPath,
|
||||
string ScreenshotPath,
|
||||
string ScreenshotId);
|
||||
|
||||
public sealed record MeetingNoteImageOcrQueueResult(int QueuedCount);
|
||||
|
||||
public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
|
||||
{
|
||||
public static NoopMeetingScreenshotService Instance { get; } = new();
|
||||
@@ -80,12 +124,31 @@ public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
public sealed class NoopMeetingNoteImageOcrService : IMeetingNoteImageOcrService
|
||||
{
|
||||
public static NoopMeetingNoteImageOcrService Instance { get; } = new();
|
||||
|
||||
public Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new MeetingNoteImageOcrQueueResult(0));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed partial class MeetingScreenshotService :
|
||||
IMeetingScreenshotService,
|
||||
IScreenshotOcrRetryRunner,
|
||||
IMeetingNoteImageOcrService
|
||||
{
|
||||
private readonly IActiveWindowScreenshotCapture screenshotCapture;
|
||||
private readonly IMeetingArtifactStore artifactStore;
|
||||
private readonly IMeetingNoteStore meetingNoteStore;
|
||||
private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer;
|
||||
private readonly IScreenshotOcrClient ocrClient;
|
||||
private readonly ILogger<MeetingScreenshotService> logger;
|
||||
private readonly ConcurrentDictionary<string, List<PendingOcrTask>> pendingOcrByContext = new(StringComparer.OrdinalIgnoreCase);
|
||||
@@ -94,11 +157,15 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
public MeetingScreenshotService(
|
||||
IActiveWindowScreenshotCapture screenshotCapture,
|
||||
IMeetingArtifactStore artifactStore,
|
||||
IMeetingNoteStore meetingNoteStore,
|
||||
ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer,
|
||||
IScreenshotOcrClient ocrClient,
|
||||
ILogger<MeetingScreenshotService> logger)
|
||||
{
|
||||
this.screenshotCapture = screenshotCapture;
|
||||
this.artifactStore = artifactStore;
|
||||
this.meetingNoteStore = meetingNoteStore;
|
||||
this.attendeeCanonicalizer = attendeeCanonicalizer;
|
||||
this.ocrClient = ocrClient;
|
||||
this.logger = logger;
|
||||
}
|
||||
@@ -135,7 +202,13 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
TrackOcr(
|
||||
artifacts,
|
||||
ocrCancellation,
|
||||
RunOcrAsync(artifacts, screenshotPath, screenshotId, options, ocrCancellation.Token));
|
||||
RunOcrAsync(
|
||||
artifacts,
|
||||
screenshotPath,
|
||||
screenshotId,
|
||||
options,
|
||||
ScreenshotOcrPolicy.CapturedScreenshot,
|
||||
ocrCancellation.Token));
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
@@ -212,6 +285,100 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
await WaitForPendingOcrAsync(artifacts, timeout, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<MeetingScreenshotOcrRetryTriggerResult?> TriggerOcrRetryAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
string screenshotPath,
|
||||
string screenshotId,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(screenshotId) ||
|
||||
string.IsNullOrWhiteSpace(screenshotPath) ||
|
||||
!File.Exists(screenshotPath) ||
|
||||
!File.Exists(artifacts.AssistantContextPath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var replaced = await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
"_OCR retry pending..._",
|
||||
cancellationToken,
|
||||
preserveMarkers: true,
|
||||
appendWhenMissing: false);
|
||||
if (!replaced)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var ocrCancellation = new CancellationTokenSource();
|
||||
TrackOcr(
|
||||
artifacts,
|
||||
ocrCancellation,
|
||||
RunOcrAsync(
|
||||
artifacts,
|
||||
screenshotPath,
|
||||
screenshotId,
|
||||
options,
|
||||
ScreenshotOcrPolicy.CapturedScreenshot,
|
||||
ocrCancellation.Token));
|
||||
return new MeetingScreenshotOcrRetryTriggerResult(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotPath,
|
||||
screenshotId);
|
||||
}
|
||||
|
||||
public async Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Screenshots.Ocr.Enabled || !File.Exists(artifacts.MeetingNotePath))
|
||||
{
|
||||
return new MeetingNoteImageOcrQueueResult(0);
|
||||
}
|
||||
|
||||
var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken);
|
||||
var embeds = FindMeetingNoteImageEmbeds(
|
||||
meetingNote.UserNotes,
|
||||
artifacts.MeetingNotePath,
|
||||
options);
|
||||
var queuedEmbeds = new List<(MeetingNoteImageEmbed Embed, string OcrId)>();
|
||||
foreach (var embed in embeds)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var ocrId = Guid.NewGuid().ToString("N");
|
||||
await artifactStore.AppendAssistantContextAsync(
|
||||
artifacts,
|
||||
CreateMeetingNoteImageMarkdown(
|
||||
ocrId,
|
||||
embed.OriginalEmbed,
|
||||
embed.ImagePath,
|
||||
artifacts.AssistantContextPath),
|
||||
cancellationToken);
|
||||
queuedEmbeds.Add((embed, ocrId));
|
||||
}
|
||||
|
||||
foreach (var (embed, ocrId) in queuedEmbeds)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var ocrCancellation = new CancellationTokenSource();
|
||||
TrackOcr(
|
||||
artifacts,
|
||||
ocrCancellation,
|
||||
RunOcrAsync(
|
||||
artifacts,
|
||||
embed.ImagePath,
|
||||
ocrId,
|
||||
options,
|
||||
ScreenshotOcrPolicy.MeetingNoteImage,
|
||||
ocrCancellation.Token));
|
||||
}
|
||||
|
||||
return new MeetingNoteImageOcrQueueResult(queuedEmbeds.Count);
|
||||
}
|
||||
|
||||
private async Task<string> SaveScreenshotAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
byte[] pngBytes,
|
||||
@@ -234,6 +401,7 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
string screenshotPath,
|
||||
string screenshotId,
|
||||
MeetingAssistantOptions options,
|
||||
ScreenshotOcrPolicy policy,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var timeout = options.Screenshots.Ocr.GetEffectiveTimeout();
|
||||
@@ -248,17 +416,26 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
: options.Screenshots.Ocr.Prompt,
|
||||
options,
|
||||
timeoutSource.Token);
|
||||
var cropMarkdown = await TryCreateCropMarkdownAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotPath,
|
||||
result.Crop,
|
||||
timeoutSource.Token);
|
||||
var cropMarkdown = policy.AllowCrop
|
||||
? await TryCreateCropMarkdownAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotPath,
|
||||
result.Crop,
|
||||
timeoutSource.Token)
|
||||
: "";
|
||||
await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
cropMarkdown +
|
||||
"### OCR" + Environment.NewLine + Environment.NewLine + result.Text.Trim(),
|
||||
CancellationToken.None);
|
||||
if (policy.AddAttendees)
|
||||
{
|
||||
await AddOcrAttendeesAsync(
|
||||
artifacts,
|
||||
result.Attendees,
|
||||
CancellationToken.None);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -269,8 +446,15 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
$"_OCR timed out after {timeout:g}._",
|
||||
CancellationToken.None);
|
||||
CreateOcrFailureMarkdown(
|
||||
artifacts,
|
||||
screenshotPath,
|
||||
screenshotId,
|
||||
options,
|
||||
$"_OCR timed out after {timeout:g}._",
|
||||
policy.IncludeRetryLink),
|
||||
CancellationToken.None,
|
||||
preserveMarkers: true);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
@@ -278,16 +462,63 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
$"_OCR failed: {exception.Message}_",
|
||||
CancellationToken.None);
|
||||
CreateOcrFailureMarkdown(
|
||||
artifacts,
|
||||
screenshotPath,
|
||||
screenshotId,
|
||||
options,
|
||||
$"_OCR failed: {exception.Message}_",
|
||||
policy.IncludeRetryLink),
|
||||
CancellationToken.None,
|
||||
preserveMarkers: true);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReplaceOcrPlaceholderAsync(
|
||||
private async Task AddOcrAttendeesAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
IReadOnlyList<string> attendees,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var additions = MeetingAttendeeNames.NormalizeDistinct(attendees);
|
||||
if (additions.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken);
|
||||
var canonicalized = await attendeeCanonicalizer.CanonicalizeAsync(
|
||||
meetingNote.Frontmatter.Attendees.Concat(additions).ToList(),
|
||||
cancellationToken);
|
||||
if (meetingNote.Frontmatter.Attendees.SequenceEqual(canonicalized, StringComparer.Ordinal))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
meetingNote.Frontmatter.Attendees = canonicalized.ToList();
|
||||
await meetingNoteStore.SaveAsync(meetingNote, cancellationToken);
|
||||
logger.LogInformation(
|
||||
"Added {AttendeeCount} screenshot OCR attendee candidate(s) to meeting note {MeetingNotePath}",
|
||||
additions.Count,
|
||||
artifacts.MeetingNotePath);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Could not add screenshot OCR attendees to meeting note {MeetingNotePath}",
|
||||
artifacts.MeetingNotePath);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> ReplaceOcrPlaceholderAsync(
|
||||
string assistantContextPath,
|
||||
string screenshotId,
|
||||
string replacement,
|
||||
CancellationToken cancellationToken)
|
||||
CancellationToken cancellationToken,
|
||||
bool preserveMarkers = false,
|
||||
bool appendWhenMissing = true)
|
||||
{
|
||||
var fileLock = contextFileLocks.GetOrAdd(
|
||||
NormalizeContextKey(assistantContextPath),
|
||||
@@ -297,28 +528,41 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
{
|
||||
if (!File.Exists(assistantContextPath))
|
||||
{
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
var content = await File.ReadAllTextAsync(assistantContextPath, cancellationToken);
|
||||
var startMarker = $"<!-- screenshot-ocr:{screenshotId} -->";
|
||||
var endMarker = $"<!-- /screenshot-ocr:{screenshotId} -->";
|
||||
var startMarker = CreateOcrStartMarker(screenshotId);
|
||||
var endMarker = CreateOcrEndMarker(screenshotId);
|
||||
var startIndex = content.IndexOf(startMarker, StringComparison.Ordinal);
|
||||
var endIndex = content.IndexOf(endMarker, StringComparison.Ordinal);
|
||||
if (startIndex < 0 || endIndex < startIndex)
|
||||
{
|
||||
await File.AppendAllTextAsync(
|
||||
assistantContextPath,
|
||||
Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
|
||||
cancellationToken);
|
||||
return;
|
||||
if (appendWhenMissing)
|
||||
{
|
||||
await File.AppendAllTextAsync(
|
||||
assistantContextPath,
|
||||
Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
endIndex += endMarker.Length;
|
||||
var updated = content[..startIndex] +
|
||||
replacement.TrimEnd() +
|
||||
content[endIndex..];
|
||||
var updated = preserveMarkers
|
||||
? content[..startIndex] +
|
||||
startMarker +
|
||||
Environment.NewLine +
|
||||
replacement.TrimEnd() +
|
||||
Environment.NewLine +
|
||||
endMarker +
|
||||
content[endIndex..]
|
||||
: content[..startIndex] +
|
||||
replacement.TrimEnd() +
|
||||
content[endIndex..];
|
||||
await File.WriteAllTextAsync(assistantContextPath, updated, cancellationToken);
|
||||
return true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -326,6 +570,29 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
}
|
||||
}
|
||||
|
||||
private static string CreateOcrFailureMarkdown(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
string screenshotPath,
|
||||
string screenshotId,
|
||||
MeetingAssistantOptions options,
|
||||
string status,
|
||||
bool includeRetryLink = true)
|
||||
{
|
||||
var markdown = status.TrimEnd();
|
||||
if (includeRetryLink)
|
||||
{
|
||||
markdown += Environment.NewLine +
|
||||
Environment.NewLine +
|
||||
MeetingNoteActionLinks.CreateScreenshotOcrRetryLink(
|
||||
options.Api.PublicBaseUrl,
|
||||
artifacts,
|
||||
screenshotPath,
|
||||
screenshotId);
|
||||
}
|
||||
|
||||
return markdown;
|
||||
}
|
||||
|
||||
private async Task<string> TryCreateCropMarkdownAsync(
|
||||
string assistantContextPath,
|
||||
string screenshotPath,
|
||||
@@ -435,13 +702,189 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
return content +
|
||||
Environment.NewLine +
|
||||
Environment.NewLine +
|
||||
$"<!-- screenshot-ocr:{screenshotId} -->" +
|
||||
Environment.NewLine +
|
||||
"_OCR pending..._" +
|
||||
Environment.NewLine +
|
||||
$"<!-- /screenshot-ocr:{screenshotId} -->";
|
||||
CreateOcrBlock(screenshotId, "_OCR pending..._");
|
||||
}
|
||||
|
||||
private static string CreateMeetingNoteImageMarkdown(
|
||||
string ocrId,
|
||||
string originalEmbed,
|
||||
string imagePath,
|
||||
string assistantContextPath)
|
||||
{
|
||||
var relativePath = ToMarkdownPath(Path.GetRelativePath(
|
||||
Path.GetDirectoryName(assistantContextPath)!,
|
||||
imagePath));
|
||||
return "## Meeting Note Image" +
|
||||
Environment.NewLine +
|
||||
"Image from meeting note." +
|
||||
Environment.NewLine +
|
||||
$"Original embed: `{originalEmbed.Replace("`", "\\`", StringComparison.Ordinal)}`" +
|
||||
Environment.NewLine +
|
||||
$"" +
|
||||
Environment.NewLine +
|
||||
Environment.NewLine +
|
||||
CreateOcrBlock(ocrId, "_OCR pending..._");
|
||||
}
|
||||
|
||||
private static string CreateOcrBlock(string screenshotId, string content)
|
||||
{
|
||||
return CreateOcrStartMarker(screenshotId) +
|
||||
Environment.NewLine +
|
||||
content.TrimEnd() +
|
||||
Environment.NewLine +
|
||||
CreateOcrEndMarker(screenshotId);
|
||||
}
|
||||
|
||||
private static string CreateOcrStartMarker(string screenshotId)
|
||||
{
|
||||
return $"<!-- screenshot-ocr:{screenshotId} -->";
|
||||
}
|
||||
|
||||
private static string CreateOcrEndMarker(string screenshotId)
|
||||
{
|
||||
return $"<!-- /screenshot-ocr:{screenshotId} -->";
|
||||
}
|
||||
|
||||
private static IReadOnlyList<MeetingNoteImageEmbed> FindMeetingNoteImageEmbeds(
|
||||
string userNotes,
|
||||
string meetingNotePath,
|
||||
MeetingAssistantOptions options)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(userNotes))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var result = new List<MeetingNoteImageEmbed>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (Match match in ObsidianImageEmbedRegex().Matches(userNotes))
|
||||
{
|
||||
AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options);
|
||||
}
|
||||
|
||||
foreach (Match match in MarkdownImageEmbedRegex().Matches(userNotes))
|
||||
{
|
||||
AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static void AddResolvedEmbed(
|
||||
List<MeetingNoteImageEmbed> embeds,
|
||||
HashSet<string> seen,
|
||||
string originalEmbed,
|
||||
string target,
|
||||
string meetingNotePath,
|
||||
MeetingAssistantOptions options)
|
||||
{
|
||||
var imagePath = ResolveMeetingNoteImagePath(target, meetingNotePath, options);
|
||||
if (imagePath is null || !seen.Add(imagePath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
embeds.Add(new MeetingNoteImageEmbed(originalEmbed, imagePath));
|
||||
}
|
||||
|
||||
private static string? ResolveMeetingNoteImagePath(
|
||||
string target,
|
||||
string meetingNotePath,
|
||||
MeetingAssistantOptions options)
|
||||
{
|
||||
var cleaned = CleanEmbedTarget(target);
|
||||
if (string.IsNullOrWhiteSpace(cleaned) ||
|
||||
cleaned.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
|
||||
cleaned.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ||
|
||||
cleaned.StartsWith("data:", StringComparison.OrdinalIgnoreCase) ||
|
||||
!IsSupportedImagePath(cleaned))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var candidates = new List<string>();
|
||||
if (Path.IsPathRooted(cleaned))
|
||||
{
|
||||
candidates.Add(VaultPath.Resolve(cleaned));
|
||||
}
|
||||
else
|
||||
{
|
||||
candidates.Add(Path.GetFullPath(Path.Combine(Path.GetDirectoryName(meetingNotePath)!, cleaned)));
|
||||
candidates.Add(VaultPath.Resolve(options.Vault, cleaned));
|
||||
}
|
||||
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (File.Exists(candidate))
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
if (Path.GetFileName(cleaned).Equals(cleaned, StringComparison.Ordinal))
|
||||
{
|
||||
try
|
||||
{
|
||||
var vaultRoot = VaultPath.Resolve(options.Vault.BaseFolder);
|
||||
return Directory.EnumerateFiles(vaultRoot, cleaned, SearchOption.AllDirectories)
|
||||
.FirstOrDefault(IsSupportedImagePath);
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or DirectoryNotFoundException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string CleanEmbedTarget(string target)
|
||||
{
|
||||
var cleaned = target.Trim();
|
||||
if (cleaned.StartsWith("<", StringComparison.Ordinal) &&
|
||||
cleaned.EndsWith(">", StringComparison.Ordinal) &&
|
||||
cleaned.Length > 1)
|
||||
{
|
||||
cleaned = cleaned[1..^1].Trim();
|
||||
}
|
||||
|
||||
var pipeIndex = cleaned.IndexOf('|', StringComparison.Ordinal);
|
||||
if (pipeIndex >= 0)
|
||||
{
|
||||
cleaned = cleaned[..pipeIndex].Trim();
|
||||
}
|
||||
|
||||
var hashIndex = cleaned.IndexOf('#', StringComparison.Ordinal);
|
||||
if (hashIndex >= 0)
|
||||
{
|
||||
cleaned = cleaned[..hashIndex].Trim();
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return Uri.UnescapeDataString(cleaned.Replace('/', Path.DirectorySeparatorChar));
|
||||
}
|
||||
catch (UriFormatException)
|
||||
{
|
||||
return cleaned.Replace('/', Path.DirectorySeparatorChar);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsSupportedImagePath(string path)
|
||||
{
|
||||
return Path.GetExtension(path).ToLowerInvariant() switch
|
||||
{
|
||||
".png" or ".jpg" or ".jpeg" or ".gif" or ".webp" or ".bmp" or ".tif" or ".tiff" => true,
|
||||
_ => false
|
||||
};
|
||||
}
|
||||
|
||||
[GeneratedRegex(@"!\[\[(?<target>[^\]\r\n]+)\]\]", RegexOptions.CultureInvariant)]
|
||||
private static partial Regex ObsidianImageEmbedRegex();
|
||||
|
||||
[GeneratedRegex(@"!\[[^\]\r\n]*\]\((?<target>[^)\r\n]+)\)", RegexOptions.CultureInvariant)]
|
||||
private static partial Regex MarkdownImageEmbedRegex();
|
||||
|
||||
private static string ResolveAttachmentsFolder(string assistantContextPath, string configuredFolder)
|
||||
{
|
||||
var expanded = Environment.ExpandEnvironmentVariables(
|
||||
@@ -479,4 +922,19 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
}
|
||||
|
||||
private sealed record PendingOcrTask(Task Task, CancellationTokenSource Cancellation);
|
||||
|
||||
private sealed record ScreenshotOcrPolicy(bool AllowCrop, bool AddAttendees, bool IncludeRetryLink)
|
||||
{
|
||||
public static ScreenshotOcrPolicy CapturedScreenshot { get; } = new(
|
||||
AllowCrop: true,
|
||||
AddAttendees: true,
|
||||
IncludeRetryLink: true);
|
||||
|
||||
public static ScreenshotOcrPolicy MeetingNoteImage { get; } = new(
|
||||
AllowCrop: false,
|
||||
AddAttendees: false,
|
||||
IncludeRetryLink: false);
|
||||
}
|
||||
|
||||
private sealed record MeetingNoteImageEmbed(string OriginalEmbed, string ImagePath);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user