Archive completed meeting assistant changes

MeetingAssistant/Screenshots/MeetingScreenshotService.cs

@@ -2,7 +2,9 @@ using System.Collections.Concurrent;
 using System.Drawing;
 using System.Drawing.Imaging;
 using System.Runtime.InteropServices;
+using System.Text.RegularExpressions;
 using MeetingAssistant.MeetingNotes;
+using MeetingAssistant.Speakers;
 
 namespace MeetingAssistant.Screenshots;
 
@@ -22,7 +24,24 @@ public interface IScreenshotOcrClient
         CancellationToken cancellationToken);
 }
 
-public sealed record ScreenshotOcrResult(string Text, ScreenshotCropCoordinates? Crop);
+public sealed record ScreenshotOcrResult
+{
+    public ScreenshotOcrResult(
+        string text,
+        ScreenshotCropCoordinates? crop,
+        IReadOnlyList<string>? attendees = null)
+    {
+        Text = text;
+        Crop = crop;
+        Attendees = attendees ?? [];
+    }
+
+    public string Text { get; init; }
+
+    public ScreenshotCropCoordinates? Crop { get; init; }
+
+    public IReadOnlyList<string> Attendees { get; init; }
+}
 
 public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height);
 
@@ -46,11 +65,36 @@ public interface IMeetingScreenshotService
         CancellationToken cancellationToken);
 }
 
+public interface IMeetingNoteImageOcrService
+{
+    Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
+        MeetingSessionArtifacts artifacts,
+        MeetingAssistantOptions options,
+        CancellationToken cancellationToken);
+}
+
+public interface IScreenshotOcrRetryRunner
+{
+    Task<MeetingScreenshotOcrRetryTriggerResult?> TriggerOcrRetryAsync(
+        MeetingSessionArtifacts artifacts,
+        string screenshotPath,
+        string screenshotId,
+        MeetingAssistantOptions options,
+        CancellationToken cancellationToken);
+}
+
 public sealed record MeetingScreenshotCaptureResult(
     string ScreenshotPath,
     TimeSpan MeetingTimestamp,
     bool OcrStarted);
 
+public sealed record MeetingScreenshotOcrRetryTriggerResult(
+    string AssistantContextPath,
+    string ScreenshotPath,
+    string ScreenshotId);
+
+public sealed record MeetingNoteImageOcrQueueResult(int QueuedCount);
+
 public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
 {
     public static NoopMeetingScreenshotService Instance { get; } = new();
@@ -80,12 +124,31 @@ public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
     {
         return Task.CompletedTask;
     }
+
 }
 
-public sealed class MeetingScreenshotService : IMeetingScreenshotService
+public sealed class NoopMeetingNoteImageOcrService : IMeetingNoteImageOcrService
+{
+    public static NoopMeetingNoteImageOcrService Instance { get; } = new();
+
+    public Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
+        MeetingSessionArtifacts artifacts,
+        MeetingAssistantOptions options,
+        CancellationToken cancellationToken)
+    {
+        return Task.FromResult(new MeetingNoteImageOcrQueueResult(0));
+    }
+}
+
+public sealed partial class MeetingScreenshotService :
+    IMeetingScreenshotService,
+    IScreenshotOcrRetryRunner,
+    IMeetingNoteImageOcrService
 {
     private readonly IActiveWindowScreenshotCapture screenshotCapture;
     private readonly IMeetingArtifactStore artifactStore;
+    private readonly IMeetingNoteStore meetingNoteStore;
+    private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer;
     private readonly IScreenshotOcrClient ocrClient;
     private readonly ILogger<MeetingScreenshotService> logger;
     private readonly ConcurrentDictionary<string, List<PendingOcrTask>> pendingOcrByContext = new(StringComparer.OrdinalIgnoreCase);
@@ -94,11 +157,15 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
     public MeetingScreenshotService(
         IActiveWindowScreenshotCapture screenshotCapture,
         IMeetingArtifactStore artifactStore,
+        IMeetingNoteStore meetingNoteStore,
+        ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer,
         IScreenshotOcrClient ocrClient,
         ILogger<MeetingScreenshotService> logger)
     {
         this.screenshotCapture = screenshotCapture;
         this.artifactStore = artifactStore;
+        this.meetingNoteStore = meetingNoteStore;
+        this.attendeeCanonicalizer = attendeeCanonicalizer;
         this.ocrClient = ocrClient;
         this.logger = logger;
     }
@@ -135,7 +202,13 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
             TrackOcr(
                 artifacts,
                 ocrCancellation,
-                RunOcrAsync(artifacts, screenshotPath, screenshotId, options, ocrCancellation.Token));
+                RunOcrAsync(
+                    artifacts,
+                    screenshotPath,
+                    screenshotId,
+                    options,
+                    ScreenshotOcrPolicy.CapturedScreenshot,
+                    ocrCancellation.Token));
         }
 
         logger.LogInformation(
@@ -212,6 +285,100 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
         await WaitForPendingOcrAsync(artifacts, timeout, cancellationToken);
     }
 
+    public async Task<MeetingScreenshotOcrRetryTriggerResult?> TriggerOcrRetryAsync(
+        MeetingSessionArtifacts artifacts,
+        string screenshotPath,
+        string screenshotId,
+        MeetingAssistantOptions options,
+        CancellationToken cancellationToken)
+    {
+        if (string.IsNullOrWhiteSpace(screenshotId) ||
+            string.IsNullOrWhiteSpace(screenshotPath) ||
+            !File.Exists(screenshotPath) ||
+            !File.Exists(artifacts.AssistantContextPath))
+        {
+            return null;
+        }
+
+        var replaced = await ReplaceOcrPlaceholderAsync(
+            artifacts.AssistantContextPath,
+            screenshotId,
+            "_OCR retry pending..._",
+            cancellationToken,
+            preserveMarkers: true,
+            appendWhenMissing: false);
+        if (!replaced)
+        {
+            return null;
+        }
+
+        var ocrCancellation = new CancellationTokenSource();
+        TrackOcr(
+                artifacts,
+                ocrCancellation,
+                RunOcrAsync(
+                    artifacts,
+                    screenshotPath,
+                    screenshotId,
+                    options,
+                    ScreenshotOcrPolicy.CapturedScreenshot,
+                    ocrCancellation.Token));
+        return new MeetingScreenshotOcrRetryTriggerResult(
+            artifacts.AssistantContextPath,
+            screenshotPath,
+            screenshotId);
+    }
+
+    public async Task<MeetingNoteImageOcrQueueResult> ProcessMeetingNoteImageEmbedsAsync(
+        MeetingSessionArtifacts artifacts,
+        MeetingAssistantOptions options,
+        CancellationToken cancellationToken)
+    {
+        if (!options.Screenshots.Ocr.Enabled || !File.Exists(artifacts.MeetingNotePath))
+        {
+            return new MeetingNoteImageOcrQueueResult(0);
+        }
+
+        var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken);
+        var embeds = FindMeetingNoteImageEmbeds(
+            meetingNote.UserNotes,
+            artifacts.MeetingNotePath,
+            options);
+        var queuedEmbeds = new List<(MeetingNoteImageEmbed Embed, string OcrId)>();
+        foreach (var embed in embeds)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            var ocrId = Guid.NewGuid().ToString("N");
+            await artifactStore.AppendAssistantContextAsync(
+                artifacts,
+                CreateMeetingNoteImageMarkdown(
+                    ocrId,
+                    embed.OriginalEmbed,
+                    embed.ImagePath,
+                    artifacts.AssistantContextPath),
+                cancellationToken);
+            queuedEmbeds.Add((embed, ocrId));
+        }
+
+        foreach (var (embed, ocrId) in queuedEmbeds)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            var ocrCancellation = new CancellationTokenSource();
+            TrackOcr(
+                artifacts,
+                ocrCancellation,
+                RunOcrAsync(
+                    artifacts,
+                    embed.ImagePath,
+                    ocrId,
+                    options,
+                    ScreenshotOcrPolicy.MeetingNoteImage,
+                    ocrCancellation.Token));
+        }
+
+        return new MeetingNoteImageOcrQueueResult(queuedEmbeds.Count);
+    }
+
     private async Task<string> SaveScreenshotAsync(
         MeetingSessionArtifacts artifacts,
         byte[] pngBytes,
@@ -234,6 +401,7 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
         string screenshotPath,
         string screenshotId,
         MeetingAssistantOptions options,
+        ScreenshotOcrPolicy policy,
         CancellationToken cancellationToken)
     {
         var timeout = options.Screenshots.Ocr.GetEffectiveTimeout();
@@ -248,17 +416,26 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
                     : options.Screenshots.Ocr.Prompt,
                 options,
                 timeoutSource.Token);
-            var cropMarkdown = await TryCreateCropMarkdownAsync(
-                artifacts.AssistantContextPath,
-                screenshotPath,
-                result.Crop,
-                timeoutSource.Token);
+            var cropMarkdown = policy.AllowCrop
+                ? await TryCreateCropMarkdownAsync(
+                    artifacts.AssistantContextPath,
+                    screenshotPath,
+                    result.Crop,
+                    timeoutSource.Token)
+                : "";
             await ReplaceOcrPlaceholderAsync(
                 artifacts.AssistantContextPath,
                 screenshotId,
                 cropMarkdown +
                 "### OCR" + Environment.NewLine + Environment.NewLine + result.Text.Trim(),
                 CancellationToken.None);
+            if (policy.AddAttendees)
+            {
+                await AddOcrAttendeesAsync(
+                    artifacts,
+                    result.Attendees,
+                    CancellationToken.None);
+            }
         }
         catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
         {
@@ -269,8 +446,15 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
             await ReplaceOcrPlaceholderAsync(
                 artifacts.AssistantContextPath,
                 screenshotId,
-                $"_OCR timed out after {timeout:g}._",
-                CancellationToken.None);
+                CreateOcrFailureMarkdown(
+                    artifacts,
+                    screenshotPath,
+                    screenshotId,
+                    options,
+                    $"_OCR timed out after {timeout:g}._",
+                    policy.IncludeRetryLink),
+                CancellationToken.None,
+                preserveMarkers: true);
         }
         catch (Exception exception)
         {
@@ -278,16 +462,63 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
             await ReplaceOcrPlaceholderAsync(
                 artifacts.AssistantContextPath,
                 screenshotId,
-                $"_OCR failed: {exception.Message}_",
-                CancellationToken.None);
+                CreateOcrFailureMarkdown(
+                    artifacts,
+                    screenshotPath,
+                    screenshotId,
+                    options,
+                    $"_OCR failed: {exception.Message}_",
+                    policy.IncludeRetryLink),
+                CancellationToken.None,
+                preserveMarkers: true);
         }
     }
 
-    private async Task ReplaceOcrPlaceholderAsync(
+    private async Task AddOcrAttendeesAsync(
+        MeetingSessionArtifacts artifacts,
+        IReadOnlyList<string> attendees,
+        CancellationToken cancellationToken)
+    {
+        var additions = MeetingAttendeeNames.NormalizeDistinct(attendees);
+        if (additions.Count == 0)
+        {
+            return;
+        }
+
+        try
+        {
+            var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken);
+            var canonicalized = await attendeeCanonicalizer.CanonicalizeAsync(
+                meetingNote.Frontmatter.Attendees.Concat(additions).ToList(),
+                cancellationToken);
+            if (meetingNote.Frontmatter.Attendees.SequenceEqual(canonicalized, StringComparer.Ordinal))
+            {
+                return;
+            }
+
+            meetingNote.Frontmatter.Attendees = canonicalized.ToList();
+            await meetingNoteStore.SaveAsync(meetingNote, cancellationToken);
+            logger.LogInformation(
+                "Added {AttendeeCount} screenshot OCR attendee candidate(s) to meeting note {MeetingNotePath}",
+                additions.Count,
+                artifacts.MeetingNotePath);
+        }
+        catch (Exception exception) when (exception is not OperationCanceledException)
+        {
+            logger.LogWarning(
+                exception,
+                "Could not add screenshot OCR attendees to meeting note {MeetingNotePath}",
+                artifacts.MeetingNotePath);
+        }
+    }
+
+    private async Task<bool> ReplaceOcrPlaceholderAsync(
         string assistantContextPath,
         string screenshotId,
         string replacement,
-        CancellationToken cancellationToken)
+        CancellationToken cancellationToken,
+        bool preserveMarkers = false,
+        bool appendWhenMissing = true)
     {
         var fileLock = contextFileLocks.GetOrAdd(
             NormalizeContextKey(assistantContextPath),
@@ -297,28 +528,41 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
         {
             if (!File.Exists(assistantContextPath))
             {
-                return;
+                return false;
             }
 
             var content = await File.ReadAllTextAsync(assistantContextPath, cancellationToken);
-            var startMarker = $"<!-- screenshot-ocr:{screenshotId} -->";
-            var endMarker = $"<!-- /screenshot-ocr:{screenshotId} -->";
+            var startMarker = CreateOcrStartMarker(screenshotId);
+            var endMarker = CreateOcrEndMarker(screenshotId);
             var startIndex = content.IndexOf(startMarker, StringComparison.Ordinal);
             var endIndex = content.IndexOf(endMarker, StringComparison.Ordinal);
             if (startIndex < 0 || endIndex < startIndex)
             {
-                await File.AppendAllTextAsync(
-                    assistantContextPath,
-                    Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
-                    cancellationToken);
-                return;
+                if (appendWhenMissing)
+                {
+                    await File.AppendAllTextAsync(
+                        assistantContextPath,
+                        Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
+                        cancellationToken);
+                }
+
+                return false;
             }
 
             endIndex += endMarker.Length;
-            var updated = content[..startIndex] +
-                replacement.TrimEnd() +
-                content[endIndex..];
+            var updated = preserveMarkers
+                ? content[..startIndex] +
+                    startMarker +
+                    Environment.NewLine +
+                    replacement.TrimEnd() +
+                    Environment.NewLine +
+                    endMarker +
+                    content[endIndex..]
+                : content[..startIndex] +
+                    replacement.TrimEnd() +
+                    content[endIndex..];
             await File.WriteAllTextAsync(assistantContextPath, updated, cancellationToken);
+            return true;
         }
         finally
         {
@@ -326,6 +570,29 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
         }
     }
 
+    private static string CreateOcrFailureMarkdown(
+        MeetingSessionArtifacts artifacts,
+        string screenshotPath,
+        string screenshotId,
+        MeetingAssistantOptions options,
+        string status,
+        bool includeRetryLink = true)
+    {
+        var markdown = status.TrimEnd();
+        if (includeRetryLink)
+        {
+            markdown += Environment.NewLine +
+                Environment.NewLine +
+                MeetingNoteActionLinks.CreateScreenshotOcrRetryLink(
+                    options.Api.PublicBaseUrl,
+                    artifacts,
+                    screenshotPath,
+                    screenshotId);
+        }
+
+        return markdown;
+    }
+
     private async Task<string> TryCreateCropMarkdownAsync(
         string assistantContextPath,
         string screenshotPath,
@@ -435,13 +702,189 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
         return content +
             Environment.NewLine +
             Environment.NewLine +
-            $"<!-- screenshot-ocr:{screenshotId} -->" +
-            Environment.NewLine +
-            "_OCR pending..._" +
-            Environment.NewLine +
-            $"<!-- /screenshot-ocr:{screenshotId} -->";
+            CreateOcrBlock(screenshotId, "_OCR pending..._");
     }
 
+    private static string CreateMeetingNoteImageMarkdown(
+        string ocrId,
+        string originalEmbed,
+        string imagePath,
+        string assistantContextPath)
+    {
+        var relativePath = ToMarkdownPath(Path.GetRelativePath(
+            Path.GetDirectoryName(assistantContextPath)!,
+            imagePath));
+        return "## Meeting Note Image" +
+            Environment.NewLine +
+            "Image from meeting note." +
+            Environment.NewLine +
+            $"Original embed: `{originalEmbed.Replace("`", "\\`", StringComparison.Ordinal)}`" +
+            Environment.NewLine +
+            $"![Meeting note image]({relativePath})" +
+            Environment.NewLine +
+            Environment.NewLine +
+            CreateOcrBlock(ocrId, "_OCR pending..._");
+    }
+
+    private static string CreateOcrBlock(string screenshotId, string content)
+    {
+        return CreateOcrStartMarker(screenshotId) +
+            Environment.NewLine +
+            content.TrimEnd() +
+            Environment.NewLine +
+            CreateOcrEndMarker(screenshotId);
+    }
+
+    private static string CreateOcrStartMarker(string screenshotId)
+    {
+        return $"<!-- screenshot-ocr:{screenshotId} -->";
+    }
+
+    private static string CreateOcrEndMarker(string screenshotId)
+    {
+        return $"<!-- /screenshot-ocr:{screenshotId} -->";
+    }
+
+    private static IReadOnlyList<MeetingNoteImageEmbed> FindMeetingNoteImageEmbeds(
+        string userNotes,
+        string meetingNotePath,
+        MeetingAssistantOptions options)
+    {
+        if (string.IsNullOrWhiteSpace(userNotes))
+        {
+            return [];
+        }
+
+        var result = new List<MeetingNoteImageEmbed>();
+        var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        foreach (Match match in ObsidianImageEmbedRegex().Matches(userNotes))
+        {
+            AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options);
+        }
+
+        foreach (Match match in MarkdownImageEmbedRegex().Matches(userNotes))
+        {
+            AddResolvedEmbed(result, seen, match.Value, match.Groups["target"].Value, meetingNotePath, options);
+        }
+
+        return result;
+    }
+
+    private static void AddResolvedEmbed(
+        List<MeetingNoteImageEmbed> embeds,
+        HashSet<string> seen,
+        string originalEmbed,
+        string target,
+        string meetingNotePath,
+        MeetingAssistantOptions options)
+    {
+        var imagePath = ResolveMeetingNoteImagePath(target, meetingNotePath, options);
+        if (imagePath is null || !seen.Add(imagePath))
+        {
+            return;
+        }
+
+        embeds.Add(new MeetingNoteImageEmbed(originalEmbed, imagePath));
+    }
+
+    private static string? ResolveMeetingNoteImagePath(
+        string target,
+        string meetingNotePath,
+        MeetingAssistantOptions options)
+    {
+        var cleaned = CleanEmbedTarget(target);
+        if (string.IsNullOrWhiteSpace(cleaned) ||
+            cleaned.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
+            cleaned.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ||
+            cleaned.StartsWith("data:", StringComparison.OrdinalIgnoreCase) ||
+            !IsSupportedImagePath(cleaned))
+        {
+            return null;
+        }
+
+        var candidates = new List<string>();
+        if (Path.IsPathRooted(cleaned))
+        {
+            candidates.Add(VaultPath.Resolve(cleaned));
+        }
+        else
+        {
+            candidates.Add(Path.GetFullPath(Path.Combine(Path.GetDirectoryName(meetingNotePath)!, cleaned)));
+            candidates.Add(VaultPath.Resolve(options.Vault, cleaned));
+        }
+
+        foreach (var candidate in candidates)
+        {
+            if (File.Exists(candidate))
+            {
+                return candidate;
+            }
+        }
+
+        if (Path.GetFileName(cleaned).Equals(cleaned, StringComparison.Ordinal))
+        {
+            try
+            {
+                var vaultRoot = VaultPath.Resolve(options.Vault.BaseFolder);
+                return Directory.EnumerateFiles(vaultRoot, cleaned, SearchOption.AllDirectories)
+                    .FirstOrDefault(IsSupportedImagePath);
+            }
+            catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or DirectoryNotFoundException)
+            {
+                return null;
+            }
+        }
+
+        return null;
+    }
+
+    private static string CleanEmbedTarget(string target)
+    {
+        var cleaned = target.Trim();
+        if (cleaned.StartsWith("<", StringComparison.Ordinal) &&
+            cleaned.EndsWith(">", StringComparison.Ordinal) &&
+            cleaned.Length > 1)
+        {
+            cleaned = cleaned[1..^1].Trim();
+        }
+
+        var pipeIndex = cleaned.IndexOf('|', StringComparison.Ordinal);
+        if (pipeIndex >= 0)
+        {
+            cleaned = cleaned[..pipeIndex].Trim();
+        }
+
+        var hashIndex = cleaned.IndexOf('#', StringComparison.Ordinal);
+        if (hashIndex >= 0)
+        {
+            cleaned = cleaned[..hashIndex].Trim();
+        }
+
+        try
+        {
+            return Uri.UnescapeDataString(cleaned.Replace('/', Path.DirectorySeparatorChar));
+        }
+        catch (UriFormatException)
+        {
+            return cleaned.Replace('/', Path.DirectorySeparatorChar);
+        }
+    }
+
+    private static bool IsSupportedImagePath(string path)
+    {
+        return Path.GetExtension(path).ToLowerInvariant() switch
+        {
+            ".png" or ".jpg" or ".jpeg" or ".gif" or ".webp" or ".bmp" or ".tif" or ".tiff" => true,
+            _ => false
+        };
+    }
+
+    [GeneratedRegex(@"!\[\[(?<target>[^\]\r\n]+)\]\]", RegexOptions.CultureInvariant)]
+    private static partial Regex ObsidianImageEmbedRegex();
+
+    [GeneratedRegex(@"!\[[^\]\r\n]*\]\((?<target>[^)\r\n]+)\)", RegexOptions.CultureInvariant)]
+    private static partial Regex MarkdownImageEmbedRegex();
+
     private static string ResolveAttachmentsFolder(string assistantContextPath, string configuredFolder)
     {
         var expanded = Environment.ExpandEnvironmentVariables(
@@ -479,4 +922,19 @@ public sealed class MeetingScreenshotService : IMeetingScreenshotService
     }
 
     private sealed record PendingOcrTask(Task Task, CancellationTokenSource Cancellation);
+
+    private sealed record ScreenshotOcrPolicy(bool AllowCrop, bool AddAttendees, bool IncludeRetryLink)
+    {
+        public static ScreenshotOcrPolicy CapturedScreenshot { get; } = new(
+            AllowCrop: true,
+            AddAttendees: true,
+            IncludeRetryLink: true);
+
+        public static ScreenshotOcrPolicy MeetingNoteImage { get; } = new(
+            AllowCrop: false,
+            AddAttendees: false,
+            IncludeRetryLink: false);
+    }
+
+    private sealed record MeetingNoteImageEmbed(string OriginalEmbed, string ImagePath);
 }