From 5d0ae84426f922075535af46eb1c472da728aec3 Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Mon, 3 Aug 2026 16:14:01 +0200 Subject: [PATCH] fix: support streaming screenshot OCR --- .../LiteLlmScreenshotOcrClientTests.cs | 163 ++++++++++++++---- .../Screenshots/LiteLlmScreenshotOcrClient.cs | 124 +++---------- .../Summary/LiteLlmResponsesChatClient.cs | 40 +++-- docs/meeting-assistant-configuration.md | 2 +- .../.openspec.yaml | 2 + .../design.md | 34 ++++ .../proposal.md | 24 +++ .../specs/meeting-summary/spec.md | 103 +++++++++++ .../tasks.md | 15 ++ 9 files changed, 362 insertions(+), 145 deletions(-) create mode 100644 openspec/changes/apply-streaming-to-screenshot-ocr/.openspec.yaml create mode 100644 openspec/changes/apply-streaming-to-screenshot-ocr/design.md create mode 100644 openspec/changes/apply-streaming-to-screenshot-ocr/proposal.md create mode 100644 openspec/changes/apply-streaming-to-screenshot-ocr/specs/meeting-summary/spec.md create mode 100644 openspec/changes/apply-streaming-to-screenshot-ocr/tasks.md diff --git a/MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs b/MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs index f67fe81..b1d0b95 100644 --- a/MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs +++ b/MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs @@ -13,21 +13,50 @@ namespace MeetingAssistant.Tests; public sealed class LiteLlmScreenshotOcrClientTests { + [Fact] + public async Task ExtractUsesAgentStreamingTransportByDefault() + { + var screenshotPath = await CreateScreenshotAsync([1, 2, 3]); + var handler = new RecordingHandler( + CreateStreamedTextResponse("Streamed OCR text"), + "text/event-stream"); + var client = new LiteLlmScreenshotOcrClient( + () => handler, + NullLogger.Instance); + var options = new MeetingAssistantOptions + { + Agent = + { + Endpoint = "https://summary.local", + Model = "vision-model", + Key = "agent-key", + UseStreaming = true + } + }; + + var result = await client.ExtractAsync( + screenshotPath, + "Extract screenshot.", + options, + CancellationToken.None); + + Assert.Equal("Streamed OCR text", result.Text); + using var payload = JsonDocument.Parse(handler.RequestBody!); + Assert.True(payload.RootElement.GetProperty("stream").GetBoolean()); + var message = Assert.Single(payload.RootElement.GetProperty("input").EnumerateArray()); + Assert.Equal("message", message.GetProperty("type").GetString()); + var content = message.GetProperty("content").EnumerateArray().ToArray(); + Assert.Equal("input_text", content[0].GetProperty("type").GetString()); + Assert.Equal("Extract screenshot.", content[0].GetProperty("text").GetString()); + Assert.Equal("input_image", content[1].GetProperty("type").GetString()); + Assert.Equal("data:image/png;base64,AQID", content[1].GetProperty("image_url").GetString()); + } + [Fact] public async Task ExtractUsesAgentEndpointAndModelWhenOcrEndpointAndModelAreBlank() { var screenshotPath = await CreateScreenshotAsync([1, 2, 3]); - var handler = new RecordingHandler(""" - { - "output": [ - { - "content": [ - { "type": "output_text", "text": "Visible slide text" } - ] - } - ] - } - """); + var handler = new RecordingHandler(CreateNonStreamingTextResponse("Visible slide text")); var client = new LiteLlmScreenshotOcrClient( () => handler, NullLogger.Instance); @@ -37,7 +66,8 @@ public sealed class LiteLlmScreenshotOcrClientTests { Endpoint = "https://summary.local", Model = "summary-model", - Key = "agent-key" + Key = "agent-key", + UseStreaming = false }, Screenshots = { @@ -62,6 +92,7 @@ public sealed class LiteLlmScreenshotOcrClientTests Assert.Equal("Bearer", handler.Authorization?.Scheme); Assert.Equal("ocr-key", handler.Authorization?.Parameter); using var payload = JsonDocument.Parse(handler.RequestBody!); + Assert.False(payload.RootElement.GetProperty("stream").GetBoolean()); Assert.Equal("summary-model", payload.RootElement.GetProperty("model").GetString()); var content = payload.RootElement .GetProperty("input")[0] @@ -74,7 +105,7 @@ public sealed class LiteLlmScreenshotOcrClientTests public async Task ExtractUsesScreenshotOcrEndpointAndModelWhenConfigured() { var screenshotPath = await CreateScreenshotAsync([4, 5, 6]); - var handler = new RecordingHandler("""{ "output_text": "OCR result" }"""); + var handler = new RecordingHandler(CreateNonStreamingTextResponse("OCR result")); var client = new LiteLlmScreenshotOcrClient( () => handler, NullLogger.Instance); @@ -84,7 +115,8 @@ public sealed class LiteLlmScreenshotOcrClientTests { Endpoint = "https://summary.local", Model = "summary-model", - Key = "agent-key" + Key = "agent-key", + UseStreaming = false }, Screenshots = { @@ -113,11 +145,14 @@ public sealed class LiteLlmScreenshotOcrClientTests public async Task ExtractParsesCropMetadataAndOmitsMetadataFromReturnedText() { var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6)); - var handler = new RecordingHandler(""" - { - "output_text": "Slide text\n\n```json\n{ \"crop\": { \"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4 } }\n```" - } - """); + var handler = new RecordingHandler(CreateNonStreamingTextResponse( + """ + Slide text + + ```json + { "crop": { "x": 1, "y": 2, "width": 3, "height": 4 } } + ``` + """)); var client = new LiteLlmScreenshotOcrClient( () => handler, NullLogger.Instance); @@ -125,7 +160,8 @@ public sealed class LiteLlmScreenshotOcrClientTests { Agent = { - Key = "agent-key" + Key = "agent-key", + UseStreaming = false } }; @@ -148,11 +184,14 @@ public sealed class LiteLlmScreenshotOcrClientTests public async Task ExtractParsesAttendeeMetadataAndOmitsMetadataFromReturnedText() { var screenshotPath = await CreateScreenshotAsync([1, 2, 3]); - var handler = new RecordingHandler(""" - { - "output_text": "Visible participant tiles: Ada and Grace.\n\n```json\n{ \"crop\": null, \"attendees\": [\"Ada Lovelace\", \"Grace Hopper\"] }\n```" - } - """); + var handler = new RecordingHandler(CreateNonStreamingTextResponse( + """ + Visible participant tiles: Ada and Grace. + + ```json + { "crop": null, "attendees": ["Ada Lovelace", "Grace Hopper"] } + ``` + """)); var client = new LiteLlmScreenshotOcrClient( () => handler, NullLogger.Instance); @@ -160,7 +199,8 @@ public sealed class LiteLlmScreenshotOcrClientTests { Agent = { - Key = "agent-key" + Key = "agent-key", + UseStreaming = false } }; @@ -178,11 +218,14 @@ public sealed class LiteLlmScreenshotOcrClientTests public async Task ExtractIgnoresMalformedAttendeesMetadataAndStillParsesCrop() { var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6)); - var handler = new RecordingHandler(""" - { - "output_text": "Slide text\n\n```json\n{ \"crop\": { \"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4 }, \"attendees\": \"Ada\" }\n```" - } - """); + var handler = new RecordingHandler(CreateNonStreamingTextResponse( + """ + Slide text + + ```json + { "crop": { "x": 1, "y": 2, "width": 3, "height": 4 }, "attendees": "Ada" } + ``` + """)); var client = new LiteLlmScreenshotOcrClient( () => handler, NullLogger.Instance); @@ -190,7 +233,8 @@ public sealed class LiteLlmScreenshotOcrClientTests { Agent = { - Key = "agent-key" + Key = "agent-key", + UseStreaming = false } }; @@ -208,10 +252,12 @@ public sealed class LiteLlmScreenshotOcrClientTests private sealed class RecordingHandler : HttpMessageHandler { private readonly string responseBody; + private readonly string mediaType; - public RecordingHandler(string responseBody) + public RecordingHandler(string responseBody, string mediaType = "application/json") { this.responseBody = responseBody; + this.mediaType = mediaType; } public Uri? RequestUri { get; private set; } @@ -231,7 +277,7 @@ public sealed class LiteLlmScreenshotOcrClientTests : await request.Content.ReadAsStringAsync(cancellationToken); return new HttpResponseMessage(HttpStatusCode.OK) { - Content = new StringContent(responseBody, Encoding.UTF8, "application/json") + Content = new StringContent(responseBody, Encoding.UTF8, mediaType) }; } } @@ -249,6 +295,55 @@ public sealed class LiteLlmScreenshotOcrClientTests return stream.ToArray(); } + private static string CreateNonStreamingTextResponse(string text) + { + return JsonSerializer.Serialize(new + { + id = "resp_ocr", + created_at = 1779147100, + model = "vision-model", + @object = "response", + output = new[] + { + new + { + id = "msg_ocr", + type = "message", + status = "completed", + content = new[] + { + new + { + type = "output_text", + annotations = Array.Empty(), + text + } + }, + role = "assistant" + } + }, + parallel_tool_calls = true, + status = "completed", + store = false + }); + } + + private static string CreateStreamedTextResponse(string text) + { + var delta = JsonSerializer.Serialize(new + { + type = "response.output_text.delta", + item_id = "msg_ocr", + output_index = 0, + content_index = 0, + delta = text + }); + return + $"data: {delta}{Environment.NewLine}{Environment.NewLine}" + + """data: {"type":"response.completed","response":{"id":"resp_ocr","created_at":1779147100,"model":"vision-model","object":"response","output":[],"parallel_tool_calls":true,"status":"completed","store":false}}""" + + $"{Environment.NewLine}{Environment.NewLine}data: [DONE]{Environment.NewLine}{Environment.NewLine}"; + } + private static async Task CreateScreenshotAsync(byte[] bytes) { var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png"); diff --git a/MeetingAssistant/Screenshots/LiteLlmScreenshotOcrClient.cs b/MeetingAssistant/Screenshots/LiteLlmScreenshotOcrClient.cs index 9ad3f45..3c054f0 100644 --- a/MeetingAssistant/Screenshots/LiteLlmScreenshotOcrClient.cs +++ b/MeetingAssistant/Screenshots/LiteLlmScreenshotOcrClient.cs @@ -1,15 +1,13 @@ -using System.Net.Http.Headers; -using System.Text; using System.Text.Json; -using System.Text.Json.Nodes; using System.Text.RegularExpressions; using MeetingAssistant.MeetingNotes; +using MeetingAssistant.Summary; +using Microsoft.Extensions.AI; namespace MeetingAssistant.Screenshots; public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient { - private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); private readonly ILogger logger; private readonly Func? httpMessageHandlerFactory; @@ -40,20 +38,30 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient : options.Agent.Model; var key = ResolveApiKey(options); var imageBytes = await File.ReadAllBytesAsync(screenshotPath, cancellationToken); - using var httpClient = CreateHttpClient(); - httpClient.BaseAddress = NormalizeEndpoint(new Uri(endpoint)); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", key); - var payload = CreatePayload(model, CreatePrompt(prompt, imageBytes), imageBytes); - using var content = new StringContent(payload.ToJsonString(JsonOptions), Encoding.UTF8, "application/json"); - using var response = await httpClient.PostAsync("responses", content, cancellationToken); - var responseJson = await response.Content.ReadAsStringAsync(cancellationToken); - if (!response.IsSuccessStatusCode) - { - throw new InvalidOperationException( - $"Screenshot OCR request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}"); - } - - var text = ParseOutputText(responseJson); + var httpClient = CreateHttpClient(); + httpClient.BaseAddress = LiteLlmResponsesChatClient.NormalizeEndpoint(new Uri(endpoint)); + using var chatClient = new LiteLlmResponsesChatClient( + httpClient, + key, + model, + enableThinking: false, + reasoningEffort: "none", + reconnectionAttempts: options.Agent.ReconnectionAttempts, + reconnectionDelay: options.Agent.ReconnectionDelay, + logger: logger, + firstRequestIsUser: false, + useStreaming: options.Agent.UseStreaming); + var response = await chatClient.GetResponseAsync( + [ + new ChatMessage( + ChatRole.User, + [ + new TextContent(CreatePrompt(prompt, imageBytes)), + new DataContent(imageBytes, "image/png") + ]) + ], + cancellationToken: cancellationToken); + var text = response.Text ?? string.Empty; logger.LogInformation("Screenshot OCR completed for {ScreenshotPath}", screenshotPath); return ParseOcrResult(text); } @@ -65,35 +73,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient : new HttpClient(httpMessageHandlerFactory()); } - private static JsonObject CreatePayload(string model, string prompt, byte[] imageBytes) - { - return new JsonObject - { - ["model"] = model, - ["store"] = false, - ["input"] = new JsonArray - { - new JsonObject - { - ["role"] = "user", - ["content"] = new JsonArray - { - new JsonObject - { - ["type"] = "input_text", - ["text"] = prompt - }, - new JsonObject - { - ["type"] = "input_image", - ["image_url"] = "data:image/png;base64," + Convert.ToBase64String(imageBytes) - } - } - } - } - }; - } - private static string CreatePrompt(string prompt, byte[] imageBytes) { return TryReadPngDimensions(imageBytes, out var width, out var height) @@ -210,46 +189,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient bytes[offset + 3]; } - private static string ParseOutputText(string responseJson) - { - using var document = JsonDocument.Parse(responseJson); - var root = document.RootElement; - var parts = new List(); - if (root.TryGetProperty("output_text", out var outputText) && - outputText.ValueKind == JsonValueKind.String && - !string.IsNullOrWhiteSpace(outputText.GetString())) - { - parts.Add(outputText.GetString()!); - } - - if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array) - { - foreach (var item in output.EnumerateArray()) - { - if (!item.TryGetProperty("content", out var content) || content.ValueKind != JsonValueKind.Array) - { - continue; - } - - foreach (var block in content.EnumerateArray()) - { - if (block.TryGetProperty("type", out var type) && - type.GetString() == "output_text" && - block.TryGetProperty("text", out var text) && - text.ValueKind == JsonValueKind.String && - !string.IsNullOrWhiteSpace(text.GetString())) - { - parts.Add(text.GetString()!); - } - } - } - } - - return parts.Count == 0 - ? "" - : string.Join(Environment.NewLine + Environment.NewLine, parts); - } - private static string ResolveApiKey(MeetingAssistantOptions options) { if (!string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Key)) @@ -278,17 +217,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient $"No screenshot OCR API key configured. Set MeetingAssistant:Screenshots:Ocr:Key or environment variable '{options.Screenshots.Ocr.KeyEnv}'."); } - private static Uri NormalizeEndpoint(Uri endpoint) - { - var value = endpoint.ToString().TrimEnd('/'); - if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase)) - { - value += "/v1"; - } - - return new Uri(value + "/"); - } - [GeneratedRegex("```json\\s*(?.*?)\\s*```", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex JsonCodeBlockRegex(); } diff --git a/MeetingAssistant/Summary/LiteLlmResponsesChatClient.cs b/MeetingAssistant/Summary/LiteLlmResponsesChatClient.cs index 5b1e39c..b571edb 100644 --- a/MeetingAssistant/Summary/LiteLlmResponsesChatClient.cs +++ b/MeetingAssistant/Summary/LiteLlmResponsesChatClient.cs @@ -659,9 +659,9 @@ public sealed class LiteLlmResponsesChatClient : IChatClient private static void AddInputItem(JsonArray input, StringBuilder instructions, ChatMessage message) { var role = message.Role.Value; - var text = message.Text; if (role == ChatRole.System.Value) { + var text = message.Text; if (!string.IsNullOrWhiteSpace(text)) { instructions.AppendLine(text); @@ -670,21 +670,37 @@ public sealed class LiteLlmResponsesChatClient : IChatClient return; } - if (!string.IsNullOrWhiteSpace(text)) + var isAssistant = role == ChatRole.Assistant.Value; + var messageContent = new JsonArray(); + foreach (var content in message.Contents) + { + if (content is TextContent textContent && !string.IsNullOrWhiteSpace(textContent.Text)) + { + messageContent.Add(new JsonObject + { + ["type"] = isAssistant ? "output_text" : "input_text", + ["text"] = textContent.Text + }); + } + else if (!isAssistant && + content is DataContent dataContent && + dataContent.HasTopLevelMediaType("image")) + { + messageContent.Add(new JsonObject + { + ["type"] = "input_image", + ["image_url"] = dataContent.Uri.ToString() + }); + } + } + + if (messageContent.Count > 0) { - var isAssistant = role == ChatRole.Assistant.Value; input.Add(new JsonObject { ["type"] = "message", ["role"] = isAssistant ? "assistant" : "user", - ["content"] = new JsonArray - { - new JsonObject - { - ["type"] = isAssistant ? "output_text" : "input_text", - ["text"] = text - } - } + ["content"] = messageContent }); } @@ -761,7 +777,7 @@ public sealed class LiteLlmResponsesChatClient : IChatClient return Math.Max(1, (int)Math.Ceiling(json.Length / 4.0)); } - private static Uri NormalizeEndpoint(Uri endpoint) + internal static Uri NormalizeEndpoint(Uri endpoint) { var value = endpoint.ToString().TrimEnd('/'); if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase)) diff --git a/docs/meeting-assistant-configuration.md b/docs/meeting-assistant-configuration.md index 7d4077a..8af2138 100644 --- a/docs/meeting-assistant-configuration.md +++ b/docs/meeting-assistant-configuration.md @@ -317,7 +317,7 @@ When enabled on Windows, Meeting Assistant periodically syncs today's Outlook Cl `Screenshots:Hotkey` configures a global hotkey that captures the currently active window during an active meeting. Screenshots are written under `Screenshots:AttachmentsFolder`, which defaults to an `Attachments` folder beside the assistant context note, and each capture appends a timestamped markdown image link to the assistant context. -`Screenshots:Ocr` optionally enables vision extraction for screenshots. Blank `Endpoint` or `Model` values fall back to the summary `Agent` endpoint and model. `Key` or `KeyEnv` can be set specifically for OCR; otherwise the summary agent key configuration is used. Automatic summarization scans the meeting note for user-added Obsidian image embeds such as `![[whiteboard.png]]` and Markdown image embeds such as `![Diagram](attachments/diagram.png)`, adds resolvable local images to the assistant context without copying them or changing the meeting note, runs OCR without crop or attendee updates, and waits for all pending OCR work to complete or hit `Timeout` before the assistant context moves to `summarizing`. Failed or timed-out screenshot OCR writes a retry link that targets `/meetings/screenshot-ocr/retry` with the exact saved screenshot and OCR block id. +`Screenshots:Ocr` optionally enables vision extraction for screenshots. Blank `Endpoint` or `Model` values fall back to the summary `Agent` endpoint and model. `Key` or `KeyEnv` can be set specifically for OCR; otherwise the summary agent key configuration is used. Screenshot OCR also inherits `Agent:UseStreaming`, using the Responses SSE transport when it is `true` and the non-streaming Responses transport when it is `false`. Automatic summarization scans the meeting note for user-added Obsidian image embeds such as `![[whiteboard.png]]` and Markdown image embeds such as `![Diagram](attachments/diagram.png)`, adds resolvable local images to the assistant context without copying them or changing the meeting note, runs OCR without crop or attendee updates, and waits for all pending OCR work to complete or hit `Timeout` before the assistant context moves to `summarizing`. Failed or timed-out screenshot OCR writes a retry link that targets `/meetings/screenshot-ocr/retry` with the exact saved screenshot and OCR block id. | Setting | Purpose | | --- | --- | diff --git a/openspec/changes/apply-streaming-to-screenshot-ocr/.openspec.yaml b/openspec/changes/apply-streaming-to-screenshot-ocr/.openspec.yaml new file mode 100644 index 0000000..f205fc7 --- /dev/null +++ b/openspec/changes/apply-streaming-to-screenshot-ocr/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-29 diff --git a/openspec/changes/apply-streaming-to-screenshot-ocr/design.md b/openspec/changes/apply-streaming-to-screenshot-ocr/design.md new file mode 100644 index 0000000..5918c40 --- /dev/null +++ b/openspec/changes/apply-streaming-to-screenshot-ocr/design.md @@ -0,0 +1,34 @@ +## Context + +`LiteLlmScreenshotOcrClient` currently builds and posts a raw Responses JSON payload, then parses the successful body as one JSON document. It inherits endpoint, model, and key values from `AgentOptions`, but never reads `AgentOptions.UseStreaming`. The summary and workflow agents already use `LiteLlmResponsesChatClient`, which selects the OpenAI SDK streaming or non-streaming Responses method and maps both through the Microsoft.Extensions.AI adapter. + +## Goals / Non-Goals + +**Goals:** + +- Make screenshot OCR use `Agent:UseStreaming` without adding another setting. +- Reuse the supported Responses SDK transport and response adapter. +- Preserve screenshot-specific endpoint, model, key, prompt, image, crop, attendee, and timeout behavior. +- Keep non-streaming screenshot OCR working when streaming is disabled. + +**Non-Goals:** + +- Expose OCR token deltas to the UI or assistant context. +- Change screenshot OCR retry, crop, attendee, or note-block semantics. +- Add file upload or remote image URL support. + +## Decisions + +1. Route screenshot OCR through `LiteLlmResponsesChatClient` instead of maintaining a second Responses parser. The screenshot client will construct one user chat message containing prompt text and PNG `DataContent`, then consume the buffered `ChatResponse.Text`. This keeps transport selection, SDK request creation, SSE assembly, and non-streaming mapping in one client. + +2. Extend the shared Responses message translator to map image `DataContent` to an `input_image` content block using its data URI. Text and image blocks remain in one `type: message` input item, matching the existing OCR payload shape. + +3. Use `AgentOptions.UseStreaming` for screenshot OCR even when the screenshot-specific endpoint or model overrides are set. Endpoint/model/key remain independently overrideable; transport is an agent-wide behavior setting. + +4. Disable reasoning and compaction for the one-turn OCR request, preserving the existing screenshot client behavior while still using the agent reconnection settings and selected transport. + +## Risks / Trade-offs + +- **[Shared-client diagnostics mention summary context]** Some low-level logs are named for the summary pipeline. → Avoid passing summary compaction state and keep screenshot-specific completion/failure logs at the screenshot client boundary. +- **[Multimodal translation expands shared client scope]** Incorrect content mapping could affect summary requests. → Add request-body behavior coverage proving prompt and PNG data URI are preserved, while existing summary message tests protect text translation. +- **[Provider image support varies]** A configured model may reject image input. → Preserve the provider error and existing screenshot OCR failure/retry behavior. diff --git a/openspec/changes/apply-streaming-to-screenshot-ocr/proposal.md b/openspec/changes/apply-streaming-to-screenshot-ocr/proposal.md new file mode 100644 index 0000000..228a69e --- /dev/null +++ b/openspec/changes/apply-streaming-to-screenshot-ocr/proposal.md @@ -0,0 +1,24 @@ +## Why + +Screenshot OCR inherits its endpoint, model, and key from the summary agent, but it bypasses the shared Responses client and ignores `Agent:UseStreaming`. With streaming enabled, the screenshot client still expects one JSON document and cannot consume the configured LiteLLM Responses event stream. + +## What Changes + +- Make screenshot OCR inherit the summary agent's streaming transport selection. +- Send screenshot image input through the same OpenAI Responses SDK and Agent Framework adapter used by the summary client. +- Preserve the existing non-streaming screenshot OCR path when streaming is disabled. +- Add behavior coverage for streamed and non-streamed screenshot OCR responses. + +## Capabilities + +### New Capabilities + +- None. + +### Modified Capabilities + +- `meeting-summary`: Require screenshot OCR to honor the configured agent streaming transport while preserving image input and OCR metadata parsing. + +## Impact + +The change affects the screenshot OCR client, the shared LiteLLM Responses message translation, focused tests, and agent configuration documentation. It does not change the local HTTP API or screenshot note format. diff --git a/openspec/changes/apply-streaming-to-screenshot-ocr/specs/meeting-summary/spec.md b/openspec/changes/apply-streaming-to-screenshot-ocr/specs/meeting-summary/spec.md new file mode 100644 index 0000000..a5db404 --- /dev/null +++ b/openspec/changes/apply-streaming-to-screenshot-ocr/specs/meeting-summary/spec.md @@ -0,0 +1,103 @@ +## MODIFIED Requirements + +### Requirement: Meeting screenshots are captured into assistant context +Meeting Assistant SHALL expose a configurable screenshot hotkey. + +When a meeting is active and the screenshot hotkey is pressed, Meeting Assistant SHALL capture the currently active window. + +The screenshot image SHALL be saved into a configurable attachments folder for the assistant context note. By default, the folder SHALL be `Attachments` beside the assistant context note. + +After the image is saved, Meeting Assistant SHALL append a markdown image link to the assistant context note with a meeting-relative timestamp that correlates to transcript timestamps. + +Meeting Assistant SHALL allow optional screenshot OCR configuration with endpoint URL, API key or key environment variable, model, prompt, and timeout. + +When screenshot OCR is configured, Meeting Assistant SHALL send the screenshot and prompt to the configured OpenAI-compatible Responses endpoint and append the model result after the screenshot link in the assistant context note. + +Screenshot OCR SHALL honor the configured `MeetingAssistant:Agent:UseStreaming` transport selection. + +When streaming is enabled, screenshot OCR SHALL consume the Responses result through the supported OpenAI Responses and Agent Framework Server-Sent Events adapter. + +When streaming is disabled, screenshot OCR SHALL consume the result through the supported non-streaming OpenAI Responses client and adapter. + +The screenshot OCR prompt SHALL ask the model to return pixel crop coordinates when it can confidently isolate only the presentation, shared screen, or similarly relevant meeting content. + +When OCR returns valid crop coordinates within the original image bounds, Meeting Assistant SHALL save a cropped PNG beside the original screenshot and SHALL link the cropped image before the OCR result in the assistant context note. + +When OCR returns no crop coordinates or invalid crop coordinates, Meeting Assistant SHALL keep the original screenshot link and OCR result without writing a cropped image. + +After transcription finishes and before summarization starts, Meeting Assistant SHALL scan the meeting note for user-authored Obsidian image embeds and Markdown image embeds. + +When configured screenshot OCR is enabled and the meeting note contains image embeds, Meeting Assistant SHALL append each resolvable image to the assistant context note, state that the image came from the meeting note, preserve the original embed text for cross-reference, and run OCR for the linked image. + +Meeting-note image OCR SHALL NOT copy the image file, SHALL NOT write crop images, SHALL NOT add attendees from OCR metadata, and SHALL NOT modify the meeting note. + +Meeting Assistant SHALL wait for meeting-note image OCR to finish or time out before transitioning the assistant context to summarizing. + +When screenshot OCR fails or times out, Meeting Assistant SHALL write the failure status into the assistant context note with a retry link for that exact screenshot. + +When the screenshot OCR retry link is activated, Meeting Assistant SHALL rerun OCR for the saved screenshot and replace that screenshot's OCR block in the assistant context note. + +When screenshot OCR is not configured, Meeting Assistant SHALL skip OCR and keep the screenshot link. + +The default OCR prompt SHALL explain that the image is from a meeting and ask the model to identify who is talking, who is presenting, what is presented, capture slide text in markdown, convert diagrams to Mermaid when possible, indicate whether visible people are clearly the exact meeting participants or only a partial result, return crop coordinates only for confidently isolated presentation/shared-screen content, and otherwise describe the scene. + +#### Scenario: Screenshot is linked with meeting timestamp +- **GIVEN** a meeting started at `10:00:00` +- **WHEN** the user captures a screenshot at `10:03:05` +- **THEN** Meeting Assistant saves the screenshot under the configured attachments folder +- **AND** appends a markdown image link to assistant context with timestamp `[00:03:05]` + +#### Scenario: OCR result is appended after screenshot +- **GIVEN** screenshot OCR is configured +- **WHEN** the user captures a screenshot +- **THEN** Meeting Assistant appends the screenshot link to assistant context +- **AND** appends the OCR result for that screenshot after the link when processing completes + +#### Scenario: Streaming screenshot OCR is assembled +- **GIVEN** screenshot OCR is configured +- **AND** `MeetingAssistant:Agent:UseStreaming` is `true` +- **WHEN** the Responses endpoint returns screenshot OCR output as Server-Sent Events +- **THEN** Meeting Assistant sends the prompt and screenshot as one multimodal Responses message +- **AND** appends the assembled OCR text without a JSON document parse failure + +#### Scenario: Screenshot OCR streaming can be disabled +- **GIVEN** screenshot OCR is configured +- **AND** `MeetingAssistant:Agent:UseStreaming` is `false` +- **WHEN** Meeting Assistant requests screenshot OCR +- **THEN** it uses the supported non-streaming Responses client and adapter +- **AND** preserves the prompt and screenshot image input + +#### Scenario: OCR crop is saved and linked before OCR text +- **GIVEN** screenshot OCR is configured +- **AND** OCR returns valid crop coordinates for a shared screen +- **WHEN** OCR processing completes +- **THEN** Meeting Assistant saves a cropped screenshot beside the original image +- **AND** links the cropped screenshot before the OCR text in assistant context + +#### Scenario: Meeting note image embeds are OCRed before summarization +- **GIVEN** screenshot OCR is configured +- **AND** the meeting note contains `![[whiteboard.png]]` +- **AND** the meeting note contains `![Diagram](attachments/diagram.png)` +- **WHEN** transcription finishes +- **THEN** Meeting Assistant appends both images to the assistant context as images from the meeting note +- **AND** includes the original embed text for each image +- **AND** runs OCR for each image without copying files, writing crop images, adding attendees, or modifying the meeting note +- **AND** waits for this OCR to finish or time out before transitioning to summarizing + +#### Scenario: OCR failure can be retried for the same screenshot +- **GIVEN** screenshot OCR is configured +- **AND** OCR fails or times out for a captured screenshot +- **WHEN** Meeting Assistant writes the OCR failure status +- **THEN** the assistant context includes a retry link for that exact screenshot +- **WHEN** the retry link is activated +- **THEN** Meeting Assistant reruns OCR against the saved screenshot +- **AND** replaces that screenshot's OCR block with the retry result + +#### Scenario: OCR is skipped when not configured +- **GIVEN** screenshot OCR is not configured +- **WHEN** the user captures a screenshot +- **THEN** Meeting Assistant saves and links the screenshot without calling a model endpoint + +#### Scenario: OCR reports whether visible people are complete or partial +- **WHEN** Meeting Assistant uses the built-in screenshot OCR prompt +- **THEN** the prompt asks the model to state whether the screenshot clearly shows exactly who is in the meeting or only a partial participant result diff --git a/openspec/changes/apply-streaming-to-screenshot-ocr/tasks.md b/openspec/changes/apply-streaming-to-screenshot-ocr/tasks.md new file mode 100644 index 0000000..a2f7366 --- /dev/null +++ b/openspec/changes/apply-streaming-to-screenshot-ocr/tasks.md @@ -0,0 +1,15 @@ +## 1. Streaming screenshot OCR + +- [x] 1.1 Add a failing screenshot-client behavior test for an SSE response with prompt and image input. +- [x] 1.2 Route screenshot OCR through the shared Responses client and add multimodal message translation. + +## 2. Non-streaming compatibility + +- [x] 2.1 Add behavior coverage proving `Agent:UseStreaming=false` preserves non-streaming screenshot OCR and image input. +- [x] 2.2 Document that screenshot OCR inherits the agent streaming setting. + +## 3. Verification + +- [x] 3.1 Refactor the touched screenshot and shared client paths for DRYness, SOLID boundaries, and simplicity while preserving behavior. +- [x] 3.2 Run focused tests, the full solution tests, and strict OpenSpec validation. +- [x] 3.3 Restart Meeting Assistant only while idle and verify screenshot OCR against the deployed LiteLLM endpoint.