Manuel/meeting-assistant
Public Access
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity Open in Coder

fix: support streaming screenshot OCR

Browse Source
This commit is contained in:
codex
2026-08-03 16:14:01 +02:00
parent b9547ae4c4
commit 5d0ae84426
9 changed files with 362 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs
+129 -34
View File
@@ -13,21 +13,50 @@ namespace MeetingAssistant.Tests;
public sealed class LiteLlmScreenshotOcrClientTests
{
[Fact]
public async Task ExtractUsesAgentStreamingTransportByDefault()
{
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
var handler = new RecordingHandler(
CreateStreamedTextResponse("Streamed OCR text"),
"text/event-stream");
var client = new LiteLlmScreenshotOcrClient(
() => handler,
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
var options = new MeetingAssistantOptions
{
Agent =
{
Endpoint = "https://summary.local",
Model = "vision-model",
Key = "agent-key",
UseStreaming = true
}
};
var result = await client.ExtractAsync(
screenshotPath,
"Extract screenshot.",
options,
CancellationToken.None);
Assert.Equal("Streamed OCR text", result.Text);
using var payload = JsonDocument.Parse(handler.RequestBody!);
Assert.True(payload.RootElement.GetProperty("stream").GetBoolean());
var message = Assert.Single(payload.RootElement.GetProperty("input").EnumerateArray());
Assert.Equal("message", message.GetProperty("type").GetString());
var content = message.GetProperty("content").EnumerateArray().ToArray();
Assert.Equal("input_text", content[0].GetProperty("type").GetString());
Assert.Equal("Extract screenshot.", content[0].GetProperty("text").GetString());
Assert.Equal("input_image", content[1].GetProperty("type").GetString());
Assert.Equal("data:image/png;base64,AQID", content[1].GetProperty("image_url").GetString());
}
[Fact]
public async Task ExtractUsesAgentEndpointAndModelWhenOcrEndpointAndModelAreBlank()
{
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
var handler = new RecordingHandler("""
{
"output": [
{
"content": [
{ "type": "output_text", "text": "Visible slide text" }
]
}
]
}
""");
var handler = new RecordingHandler(CreateNonStreamingTextResponse("Visible slide text"));
var client = new LiteLlmScreenshotOcrClient(
() => handler,
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
@@ -37,7 +66,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
{
Endpoint = "https://summary.local",
Model = "summary-model",
Key = "agent-key"
Key = "agent-key",
UseStreaming = false
},
Screenshots =
{
@@ -62,6 +92,7 @@ public sealed class LiteLlmScreenshotOcrClientTests
Assert.Equal("Bearer", handler.Authorization?.Scheme);
Assert.Equal("ocr-key", handler.Authorization?.Parameter);
using var payload = JsonDocument.Parse(handler.RequestBody!);
Assert.False(payload.RootElement.GetProperty("stream").GetBoolean());
Assert.Equal("summary-model", payload.RootElement.GetProperty("model").GetString());
var content = payload.RootElement
.GetProperty("input")[0]
@@ -74,7 +105,7 @@ public sealed class LiteLlmScreenshotOcrClientTests
public async Task ExtractUsesScreenshotOcrEndpointAndModelWhenConfigured()
{
var screenshotPath = await CreateScreenshotAsync([4, 5, 6]);
var handler = new RecordingHandler("""{ "output_text": "OCR result" }""");
var handler = new RecordingHandler(CreateNonStreamingTextResponse("OCR result"));
var client = new LiteLlmScreenshotOcrClient(
() => handler,
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
@@ -84,7 +115,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
{
Endpoint = "https://summary.local",
Model = "summary-model",
Key = "agent-key"
Key = "agent-key",
UseStreaming = false
},
Screenshots =
{
@@ -113,11 +145,14 @@ public sealed class LiteLlmScreenshotOcrClientTests
public async Task ExtractParsesCropMetadataAndOmitsMetadataFromReturnedText()
{
var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6));
var handler = new RecordingHandler("""
{
"output_text": "Slide text\n\n```json\n{ \"crop\": { \"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4 } }\n```"
}
""");
var handler = new RecordingHandler(CreateNonStreamingTextResponse(
"""
Slide text
```json
{ "crop": { "x": 1, "y": 2, "width": 3, "height": 4 } }
```
"""));
var client = new LiteLlmScreenshotOcrClient(
() => handler,
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
@@ -125,7 +160,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
{
Agent =
{
Key = "agent-key"
Key = "agent-key",
UseStreaming = false
}
};
@@ -148,11 +184,14 @@ public sealed class LiteLlmScreenshotOcrClientTests
public async Task ExtractParsesAttendeeMetadataAndOmitsMetadataFromReturnedText()
{
var screenshotPath = await CreateScreenshotAsync([1, 2, 3]);
var handler = new RecordingHandler("""
{
"output_text": "Visible participant tiles: Ada and Grace.\n\n```json\n{ \"crop\": null, \"attendees\": [\"Ada Lovelace\", \"Grace Hopper\"] }\n```"
}
""");
var handler = new RecordingHandler(CreateNonStreamingTextResponse(
"""
Visible participant tiles: Ada and Grace.
```json
{ "crop": null, "attendees": ["Ada Lovelace", "Grace Hopper"] }
```
"""));
var client = new LiteLlmScreenshotOcrClient(
() => handler,
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
@@ -160,7 +199,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
{
Agent =
{
Key = "agent-key"
Key = "agent-key",
UseStreaming = false
}
};
@@ -178,11 +218,14 @@ public sealed class LiteLlmScreenshotOcrClientTests
public async Task ExtractIgnoresMalformedAttendeesMetadataAndStillParsesCrop()
{
var screenshotPath = await CreateScreenshotAsync(CreatePngBytes(8, 6));
var handler = new RecordingHandler("""
{
"output_text": "Slide text\n\n```json\n{ \"crop\": { \"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4 }, \"attendees\": \"Ada\" }\n```"
}
""");
var handler = new RecordingHandler(CreateNonStreamingTextResponse(
"""
Slide text
```json
{ "crop": { "x": 1, "y": 2, "width": 3, "height": 4 }, "attendees": "Ada" }
```
"""));
var client = new LiteLlmScreenshotOcrClient(
() => handler,
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
@@ -190,7 +233,8 @@ public sealed class LiteLlmScreenshotOcrClientTests
{
Agent =
{
Key = "agent-key"
Key = "agent-key",
UseStreaming = false
}
};
@@ -208,10 +252,12 @@ public sealed class LiteLlmScreenshotOcrClientTests
private sealed class RecordingHandler : HttpMessageHandler
{
private readonly string responseBody;
private readonly string mediaType;
public RecordingHandler(string responseBody)
public RecordingHandler(string responseBody, string mediaType = "application/json")
{
this.responseBody = responseBody;
this.mediaType = mediaType;
}
public Uri? RequestUri { get; private set; }
@@ -231,7 +277,7 @@ public sealed class LiteLlmScreenshotOcrClientTests
: await request.Content.ReadAsStringAsync(cancellationToken);
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(responseBody, Encoding.UTF8, "application/json")
Content = new StringContent(responseBody, Encoding.UTF8, mediaType)
};
}
}
@@ -249,6 +295,55 @@ public sealed class LiteLlmScreenshotOcrClientTests
return stream.ToArray();
}
private static string CreateNonStreamingTextResponse(string text)
{
return JsonSerializer.Serialize(new
{
id = "resp_ocr",
created_at = 1779147100,
model = "vision-model",
@object = "response",
output = new[]
{
new
{
id = "msg_ocr",
type = "message",
status = "completed",
content = new[]
{
new
{
type = "output_text",
annotations = Array.Empty<object>(),
text
}
},
role = "assistant"
}
},
parallel_tool_calls = true,
status = "completed",
store = false
});
}
private static string CreateStreamedTextResponse(string text)
{
var delta = JsonSerializer.Serialize(new
{
type = "response.output_text.delta",
item_id = "msg_ocr",
output_index = 0,
content_index = 0,
delta = text
});
return
$"data: {delta}{Environment.NewLine}{Environment.NewLine}" +
"""data: {"type":"response.completed","response":{"id":"resp_ocr","created_at":1779147100,"model":"vision-model","object":"response","output":[],"parallel_tool_calls":true,"status":"completed","store":false}}""" +
$"{Environment.NewLine}{Environment.NewLine}data: [DONE]{Environment.NewLine}{Environment.NewLine}";
}
private static async Task<string> CreateScreenshotAsync(byte[] bytes)
{
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");