using System.Net; using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Drawing; using System.Drawing.Imaging; using MeetingAssistant.Screenshots; using Microsoft.Extensions.Logging.Abstractions; #pragma warning disable CA1416 namespace MeetingAssistant.Tests; public sealed class LiteLlmScreenshotOcrClientTests { [Fact] public async Task ExtractUsesAgentEndpointAndModelWhenOcrEndpointAndModelAreBlank() { var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png"); Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!); await File.WriteAllBytesAsync(screenshotPath, [1, 2, 3]); var handler = new RecordingHandler(""" { "output": [ { "content": [ { "type": "output_text", "text": "Visible slide text" } ] } ] } """); var client = new LiteLlmScreenshotOcrClient( () => handler, NullLogger.Instance); var options = new MeetingAssistantOptions { Agent = { Endpoint = "https://summary.local", Model = "summary-model", Key = "agent-key" }, Screenshots = { Ocr = { Endpoint = "", Model = "", Key = "ocr-key" } } }; var result = await client.ExtractAsync( screenshotPath, "Extract screenshot.", options, CancellationToken.None); Assert.Equal("Visible slide text", result.Text); Assert.Null(result.Crop); Assert.Equal(new Uri("https://summary.local/v1/responses"), handler.RequestUri); Assert.Equal("Bearer", handler.Authorization?.Scheme); Assert.Equal("ocr-key", handler.Authorization?.Parameter); using var payload = JsonDocument.Parse(handler.RequestBody!); Assert.Equal("summary-model", payload.RootElement.GetProperty("model").GetString()); var content = payload.RootElement .GetProperty("input")[0] .GetProperty("content"); Assert.Equal("Extract screenshot.", content[0].GetProperty("text").GetString()); Assert.Equal("data:image/png;base64,AQID", content[1].GetProperty("image_url").GetString()); } [Fact] public async Task ExtractUsesScreenshotOcrEndpointAndModelWhenConfigured() { var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png"); Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!); await File.WriteAllBytesAsync(screenshotPath, [4, 5, 6]); var handler = new RecordingHandler("""{ "output_text": "OCR result" }"""); var client = new LiteLlmScreenshotOcrClient( () => handler, NullLogger.Instance); var options = new MeetingAssistantOptions { Agent = { Endpoint = "https://summary.local", Model = "summary-model", Key = "agent-key" }, Screenshots = { Ocr = { Endpoint = "https://vision.local/v1", Model = "vision-model", Key = "ocr-key" } } }; var result = await client.ExtractAsync( screenshotPath, "Extract screenshot.", options, CancellationToken.None); Assert.Equal("OCR result", result.Text); Assert.Equal(new Uri("https://vision.local/v1/responses"), handler.RequestUri); using var payload = JsonDocument.Parse(handler.RequestBody!); Assert.Equal("vision-model", payload.RootElement.GetProperty("model").GetString()); } [Fact] public async Task ExtractParsesCropMetadataAndOmitsMetadataFromReturnedText() { var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png"); Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!); await File.WriteAllBytesAsync(screenshotPath, CreatePngBytes(8, 6)); var handler = new RecordingHandler(""" { "output_text": "Slide text\n\n```json\n{ \"crop\": { \"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4 } }\n```" } """); var client = new LiteLlmScreenshotOcrClient( () => handler, NullLogger.Instance); var options = new MeetingAssistantOptions { Agent = { Key = "agent-key" } }; var result = await client.ExtractAsync( screenshotPath, "Extract screenshot.", options, CancellationToken.None); Assert.Equal("Slide text", result.Text); Assert.Equal(new ScreenshotCropCoordinates(1, 2, 3, 4), result.Crop); using var payload = JsonDocument.Parse(handler.RequestBody!); Assert.Contains( "Original screenshot dimensions: 8x6 pixels.", payload.RootElement.GetProperty("input")[0].GetProperty("content")[0].GetProperty("text").GetString(), StringComparison.Ordinal); } private sealed class RecordingHandler : HttpMessageHandler { private readonly string responseBody; public RecordingHandler(string responseBody) { this.responseBody = responseBody; } public Uri? RequestUri { get; private set; } public AuthenticationHeaderValue? Authorization { get; private set; } public string? RequestBody { get; private set; } protected override async Task SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) { RequestUri = request.RequestUri; Authorization = request.Headers.Authorization; RequestBody = request.Content is null ? null : await request.Content.ReadAsStringAsync(cancellationToken); return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(responseBody, Encoding.UTF8, "application/json") }; } } private static byte[] CreatePngBytes(int width, int height) { using var bitmap = new Bitmap(width, height); using (var graphics = Graphics.FromImage(bitmap)) { graphics.Clear(Color.White); } using var stream = new MemoryStream(); bitmap.Save(stream, ImageFormat.Png); return stream.ToArray(); } } #pragma warning restore CA1416