meeting-assistant/MeetingAssistant.Tests/LiteLlmScreenshotOcrClientTests.cs

using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Drawing;
using System.Drawing.Imaging;
using MeetingAssistant.Screenshots;
using Microsoft.Extensions.Logging.Abstractions;

#pragma warning disable CA1416

namespace MeetingAssistant.Tests;

public sealed class LiteLlmScreenshotOcrClientTests
{
    [Fact]
    public async Task ExtractUsesAgentEndpointAndModelWhenOcrEndpointAndModelAreBlank()
    {
        var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
        Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
        await File.WriteAllBytesAsync(screenshotPath, [1, 2, 3]);
        var handler = new RecordingHandler("""
            {
              "output": [
                {
                  "content": [
                    { "type": "output_text", "text": "Visible slide text" }
                  ]
                }
              ]
            }
            """);
        var client = new LiteLlmScreenshotOcrClient(
            () => handler,
            NullLogger<LiteLlmScreenshotOcrClient>.Instance);
        var options = new MeetingAssistantOptions
        {
            Agent =
            {
                Endpoint = "https://summary.local",
                Model = "summary-model",
                Key = "agent-key"
            },
            Screenshots =
            {
                Ocr =
                {
                    Endpoint = "",
                    Model = "",
                    Key = "ocr-key"
                }
            }
        };

        var result = await client.ExtractAsync(
            screenshotPath,
            "Extract screenshot.",
            options,
            CancellationToken.None);

        Assert.Equal("Visible slide text", result.Text);
        Assert.Null(result.Crop);
        Assert.Equal(new Uri("https://summary.local/v1/responses"), handler.RequestUri);
        Assert.Equal("Bearer", handler.Authorization?.Scheme);
        Assert.Equal("ocr-key", handler.Authorization?.Parameter);
        using var payload = JsonDocument.Parse(handler.RequestBody!);
        Assert.Equal("summary-model", payload.RootElement.GetProperty("model").GetString());
        var content = payload.RootElement
            .GetProperty("input")[0]
            .GetProperty("content");
        Assert.Equal("Extract screenshot.", content[0].GetProperty("text").GetString());
        Assert.Equal("data:image/png;base64,AQID", content[1].GetProperty("image_url").GetString());
    }

    [Fact]
    public async Task ExtractUsesScreenshotOcrEndpointAndModelWhenConfigured()
    {
        var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
        Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
        await File.WriteAllBytesAsync(screenshotPath, [4, 5, 6]);
        var handler = new RecordingHandler("""{ "output_text": "OCR result" }""");
        var client = new LiteLlmScreenshotOcrClient(
            () => handler,
            NullLogger<LiteLlmScreenshotOcrClient>.Instance);
        var options = new MeetingAssistantOptions
        {
            Agent =
            {
                Endpoint = "https://summary.local",
                Model = "summary-model",
                Key = "agent-key"
            },
            Screenshots =
            {
                Ocr =
                {
                    Endpoint = "https://vision.local/v1",
                    Model = "vision-model",
                    Key = "ocr-key"
                }
            }
        };

        var result = await client.ExtractAsync(
            screenshotPath,
            "Extract screenshot.",
            options,
            CancellationToken.None);

        Assert.Equal("OCR result", result.Text);
        Assert.Equal(new Uri("https://vision.local/v1/responses"), handler.RequestUri);
        using var payload = JsonDocument.Parse(handler.RequestBody!);
        Assert.Equal("vision-model", payload.RootElement.GetProperty("model").GetString());
    }

    [Fact]
    public async Task ExtractParsesCropMetadataAndOmitsMetadataFromReturnedText()
    {
        var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
        Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
        await File.WriteAllBytesAsync(screenshotPath, CreatePngBytes(8, 6));
        var handler = new RecordingHandler("""
            {
              "output_text": "Slide text\n\n```json\n{ \"crop\": { \"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4 } }\n```"
            }
            """);
        var client = new LiteLlmScreenshotOcrClient(
            () => handler,
            NullLogger<LiteLlmScreenshotOcrClient>.Instance);
        var options = new MeetingAssistantOptions
        {
            Agent =
            {
                Key = "agent-key"
            }
        };

        var result = await client.ExtractAsync(
            screenshotPath,
            "Extract screenshot.",
            options,
            CancellationToken.None);

        Assert.Equal("Slide text", result.Text);
        Assert.Equal(new ScreenshotCropCoordinates(1, 2, 3, 4), result.Crop);
        using var payload = JsonDocument.Parse(handler.RequestBody!);
        Assert.Contains(
            "Original screenshot dimensions: 8x6 pixels.",
            payload.RootElement.GetProperty("input")[0].GetProperty("content")[0].GetProperty("text").GetString(),
            StringComparison.Ordinal);
    }

    private sealed class RecordingHandler : HttpMessageHandler
    {
        private readonly string responseBody;

        public RecordingHandler(string responseBody)
        {
            this.responseBody = responseBody;
        }

        public Uri? RequestUri { get; private set; }

        public AuthenticationHeaderValue? Authorization { get; private set; }

        public string? RequestBody { get; private set; }

        protected override async Task<HttpResponseMessage> SendAsync(
            HttpRequestMessage request,
            CancellationToken cancellationToken)
        {
            RequestUri = request.RequestUri;
            Authorization = request.Headers.Authorization;
            RequestBody = request.Content is null
                ? null
                : await request.Content.ReadAsStringAsync(cancellationToken);
            return new HttpResponseMessage(HttpStatusCode.OK)
            {
                Content = new StringContent(responseBody, Encoding.UTF8, "application/json")
            };
        }
    }

    private static byte[] CreatePngBytes(int width, int height)
    {
        using var bitmap = new Bitmap(width, height);
        using (var graphics = Graphics.FromImage(bitmap))
        {
            graphics.Clear(Color.White);
        }

        using var stream = new MemoryStream();
        bitmap.Save(stream, ImageFormat.Png);
        return stream.ToArray();
    }
}

#pragma warning restore CA1416