using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using Microsoft.Agents.AI.Compaction; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; namespace MeetingAssistant.Summary; #pragma warning disable MAAI001 public sealed class LiteLlmResponsesChatClient : IChatClient { private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); private readonly HttpClient httpClient; private readonly string model; private readonly bool enableThinking; private readonly string reasoningEffort; private readonly int reconnectionAttempts; private readonly TimeSpan reconnectionDelay; private readonly LiteLlmResponsesCompactionOptions? compactionOptions; private readonly ILogger? logger; public LiteLlmResponsesChatClient( Uri endpoint, string apiKey, string model, bool enableThinking, string reasoningEffort, int reconnectionAttempts, TimeSpan reconnectionDelay, LiteLlmResponsesCompactionOptions? compactionOptions = null, ILogger? logger = null) : this( new HttpClient { BaseAddress = NormalizeEndpoint(endpoint) }, apiKey, model, enableThinking, reasoningEffort, reconnectionAttempts, reconnectionDelay, compactionOptions, logger) { } internal LiteLlmResponsesChatClient( HttpClient httpClient, string apiKey, string model, bool enableThinking, string reasoningEffort, int reconnectionAttempts, TimeSpan reconnectionDelay, LiteLlmResponsesCompactionOptions? compactionOptions = null, ILogger? logger = null) { this.httpClient = httpClient; this.httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey); this.model = model; this.enableThinking = enableThinking; this.reasoningEffort = reasoningEffort; this.reconnectionAttempts = Math.Max(0, reconnectionAttempts); this.reconnectionDelay = reconnectionDelay > TimeSpan.Zero ? reconnectionDelay : TimeSpan.Zero; this.compactionOptions = compactionOptions; this.logger = logger; } public void Dispose() { httpClient.Dispose(); } public async Task GetResponseAsync( IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) { var payload = await CreateCompactedPayloadAsync(messages.ToList(), options, cancellationToken).ConfigureAwait(false); var responseJson = await PostWithRetryAsync(payload, cancellationToken).ConfigureAwait(false); var response = ParseResponseJson(responseJson); LogResponseUsage(response.Usage); return response; } public async IAsyncEnumerable GetStreamingResponseAsync( IEnumerable messages, ChatOptions? options = null, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) { var response = await GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false); foreach (var message in response.Messages) { yield return new ChatResponseUpdate(message.Role, message.Contents); } } public object? GetService(Type serviceType, object? serviceKey = null) { if (serviceKey is not null) { return null; } return serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("litellm-responses", httpClient.BaseAddress, model) : null; } public static ChatResponse ParseResponseJson(string responseJson) { using var document = JsonDocument.Parse(responseJson); var root = document.RootElement; var contents = new List(); if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array) { foreach (var item in output.EnumerateArray()) { var type = GetString(item, "type"); if (type == "message") { AddMessageContent(contents, item); } else if (type == "function_call") { contents.Add(new FunctionCallContent( GetRequiredString(item, "call_id"), GetRequiredString(item, "name"), ParseArguments(GetString(item, "arguments")))); } } } var message = new ChatMessage { Role = ChatRole.Assistant, Contents = contents }; return new ChatResponse(message) { ResponseId = GetString(root, "id"), ModelId = GetString(root, "model"), CreatedAt = GetUnixTimestamp(root, "created_at"), Usage = ParseUsage(root), RawRepresentation = responseJson }; } private async Task CreateCompactedPayloadAsync( IReadOnlyList messages, ChatOptions? options, CancellationToken cancellationToken) { var payload = CreatePayload(messages, options); var estimate = EstimateTokens(payload); var triggerTokens = compactionOptions?.TriggerTokens ?? int.MaxValue; if (compactionOptions?.Enabled != true || estimate < triggerTokens) { LogContextWindow(estimate, compacted: false, source: "none"); return payload; } LogContextWindow(estimate, compacted: false, source: "threshold"); var remotePayload = await TryCompactWithResponsesEndpointAsync(payload, estimate, cancellationToken) .ConfigureAwait(false); if (remotePayload is not null) { LogContextWindow(EstimateTokens(remotePayload), compacted: true, source: "responses_compact"); return remotePayload; } if (compactionOptions.FallbackStrategy is null) { return payload; } try { var compactedMessages = await CompactionProvider.CompactAsync( compactionOptions.FallbackStrategy, messages, logger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, cancellationToken) .ConfigureAwait(false); var fallbackPayload = CreatePayload(compactedMessages, options); LogContextWindow(EstimateTokens(fallbackPayload), compacted: true, source: "agent_framework"); return fallbackPayload; } catch (Exception exception) when (exception is not OperationCanceledException) { logger?.LogWarning( exception, "Agent Framework compaction fallback failed; sending un-compacted summary request"); return payload; } } private async Task TryCompactWithResponsesEndpointAsync( JsonObject payload, int estimatedTokens, CancellationToken cancellationToken) { try { var request = new JsonObject { ["model"] = payload["model"]?.DeepClone(), ["input"] = payload["input"]?.DeepClone(), ["instructions"] = payload["instructions"]?.DeepClone(), ["context_window_tokens"] = compactionOptions!.ContextWindowTokens, ["max_output_tokens"] = compactionOptions.MaxOutputTokens, ["estimated_input_tokens"] = estimatedTokens, ["target_input_tokens"] = compactionOptions.TriggerTokens, ["strategy"] = new JsonArray { "collapse_tool_results", "summarize_older_spans", "keep_last_4_user_turns", "drop_oldest_groups" } }; using var content = new StringContent(request.ToJsonString(JsonOptions), Encoding.UTF8, "application/json"); using var response = await httpClient.PostAsync( NormalizeRelativePath(compactionOptions.CompactPath), content, cancellationToken) .ConfigureAwait(false); var responseJson = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); if (!response.IsSuccessStatusCode) { logger?.LogWarning( "Responses compaction endpoint failed with {StatusCode} {ReasonPhrase}: {Body}", (int)response.StatusCode, response.ReasonPhrase, responseJson); return null; } return ApplyCompactedInput(payload, responseJson); } catch (Exception exception) when (exception is not OperationCanceledException) { logger?.LogWarning(exception, "Responses compaction endpoint failed; falling back to Agent Framework compaction"); return null; } } private static JsonObject? ApplyCompactedInput(JsonObject payload, string responseJson) { using var document = JsonDocument.Parse(responseJson); var root = document.RootElement; var input = FindCompactedInput(root); if (input is null) { return null; } var compactedPayload = (JsonObject)payload.DeepClone(); compactedPayload["input"] = JsonNode.Parse(input.Value.GetRawText()); if (root.TryGetProperty("instructions", out var instructions) && instructions.ValueKind == JsonValueKind.String) { compactedPayload["instructions"] = instructions.GetString(); } return compactedPayload; } private static JsonElement? FindCompactedInput(JsonElement root) { foreach (var propertyName in new[] { "input", "compacted_input", "messages", "output" }) { if (root.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.Array) { return value; } } return null; } private JsonObject CreatePayload(IEnumerable messages, ChatOptions? options) { var instructions = new StringBuilder(); if (!string.IsNullOrWhiteSpace(options?.Instructions)) { instructions.AppendLine(options.Instructions); } var input = new JsonArray(); foreach (var message in messages) { AddInputItem(input, instructions, message); } var payload = new JsonObject { ["model"] = options?.ModelId ?? model, ["input"] = input, ["store"] = false, ["tool_choice"] = "auto" }; if (instructions.Length > 0) { payload["instructions"] = instructions.ToString().Trim(); } if (enableThinking) { payload["reasoning"] = new JsonObject { ["effort"] = reasoningEffort }; } var tools = CreateTools(options?.Tools); if (tools.Count > 0) { payload["tools"] = tools; payload["parallel_tool_calls"] = true; } return payload; } private async Task PostWithRetryAsync(JsonObject payload, CancellationToken cancellationToken) { var payloadJson = payload.ToJsonString(JsonOptions); Exception? lastException = null; for (var attempt = 0; attempt <= reconnectionAttempts; attempt++) { try { using var content = new StringContent(payloadJson, Encoding.UTF8, "application/json"); using var response = await httpClient.PostAsync("responses", content, cancellationToken).ConfigureAwait(false); var responseJson = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); if (response.IsSuccessStatusCode) { return responseJson; } var exception = new InvalidOperationException( $"LiteLLM Responses request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}"); if (!IsRetryableStatusCode((int)response.StatusCode) || attempt == reconnectionAttempts) { throw exception; } lastException = exception; } catch (Exception exception) when (IsRetryableException(exception) && attempt < reconnectionAttempts) { lastException = exception; } await DelayBeforeRetryAsync(cancellationToken).ConfigureAwait(false); } throw lastException ?? new InvalidOperationException("LiteLLM Responses request failed."); } private async Task DelayBeforeRetryAsync(CancellationToken cancellationToken) { if (reconnectionDelay > TimeSpan.Zero) { await Task.Delay(reconnectionDelay, cancellationToken).ConfigureAwait(false); } } private static bool IsRetryableStatusCode(int statusCode) { return statusCode is 408 or 409 or 425 or 429 || statusCode >= 500; } private static bool IsRetryableException(Exception exception) { return exception is HttpRequestException or TaskCanceledException; } private void LogContextWindow(int estimatedTokens, bool compacted, string source) { if (compactionOptions is null || logger is null) { return; } var contextWindow = Math.Max(1, compactionOptions.ContextWindowTokens); var remainingTokens = Math.Max(0, contextWindow - estimatedTokens); logger.LogInformation( "Summary context estimate: {EstimatedTokens}/{ContextWindowTokens} tokens, {RemainingTokens} remaining, compacted={Compacted}, source={Source}", estimatedTokens, contextWindow, remainingTokens, compacted, source); } private void LogResponseUsage(UsageDetails? usage) { if (usage is null || compactionOptions is null || logger is null) { return; } var contextWindow = Math.Max(1, compactionOptions.ContextWindowTokens); var inputTokens = usage.InputTokenCount ?? 0; var outputTokens = usage.OutputTokenCount ?? 0; var totalTokens = usage.TotalTokenCount ?? inputTokens + outputTokens; logger.LogInformation( "Summary response usage: input={InputTokens}, output={OutputTokens}, total={TotalTokens}, contextWindow={ContextWindowTokens}", inputTokens, outputTokens, totalTokens, contextWindow); } private static void AddInputItem(JsonArray input, StringBuilder instructions, ChatMessage message) { var role = message.Role.Value; var text = message.Text; if (role == ChatRole.System.Value) { if (!string.IsNullOrWhiteSpace(text)) { instructions.AppendLine(text); } return; } if (!string.IsNullOrWhiteSpace(text)) { input.Add(new JsonObject { ["role"] = role == ChatRole.Assistant.Value ? "assistant" : "user", ["content"] = text }); } foreach (var content in message.Contents) { if (content is FunctionCallContent call) { input.Add(new JsonObject { ["type"] = "function_call", ["call_id"] = call.CallId, ["name"] = call.Name, ["arguments"] = JsonSerializer.Serialize(call.Arguments, JsonOptions) }); } else if (content is FunctionResultContent result) { input.Add(new JsonObject { ["type"] = "function_call_output", ["call_id"] = result.CallId, ["output"] = ResultToString(result.Result) }); } } } private static JsonArray CreateTools(IList? tools) { var result = new JsonArray(); if (tools is null) { return result; } foreach (var tool in tools.OfType()) { result.Add(new JsonObject { ["type"] = "function", ["name"] = tool.Name, ["description"] = tool.Description ?? string.Empty, ["parameters"] = JsonNode.Parse(tool.JsonSchema.GetRawText()) }); } return result; } private static void AddMessageContent(List contents, JsonElement item) { if (!item.TryGetProperty("content", out var messageContent) || messageContent.ValueKind != JsonValueKind.Array) { return; } foreach (var content in messageContent.EnumerateArray()) { if (GetString(content, "type") == "output_text") { var text = GetString(content, "text"); if (!string.IsNullOrEmpty(text)) { contents.Add(new TextContent(text)); } } } } private static Dictionary ParseArguments(string? arguments) { if (string.IsNullOrWhiteSpace(arguments)) { return []; } using var document = JsonDocument.Parse(arguments); return document.RootElement.EnumerateObject() .ToDictionary(property => property.Name, property => ConvertJsonValue(property.Value)); } private static object? ConvertJsonValue(JsonElement element) { return element.ValueKind switch { JsonValueKind.String => element.GetString(), JsonValueKind.Number when element.TryGetInt64(out var integer) => integer, JsonValueKind.Number when element.TryGetDouble(out var number) => number, JsonValueKind.True => true, JsonValueKind.False => false, JsonValueKind.Null => null, _ => JsonSerializer.Deserialize(element.GetRawText(), JsonOptions) }; } private static string ResultToString(object? result) { return result switch { null => string.Empty, string text => text, _ => JsonSerializer.Serialize(result, JsonOptions) }; } private static UsageDetails? ParseUsage(JsonElement root) { if (!root.TryGetProperty("usage", out var usage) || usage.ValueKind != JsonValueKind.Object) { return null; } var inputTokens = GetInt(usage, "input_tokens") ?? GetInt(usage, "prompt_tokens"); var outputTokens = GetInt(usage, "output_tokens") ?? GetInt(usage, "completion_tokens"); var totalTokens = GetInt(usage, "total_tokens"); return new UsageDetails { InputTokenCount = inputTokens, OutputTokenCount = outputTokens, TotalTokenCount = totalTokens }; } private static int EstimateTokens(JsonObject payload) { var json = payload.ToJsonString(JsonOptions); return Math.Max(1, (int)Math.Ceiling(json.Length / 4.0)); } private static int? GetInt(JsonElement element, string propertyName) { return element.TryGetProperty(propertyName, out var property) && property.TryGetInt32(out var value) ? value : null; } private static string GetRequiredString(JsonElement element, string propertyName) { return GetString(element, propertyName) ?? throw new InvalidOperationException($"LiteLLM response item did not include '{propertyName}'."); } private static string? GetString(JsonElement element, string propertyName) { return element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String ? property.GetString() : null; } private static DateTimeOffset? GetUnixTimestamp(JsonElement element, string propertyName) { return element.TryGetProperty(propertyName, out var property) && property.TryGetInt64(out var value) ? DateTimeOffset.FromUnixTimeSeconds(value) : null; } private static Uri NormalizeEndpoint(Uri endpoint) { var value = endpoint.ToString().TrimEnd('/'); if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase)) { value += "/v1"; } return new Uri(value + "/"); } private static string NormalizeRelativePath(string value) { return value.TrimStart('/'); } } #pragma warning restore MAAI001