using System.Net.Http.Headers; using System.ClientModel; using System.ClientModel.Primitives; using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using Microsoft.Agents.AI.Compaction; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; using OpenAI; using OpenAI.Responses; namespace MeetingAssistant.Summary; #pragma warning disable MAAI001 #pragma warning disable OPENAI001 public sealed class LiteLlmResponsesChatClient : IChatClient { private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); private readonly HttpClient httpClient; private readonly ResponsesClient responsesClient; private readonly AsyncLocal requestInitiator = new(); private readonly string model; private readonly bool useStreaming; private readonly bool enableThinking; private readonly string reasoningEffort; private readonly int reconnectionAttempts; private readonly TimeSpan reconnectionDelay; private readonly LiteLlmResponsesCompactionOptions? compactionOptions; private readonly ILogger? logger; private readonly Action? retrying; private readonly Action? reasoningSummaryChanged; private int responsesRequestCount; public LiteLlmResponsesChatClient( Uri endpoint, string apiKey, string model, bool enableThinking, string reasoningEffort, int reconnectionAttempts, TimeSpan reconnectionDelay, LiteLlmResponsesCompactionOptions? compactionOptions = null, ILogger? logger = null, bool firstRequestIsUser = true, Action? retrying = null, Action? reasoningSummaryChanged = null, bool useStreaming = true) : this( new HttpClient { BaseAddress = NormalizeEndpoint(endpoint) }, apiKey, model, enableThinking, reasoningEffort, reconnectionAttempts, reconnectionDelay, compactionOptions, logger, firstRequestIsUser, retrying, reasoningSummaryChanged, useStreaming) { } internal LiteLlmResponsesChatClient( HttpClient httpClient, string apiKey, string model, bool enableThinking, string reasoningEffort, int reconnectionAttempts, TimeSpan reconnectionDelay, LiteLlmResponsesCompactionOptions? compactionOptions = null, ILogger? logger = null, bool firstRequestIsUser = true, Action? retrying = null, Action? reasoningSummaryChanged = null, bool useStreaming = true) { this.httpClient = httpClient; this.httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey); responsesClient = CreateResponsesClient(httpClient, apiKey, requestInitiator); this.model = model; this.useStreaming = useStreaming; this.enableThinking = enableThinking; this.reasoningEffort = reasoningEffort; this.reconnectionAttempts = Math.Max(0, reconnectionAttempts); this.reconnectionDelay = reconnectionDelay > TimeSpan.Zero ? reconnectionDelay : TimeSpan.Zero; this.compactionOptions = compactionOptions; this.logger = logger; this.retrying = retrying; this.reasoningSummaryChanged = reasoningSummaryChanged; responsesRequestCount = firstRequestIsUser ? 0 : 1; } public void Dispose() { httpClient.Dispose(); } internal static LiteLlmResponsesChatClient Create( AgentOptions options, string apiKey, LiteLlmResponsesCompactionOptions? compactionOptions, ILogger? logger, bool firstRequestIsUser, Action? retrying = null, Action? reasoningSummaryChanged = null) { return new LiteLlmResponsesChatClient( new Uri(options.Endpoint), apiKey, options.Model, options.EnableThinking, options.ReasoningEffort switch { ReasoningEffortOption.None => "none", ReasoningEffortOption.Low => "low", ReasoningEffortOption.High => "high", ReasoningEffortOption.ExtraHigh => "xhigh", _ => "medium" }, options.ReconnectionAttempts, options.ReconnectionDelay, compactionOptions, logger, firstRequestIsUser, retrying, reasoningSummaryChanged, options.UseStreaming); } public async Task GetResponseAsync( IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) { var payload = await CreateCompactedPayloadAsync(messages.ToList(), options, cancellationToken).ConfigureAwait(false); var result = await PostWithRetryAsync(payload, cancellationToken).ConfigureAwait(false); ReportVisibleReasoningSummaries(result.ReasoningSummaries); LogResponseDiagnostics(result.Response); ThrowIfResponseHasNoContent(result.Response); LogResponseUsage(result.Response.Usage); return result.Response; } public async IAsyncEnumerable GetStreamingResponseAsync( IEnumerable messages, ChatOptions? options = null, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) { var response = await GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false); foreach (var message in response.Messages) { yield return new ChatResponseUpdate(message.Role, message.Contents); } } public object? GetService(Type serviceType, object? serviceKey = null) { if (serviceKey is not null) { return null; } return serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("litellm-responses", httpClient.BaseAddress, model) : null; } private static IReadOnlyList ExtractVisibleReasoningSummaries(ChatResponse response) { return response.Messages .SelectMany(message => message.Contents) .OfType() .Select(content => content.Text) .Where(text => !string.IsNullOrWhiteSpace(text)) .ToArray(); } private void ReportVisibleReasoningSummaries(IReadOnlyList summaries) { if (reasoningSummaryChanged is null) { return; } foreach (var summary in summaries) { try { reasoningSummaryChanged(summary); } catch (Exception exception) { logger?.LogWarning( exception, "Reasoning summary callback failed; continuing with the LiteLLM response."); } } } private async Task CreateCompactedPayloadAsync( IReadOnlyList messages, ChatOptions? options, CancellationToken cancellationToken) { var payload = CreatePayload(messages, options); var estimate = EstimateTokens(payload); var triggerTokens = compactionOptions?.TriggerTokens ?? int.MaxValue; if (compactionOptions?.Enabled != true || estimate < triggerTokens) { LogContextWindow(estimate, compacted: false, source: "none"); return payload; } LogContextWindow(estimate, compacted: false, source: "threshold"); var remotePayload = await TryCompactWithResponsesEndpointAsync(payload, estimate, cancellationToken) .ConfigureAwait(false); if (remotePayload is not null) { LogContextWindow(EstimateTokens(remotePayload), compacted: true, source: "responses_compact"); return remotePayload; } if (compactionOptions.FallbackStrategy is null) { return payload; } try { var compactedMessages = await CompactionProvider.CompactAsync( compactionOptions.FallbackStrategy, messages, logger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, cancellationToken) .ConfigureAwait(false); var fallbackPayload = CreatePayload(compactedMessages, options); LogContextWindow(EstimateTokens(fallbackPayload), compacted: true, source: "agent_framework"); return fallbackPayload; } catch (Exception exception) when (exception is not OperationCanceledException) { logger?.LogWarning( exception, "Agent Framework compaction fallback failed; sending un-compacted summary request"); return payload; } } private async Task TryCompactWithResponsesEndpointAsync( JsonObject payload, int estimatedTokens, CancellationToken cancellationToken) { try { var request = new JsonObject { ["model"] = payload["model"]?.DeepClone(), ["input"] = payload["input"]?.DeepClone(), ["instructions"] = payload["instructions"]?.DeepClone(), ["context_window_tokens"] = compactionOptions!.ContextWindowTokens, ["max_output_tokens"] = compactionOptions.MaxOutputTokens, ["estimated_input_tokens"] = estimatedTokens, ["target_input_tokens"] = compactionOptions.TriggerTokens, ["strategy"] = new JsonArray { "collapse_tool_results", "summarize_older_spans", "keep_last_4_user_turns", "drop_oldest_groups" } }; using var requestMessage = CreateJsonRequest( NormalizeRelativePath(compactionOptions.CompactPath), request.ToJsonString(JsonOptions), "agent"); using var response = await httpClient.SendAsync(requestMessage, cancellationToken).ConfigureAwait(false); var responseJson = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); if (!response.IsSuccessStatusCode) { logger?.LogWarning( "Responses compaction endpoint failed with {StatusCode} {ReasonPhrase}: {Body}", (int)response.StatusCode, response.ReasonPhrase, responseJson); return null; } return ApplyCompactedInput(payload, responseJson); } catch (Exception exception) when (exception is not OperationCanceledException) { logger?.LogWarning(exception, "Responses compaction endpoint failed; falling back to Agent Framework compaction"); return null; } } private static JsonObject? ApplyCompactedInput(JsonObject payload, string responseJson) { using var document = JsonDocument.Parse(responseJson); var root = document.RootElement; var input = FindCompactedInput(root); if (input is null) { return null; } var compactedPayload = (JsonObject)payload.DeepClone(); compactedPayload["input"] = JsonNode.Parse(input.Value.GetRawText()); if (root.TryGetProperty("instructions", out var instructions) && instructions.ValueKind == JsonValueKind.String) { compactedPayload["instructions"] = instructions.GetString(); } return compactedPayload; } private static JsonElement? FindCompactedInput(JsonElement root) { foreach (var propertyName in new[] { "input", "compacted_input", "messages", "output" }) { if (root.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.Array) { return value; } } return null; } private JsonObject CreatePayload(IEnumerable messages, ChatOptions? options) { var instructions = new StringBuilder(); if (!string.IsNullOrWhiteSpace(options?.Instructions)) { instructions.AppendLine(options.Instructions); } var input = new JsonArray(); foreach (var message in messages) { AddInputItem(input, instructions, message); } var payload = new JsonObject { ["model"] = options?.ModelId ?? model, ["input"] = input, ["store"] = false, ["tool_choice"] = "auto" }; if (instructions.Length > 0) { payload["instructions"] = instructions.ToString().Trim(); } if (enableThinking) { payload["reasoning"] = new JsonObject { ["effort"] = reasoningEffort }; } if (options?.MaxOutputTokens is > 0) { payload["max_output_tokens"] = options.MaxOutputTokens.Value; } var tools = CreateTools(options?.Tools); if (tools.Count > 0) { payload["tools"] = tools; payload["parallel_tool_calls"] = true; } return payload; } private async Task PostWithRetryAsync( JsonObject payload, CancellationToken cancellationToken) { var payloadJson = payload.ToJsonString(JsonOptions); Exception? lastException = null; for (var attempt = 0; attempt <= reconnectionAttempts; attempt++) { try { var initiator = NextInitiator(); LogRequestDiagnostics(payload, initiator, attempt); var createOptions = ModelReaderWriter.Read( BinaryData.FromString(payloadJson), ModelReaderWriterOptions.Json) ?? throw new InvalidOperationException("Unable to create OpenAI Responses request options."); createOptions.StreamingEnabled = useStreaming; var previousInitiator = requestInitiator.Value; requestInitiator.Value = initiator; try { return useStreaming ? await GetStreamingResponseAsync(createOptions, cancellationToken).ConfigureAwait(false) : CreateNonStreamingResult((await responsesClient .CreateResponseAsync(createOptions, cancellationToken) .ConfigureAwait(false)) .Value .AsChatResponse(createOptions)); } finally { requestInitiator.Value = previousInitiator; } } catch (ClientResultException exception) when (!IsRetryableStatusCode(exception.Status) || attempt == reconnectionAttempts) { throw new InvalidOperationException( $"LiteLLM Responses request failed with {exception.Status}: {exception.Message}", exception); } catch (Exception exception) when (IsRetryableException(exception) && attempt < reconnectionAttempts) { lastException = exception; } retrying?.Invoke(); await DelayBeforeRetryAsync(cancellationToken).ConfigureAwait(false); } throw lastException ?? new InvalidOperationException("LiteLLM Responses request failed."); } private async Task GetStreamingResponseAsync( CreateResponseOptions createOptions, CancellationToken cancellationToken) { var updates = new List(); var reasoningSummaries = new Dictionary<(string ItemId, int SummaryIndex), StringBuilder>(); await foreach (var update in responsesClient .CreateResponseStreamingAsync(createOptions, cancellationToken) .AsChatResponseUpdatesAsync(createOptions, cancellationToken) .ConfigureAwait(false)) { updates.Add(update); var reasoningUpdate = update.RawRepresentation as StreamingResponseReasoningSummaryTextDeltaUpdate; if (reasoningUpdate is null) { continue; } var key = (reasoningUpdate.ItemId, reasoningUpdate.SummaryIndex); if (!reasoningSummaries.TryGetValue(key, out var summary)) { summary = new StringBuilder(); reasoningSummaries[key] = summary; } summary.Append(reasoningUpdate.Delta); } return new ResponsesChatResult( updates.ToChatResponse(), reasoningSummaries.Values .Select(summary => summary.ToString()) .Where(summary => !string.IsNullOrWhiteSpace(summary)) .ToArray()); } private static ResponsesChatResult CreateNonStreamingResult(ChatResponse response) { return new ResponsesChatResult(response, ExtractVisibleReasoningSummaries(response)); } private HttpRequestMessage CreateJsonRequest( string requestUri, string payloadJson, string? initiator = null) { var request = new HttpRequestMessage(HttpMethod.Post, requestUri) { Content = new StringContent(payloadJson, Encoding.UTF8, "application/json") }; if (!string.IsNullOrWhiteSpace(initiator)) { request.Headers.TryAddWithoutValidation("X-Initiator", initiator); } return request; } private string NextInitiator() { return Interlocked.Increment(ref responsesRequestCount) == 1 ? "user" : "agent"; } private async Task DelayBeforeRetryAsync(CancellationToken cancellationToken) { if (reconnectionDelay > TimeSpan.Zero) { await Task.Delay(reconnectionDelay, cancellationToken).ConfigureAwait(false); } } private static bool IsRetryableStatusCode(int statusCode) { return statusCode is 408 or 409 or 425 or 429 || statusCode >= 500; } private static bool IsRetryableException(Exception exception) { return exception is ClientResultException clientException && IsRetryableStatusCode(clientException.Status) || exception is HttpRequestException or TaskCanceledException; } private void LogContextWindow(int estimatedTokens, bool compacted, string source) { if (compactionOptions is null || logger is null) { return; } var contextWindow = Math.Max(1, compactionOptions.ContextWindowTokens); var remainingTokens = Math.Max(0, contextWindow - estimatedTokens); logger.LogInformation( "Summary context estimate: {EstimatedTokens}/{ContextWindowTokens} tokens, {RemainingTokens} remaining, compacted={Compacted}, source={Source}", estimatedTokens, contextWindow, remainingTokens, compacted, source); } private void LogResponseUsage(UsageDetails? usage) { if (usage is null || compactionOptions is null || logger is null) { return; } var contextWindow = Math.Max(1, compactionOptions.ContextWindowTokens); var inputTokens = usage.InputTokenCount ?? 0; var outputTokens = usage.OutputTokenCount ?? 0; var totalTokens = usage.TotalTokenCount ?? inputTokens + outputTokens; logger.LogInformation( "Summary response usage: input={InputTokens}, output={OutputTokens}, total={TotalTokens}, contextWindow={ContextWindowTokens}", inputTokens, outputTokens, totalTokens, contextWindow); } private void LogRequestDiagnostics(JsonObject payload, string initiator, int attempt) { if (logger is null) { return; } var inputCount = payload.TryGetPropertyValue("input", out var input) && input is JsonArray inputArray ? inputArray.Count : 0; var tools = payload.TryGetPropertyValue("tools", out var toolsNode) && toolsNode is JsonArray toolsArray ? toolsArray : []; var toolNames = tools .OfType() .Select(tool => tool.TryGetPropertyValue("name", out var name) ? name?.GetValue() : null) .Where(name => !string.IsNullOrWhiteSpace(name)) .ToArray(); var instructionsPreview = payload.TryGetPropertyValue("instructions", out var instructionsNode) ? Truncate(instructionsNode?.GetValue() ?? "", maxLength: 500) : ""; logger.LogInformation( "LiteLLM Responses request: initiator={Initiator}, attempt={Attempt}, inputItems={InputItems}, tools={ToolCount}, toolNames={ToolNames}, instructionsPreview={InstructionsPreview}", initiator, attempt + 1, inputCount, toolNames.Length, string.Join(", ", toolNames), instructionsPreview); } private void LogResponseDiagnostics(ChatResponse response) { if (logger is null) { return; } var parsedTypes = response.Messages .SelectMany(message => message.Contents) .Select(content => content.GetType().Name) .ToArray(); var outputTypes = GetOutputItemTypes(response); var functionCalls = response.Messages .SelectMany(message => message.Contents) .OfType() .Select(call => call.Name) .ToArray(); logger.LogInformation( "LiteLLM Responses response: responseId={ResponseId}, outputTypes={OutputTypes}, parsedContents={ParsedContents}, functionCalls={FunctionCalls}, textLength={TextLength}", response.ResponseId, string.Join(", ", outputTypes), string.Join(", ", parsedTypes), string.Join(", ", functionCalls), response.Text?.Length ?? 0); if (parsedTypes.Length == 0) { logger.LogWarning( "LiteLLM Responses response contained no parseable message text or function calls."); } } private static void ThrowIfResponseHasNoContent(ChatResponse response) { if (response.Messages.SelectMany(message => message.Contents).Any()) { return; } var outputTypes = GetOutputItemTypes(response); throw new InvalidOperationException( "LiteLLM Responses returned no parseable message text or function calls. " + $"ResponseId={response.ResponseId ?? ""}, outputTypes=[{string.Join(", ", outputTypes)}]."); } private static string[] GetOutputItemTypes(ChatResponse response) { return response.Messages .SelectMany(message => message.Contents) .Select(content => content.RawRepresentation?.GetType().Name ?? content.GetType().Name) .ToArray(); } private static string Truncate(string value, int maxLength) { return value.Length <= maxLength ? value : value[..maxLength] + "..."; } private static void AddInputItem(JsonArray input, StringBuilder instructions, ChatMessage message) { var role = message.Role.Value; var text = message.Text; if (role == ChatRole.System.Value) { if (!string.IsNullOrWhiteSpace(text)) { instructions.AppendLine(text); } return; } if (!string.IsNullOrWhiteSpace(text)) { var isAssistant = role == ChatRole.Assistant.Value; input.Add(new JsonObject { ["type"] = "message", ["role"] = isAssistant ? "assistant" : "user", ["content"] = new JsonArray { new JsonObject { ["type"] = isAssistant ? "output_text" : "input_text", ["text"] = text } } }); } foreach (var content in message.Contents) { if (content is FunctionCallContent call) { input.Add(new JsonObject { ["type"] = "function_call", ["call_id"] = call.CallId, ["name"] = call.Name, ["arguments"] = SerializeFunctionArguments(call) }); } else if (content is FunctionResultContent result) { input.Add(new JsonObject { ["type"] = "function_call_output", ["call_id"] = result.CallId, ["output"] = ResultToString(result.Result) }); } } } private static JsonArray CreateTools(IList? tools) { var result = new JsonArray(); if (tools is null) { return result; } foreach (var tool in tools.OfType()) { result.Add(new JsonObject { ["type"] = "function", ["name"] = tool.Name, ["description"] = tool.Description ?? string.Empty, ["parameters"] = JsonNode.Parse(tool.JsonSchema.GetRawText()) }); } return result; } private static string SerializeFunctionArguments(FunctionCallContent call) { if (call.RawRepresentation is FunctionCallResponseItem responseItem && responseItem.FunctionArguments is not null) { return responseItem.FunctionArguments.ToString(); } return JsonSerializer.Serialize(call.Arguments, JsonOptions); } private static string ResultToString(object? result) { return result switch { null => string.Empty, string text => text, _ => JsonSerializer.Serialize(result, JsonOptions) }; } private static int EstimateTokens(JsonObject payload) { var json = payload.ToJsonString(JsonOptions); return Math.Max(1, (int)Math.Ceiling(json.Length / 4.0)); } private static Uri NormalizeEndpoint(Uri endpoint) { var value = endpoint.ToString().TrimEnd('/'); if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase)) { value += "/v1"; } return new Uri(value + "/"); } private static string NormalizeRelativePath(string value) { return value.TrimStart('/'); } private static ResponsesClient CreateResponsesClient( HttpClient httpClient, string apiKey, AsyncLocal requestInitiator) { var options = new OpenAIClientOptions { Endpoint = httpClient.BaseAddress ?? throw new InvalidOperationException("LiteLLM HTTP client requires a base address."), Transport = new HttpClientPipelineTransport(httpClient), RetryPolicy = new ClientRetryPolicy(0) }; options.AddPolicy( new InitiatorHeaderPolicy(requestInitiator), PipelinePosition.PerCall); return new ResponsesClient( new ApiKeyCredential(apiKey), options); } private sealed record ResponsesChatResult( ChatResponse Response, IReadOnlyList ReasoningSummaries); private sealed class InitiatorHeaderPolicy(AsyncLocal requestInitiator) : PipelinePolicy { public override void Process( PipelineMessage message, IReadOnlyList pipeline, int currentIndex) { SetHeader(message); ProcessNext(message, pipeline, currentIndex); } public override ValueTask ProcessAsync( PipelineMessage message, IReadOnlyList pipeline, int currentIndex) { SetHeader(message); return ProcessNextAsync(message, pipeline, currentIndex); } private void SetHeader(PipelineMessage message) { if (!string.IsNullOrWhiteSpace(requestInitiator.Value)) { message.Request.Headers.Set("X-Initiator", requestInitiator.Value); } } } } #pragma warning restore OPENAI001 #pragma warning restore MAAI001