Public Access
619 lines
21 KiB
C#
619 lines
21 KiB
C#
using System.Net.Http.Headers;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Nodes;
|
|
using Microsoft.Agents.AI.Compaction;
|
|
using Microsoft.Extensions.AI;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace MeetingAssistant.Summary;
|
|
|
|
#pragma warning disable MAAI001
|
|
public sealed class LiteLlmResponsesChatClient : IChatClient
|
|
{
|
|
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
|
|
|
private readonly HttpClient httpClient;
|
|
private readonly string model;
|
|
private readonly bool enableThinking;
|
|
private readonly string reasoningEffort;
|
|
private readonly int reconnectionAttempts;
|
|
private readonly TimeSpan reconnectionDelay;
|
|
private readonly LiteLlmResponsesCompactionOptions? compactionOptions;
|
|
private readonly ILogger? logger;
|
|
|
|
public LiteLlmResponsesChatClient(
|
|
Uri endpoint,
|
|
string apiKey,
|
|
string model,
|
|
bool enableThinking,
|
|
string reasoningEffort,
|
|
int reconnectionAttempts,
|
|
TimeSpan reconnectionDelay,
|
|
LiteLlmResponsesCompactionOptions? compactionOptions = null,
|
|
ILogger? logger = null)
|
|
: this(
|
|
new HttpClient { BaseAddress = NormalizeEndpoint(endpoint) },
|
|
apiKey,
|
|
model,
|
|
enableThinking,
|
|
reasoningEffort,
|
|
reconnectionAttempts,
|
|
reconnectionDelay,
|
|
compactionOptions,
|
|
logger)
|
|
{
|
|
}
|
|
|
|
internal LiteLlmResponsesChatClient(
|
|
HttpClient httpClient,
|
|
string apiKey,
|
|
string model,
|
|
bool enableThinking,
|
|
string reasoningEffort,
|
|
int reconnectionAttempts,
|
|
TimeSpan reconnectionDelay,
|
|
LiteLlmResponsesCompactionOptions? compactionOptions = null,
|
|
ILogger? logger = null)
|
|
{
|
|
this.httpClient = httpClient;
|
|
this.httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
|
|
this.model = model;
|
|
this.enableThinking = enableThinking;
|
|
this.reasoningEffort = reasoningEffort;
|
|
this.reconnectionAttempts = Math.Max(0, reconnectionAttempts);
|
|
this.reconnectionDelay = reconnectionDelay > TimeSpan.Zero ? reconnectionDelay : TimeSpan.Zero;
|
|
this.compactionOptions = compactionOptions;
|
|
this.logger = logger;
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
httpClient.Dispose();
|
|
}
|
|
|
|
public async Task<ChatResponse> GetResponseAsync(
|
|
IEnumerable<ChatMessage> messages,
|
|
ChatOptions? options = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var payload = await CreateCompactedPayloadAsync(messages.ToList(), options, cancellationToken).ConfigureAwait(false);
|
|
var responseJson = await PostWithRetryAsync(payload, cancellationToken).ConfigureAwait(false);
|
|
|
|
var response = ParseResponseJson(responseJson);
|
|
LogResponseUsage(response.Usage);
|
|
return response;
|
|
}
|
|
|
|
public async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
|
|
IEnumerable<ChatMessage> messages,
|
|
ChatOptions? options = null,
|
|
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default)
|
|
{
|
|
var response = await GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false);
|
|
foreach (var message in response.Messages)
|
|
{
|
|
yield return new ChatResponseUpdate(message.Role, message.Contents);
|
|
}
|
|
}
|
|
|
|
public object? GetService(Type serviceType, object? serviceKey = null)
|
|
{
|
|
if (serviceKey is not null)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
return serviceType == typeof(ChatClientMetadata)
|
|
? new ChatClientMetadata("litellm-responses", httpClient.BaseAddress, model)
|
|
: null;
|
|
}
|
|
|
|
public static ChatResponse ParseResponseJson(string responseJson)
|
|
{
|
|
using var document = JsonDocument.Parse(responseJson);
|
|
var root = document.RootElement;
|
|
var contents = new List<AIContent>();
|
|
|
|
if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array)
|
|
{
|
|
foreach (var item in output.EnumerateArray())
|
|
{
|
|
var type = GetString(item, "type");
|
|
if (type == "message")
|
|
{
|
|
AddMessageContent(contents, item);
|
|
}
|
|
else if (type == "function_call")
|
|
{
|
|
contents.Add(new FunctionCallContent(
|
|
GetRequiredString(item, "call_id"),
|
|
GetRequiredString(item, "name"),
|
|
ParseArguments(GetString(item, "arguments"))));
|
|
}
|
|
}
|
|
}
|
|
|
|
var message = new ChatMessage
|
|
{
|
|
Role = ChatRole.Assistant,
|
|
Contents = contents
|
|
};
|
|
|
|
return new ChatResponse(message)
|
|
{
|
|
ResponseId = GetString(root, "id"),
|
|
ModelId = GetString(root, "model"),
|
|
CreatedAt = GetUnixTimestamp(root, "created_at"),
|
|
Usage = ParseUsage(root),
|
|
RawRepresentation = responseJson
|
|
};
|
|
}
|
|
|
|
private async Task<JsonObject> CreateCompactedPayloadAsync(
|
|
IReadOnlyList<ChatMessage> messages,
|
|
ChatOptions? options,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var payload = CreatePayload(messages, options);
|
|
var estimate = EstimateTokens(payload);
|
|
var triggerTokens = compactionOptions?.TriggerTokens ?? int.MaxValue;
|
|
if (compactionOptions?.Enabled != true || estimate < triggerTokens)
|
|
{
|
|
LogContextWindow(estimate, compacted: false, source: "none");
|
|
return payload;
|
|
}
|
|
|
|
LogContextWindow(estimate, compacted: false, source: "threshold");
|
|
|
|
var remotePayload = await TryCompactWithResponsesEndpointAsync(payload, estimate, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
if (remotePayload is not null)
|
|
{
|
|
LogContextWindow(EstimateTokens(remotePayload), compacted: true, source: "responses_compact");
|
|
return remotePayload;
|
|
}
|
|
|
|
if (compactionOptions.FallbackStrategy is null)
|
|
{
|
|
return payload;
|
|
}
|
|
|
|
try
|
|
{
|
|
var compactedMessages = await CompactionProvider.CompactAsync(
|
|
compactionOptions.FallbackStrategy,
|
|
messages,
|
|
logger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance,
|
|
cancellationToken)
|
|
.ConfigureAwait(false);
|
|
var fallbackPayload = CreatePayload(compactedMessages, options);
|
|
LogContextWindow(EstimateTokens(fallbackPayload), compacted: true, source: "agent_framework");
|
|
return fallbackPayload;
|
|
}
|
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
|
{
|
|
logger?.LogWarning(
|
|
exception,
|
|
"Agent Framework compaction fallback failed; sending un-compacted summary request");
|
|
return payload;
|
|
}
|
|
}
|
|
|
|
private async Task<JsonObject?> TryCompactWithResponsesEndpointAsync(
|
|
JsonObject payload,
|
|
int estimatedTokens,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
var request = new JsonObject
|
|
{
|
|
["model"] = payload["model"]?.DeepClone(),
|
|
["input"] = payload["input"]?.DeepClone(),
|
|
["instructions"] = payload["instructions"]?.DeepClone(),
|
|
["context_window_tokens"] = compactionOptions!.ContextWindowTokens,
|
|
["max_output_tokens"] = compactionOptions.MaxOutputTokens,
|
|
["estimated_input_tokens"] = estimatedTokens,
|
|
["target_input_tokens"] = compactionOptions.TriggerTokens,
|
|
["strategy"] = new JsonArray
|
|
{
|
|
"collapse_tool_results",
|
|
"summarize_older_spans",
|
|
"keep_last_4_user_turns",
|
|
"drop_oldest_groups"
|
|
}
|
|
};
|
|
|
|
using var content = new StringContent(request.ToJsonString(JsonOptions), Encoding.UTF8, "application/json");
|
|
using var response = await httpClient.PostAsync(
|
|
NormalizeRelativePath(compactionOptions.CompactPath),
|
|
content,
|
|
cancellationToken)
|
|
.ConfigureAwait(false);
|
|
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
logger?.LogWarning(
|
|
"Responses compaction endpoint failed with {StatusCode} {ReasonPhrase}: {Body}",
|
|
(int)response.StatusCode,
|
|
response.ReasonPhrase,
|
|
responseJson);
|
|
return null;
|
|
}
|
|
|
|
return ApplyCompactedInput(payload, responseJson);
|
|
}
|
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
|
{
|
|
logger?.LogWarning(exception, "Responses compaction endpoint failed; falling back to Agent Framework compaction");
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static JsonObject? ApplyCompactedInput(JsonObject payload, string responseJson)
|
|
{
|
|
using var document = JsonDocument.Parse(responseJson);
|
|
var root = document.RootElement;
|
|
var input = FindCompactedInput(root);
|
|
if (input is null)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var compactedPayload = (JsonObject)payload.DeepClone();
|
|
compactedPayload["input"] = JsonNode.Parse(input.Value.GetRawText());
|
|
if (root.TryGetProperty("instructions", out var instructions) && instructions.ValueKind == JsonValueKind.String)
|
|
{
|
|
compactedPayload["instructions"] = instructions.GetString();
|
|
}
|
|
|
|
return compactedPayload;
|
|
}
|
|
|
|
private static JsonElement? FindCompactedInput(JsonElement root)
|
|
{
|
|
foreach (var propertyName in new[] { "input", "compacted_input", "messages", "output" })
|
|
{
|
|
if (root.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.Array)
|
|
{
|
|
return value;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private JsonObject CreatePayload(IEnumerable<ChatMessage> messages, ChatOptions? options)
|
|
{
|
|
var instructions = new StringBuilder();
|
|
if (!string.IsNullOrWhiteSpace(options?.Instructions))
|
|
{
|
|
instructions.AppendLine(options.Instructions);
|
|
}
|
|
|
|
var input = new JsonArray();
|
|
foreach (var message in messages)
|
|
{
|
|
AddInputItem(input, instructions, message);
|
|
}
|
|
|
|
var payload = new JsonObject
|
|
{
|
|
["model"] = options?.ModelId ?? model,
|
|
["input"] = input,
|
|
["store"] = false,
|
|
["tool_choice"] = "auto"
|
|
};
|
|
|
|
if (instructions.Length > 0)
|
|
{
|
|
payload["instructions"] = instructions.ToString().Trim();
|
|
}
|
|
|
|
if (enableThinking)
|
|
{
|
|
payload["reasoning"] = new JsonObject
|
|
{
|
|
["effort"] = reasoningEffort
|
|
};
|
|
}
|
|
|
|
var tools = CreateTools(options?.Tools);
|
|
if (tools.Count > 0)
|
|
{
|
|
payload["tools"] = tools;
|
|
payload["parallel_tool_calls"] = true;
|
|
}
|
|
|
|
return payload;
|
|
}
|
|
|
|
private async Task<string> PostWithRetryAsync(JsonObject payload, CancellationToken cancellationToken)
|
|
{
|
|
var payloadJson = payload.ToJsonString(JsonOptions);
|
|
Exception? lastException = null;
|
|
|
|
for (var attempt = 0; attempt <= reconnectionAttempts; attempt++)
|
|
{
|
|
try
|
|
{
|
|
using var content = new StringContent(payloadJson, Encoding.UTF8, "application/json");
|
|
using var response = await httpClient.PostAsync("responses", content, cancellationToken).ConfigureAwait(false);
|
|
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
|
|
|
if (response.IsSuccessStatusCode)
|
|
{
|
|
return responseJson;
|
|
}
|
|
|
|
var exception = new InvalidOperationException(
|
|
$"LiteLLM Responses request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}");
|
|
if (!IsRetryableStatusCode((int)response.StatusCode) || attempt == reconnectionAttempts)
|
|
{
|
|
throw exception;
|
|
}
|
|
|
|
lastException = exception;
|
|
}
|
|
catch (Exception exception) when (IsRetryableException(exception) && attempt < reconnectionAttempts)
|
|
{
|
|
lastException = exception;
|
|
}
|
|
|
|
await DelayBeforeRetryAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
throw lastException ?? new InvalidOperationException("LiteLLM Responses request failed.");
|
|
}
|
|
|
|
private async Task DelayBeforeRetryAsync(CancellationToken cancellationToken)
|
|
{
|
|
if (reconnectionDelay > TimeSpan.Zero)
|
|
{
|
|
await Task.Delay(reconnectionDelay, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
private static bool IsRetryableStatusCode(int statusCode)
|
|
{
|
|
return statusCode is 408 or 409 or 425 or 429
|
|
|| statusCode >= 500;
|
|
}
|
|
|
|
private static bool IsRetryableException(Exception exception)
|
|
{
|
|
return exception is HttpRequestException or TaskCanceledException;
|
|
}
|
|
|
|
private void LogContextWindow(int estimatedTokens, bool compacted, string source)
|
|
{
|
|
if (compactionOptions is null || logger is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var contextWindow = Math.Max(1, compactionOptions.ContextWindowTokens);
|
|
var remainingTokens = Math.Max(0, contextWindow - estimatedTokens);
|
|
logger.LogInformation(
|
|
"Summary context estimate: {EstimatedTokens}/{ContextWindowTokens} tokens, {RemainingTokens} remaining, compacted={Compacted}, source={Source}",
|
|
estimatedTokens,
|
|
contextWindow,
|
|
remainingTokens,
|
|
compacted,
|
|
source);
|
|
}
|
|
|
|
private void LogResponseUsage(UsageDetails? usage)
|
|
{
|
|
if (usage is null || compactionOptions is null || logger is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var contextWindow = Math.Max(1, compactionOptions.ContextWindowTokens);
|
|
var inputTokens = usage.InputTokenCount ?? 0;
|
|
var outputTokens = usage.OutputTokenCount ?? 0;
|
|
var totalTokens = usage.TotalTokenCount ?? inputTokens + outputTokens;
|
|
logger.LogInformation(
|
|
"Summary response usage: input={InputTokens}, output={OutputTokens}, total={TotalTokens}, contextWindow={ContextWindowTokens}",
|
|
inputTokens,
|
|
outputTokens,
|
|
totalTokens,
|
|
contextWindow);
|
|
}
|
|
|
|
private static void AddInputItem(JsonArray input, StringBuilder instructions, ChatMessage message)
|
|
{
|
|
var role = message.Role.Value;
|
|
var text = message.Text;
|
|
if (role == ChatRole.System.Value)
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(text))
|
|
{
|
|
instructions.AppendLine(text);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(text))
|
|
{
|
|
input.Add(new JsonObject
|
|
{
|
|
["role"] = role == ChatRole.Assistant.Value ? "assistant" : "user",
|
|
["content"] = text
|
|
});
|
|
}
|
|
|
|
foreach (var content in message.Contents)
|
|
{
|
|
if (content is FunctionCallContent call)
|
|
{
|
|
input.Add(new JsonObject
|
|
{
|
|
["type"] = "function_call",
|
|
["call_id"] = call.CallId,
|
|
["name"] = call.Name,
|
|
["arguments"] = JsonSerializer.Serialize(call.Arguments, JsonOptions)
|
|
});
|
|
}
|
|
else if (content is FunctionResultContent result)
|
|
{
|
|
input.Add(new JsonObject
|
|
{
|
|
["type"] = "function_call_output",
|
|
["call_id"] = result.CallId,
|
|
["output"] = ResultToString(result.Result)
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
private static JsonArray CreateTools(IList<AITool>? tools)
|
|
{
|
|
var result = new JsonArray();
|
|
if (tools is null)
|
|
{
|
|
return result;
|
|
}
|
|
|
|
foreach (var tool in tools.OfType<AIFunctionDeclaration>())
|
|
{
|
|
result.Add(new JsonObject
|
|
{
|
|
["type"] = "function",
|
|
["name"] = tool.Name,
|
|
["description"] = tool.Description ?? string.Empty,
|
|
["parameters"] = JsonNode.Parse(tool.JsonSchema.GetRawText())
|
|
});
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
private static void AddMessageContent(List<AIContent> contents, JsonElement item)
|
|
{
|
|
if (!item.TryGetProperty("content", out var messageContent) || messageContent.ValueKind != JsonValueKind.Array)
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var content in messageContent.EnumerateArray())
|
|
{
|
|
if (GetString(content, "type") == "output_text")
|
|
{
|
|
var text = GetString(content, "text");
|
|
if (!string.IsNullOrEmpty(text))
|
|
{
|
|
contents.Add(new TextContent(text));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static Dictionary<string, object?> ParseArguments(string? arguments)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(arguments))
|
|
{
|
|
return [];
|
|
}
|
|
|
|
using var document = JsonDocument.Parse(arguments);
|
|
return document.RootElement.EnumerateObject()
|
|
.ToDictionary(property => property.Name, property => ConvertJsonValue(property.Value));
|
|
}
|
|
|
|
private static object? ConvertJsonValue(JsonElement element)
|
|
{
|
|
return element.ValueKind switch
|
|
{
|
|
JsonValueKind.String => element.GetString(),
|
|
JsonValueKind.Number when element.TryGetInt64(out var integer) => integer,
|
|
JsonValueKind.Number when element.TryGetDouble(out var number) => number,
|
|
JsonValueKind.True => true,
|
|
JsonValueKind.False => false,
|
|
JsonValueKind.Null => null,
|
|
_ => JsonSerializer.Deserialize<object>(element.GetRawText(), JsonOptions)
|
|
};
|
|
}
|
|
|
|
private static string ResultToString(object? result)
|
|
{
|
|
return result switch
|
|
{
|
|
null => string.Empty,
|
|
string text => text,
|
|
_ => JsonSerializer.Serialize(result, JsonOptions)
|
|
};
|
|
}
|
|
|
|
private static UsageDetails? ParseUsage(JsonElement root)
|
|
{
|
|
if (!root.TryGetProperty("usage", out var usage) || usage.ValueKind != JsonValueKind.Object)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var inputTokens = GetInt(usage, "input_tokens") ?? GetInt(usage, "prompt_tokens");
|
|
var outputTokens = GetInt(usage, "output_tokens") ?? GetInt(usage, "completion_tokens");
|
|
var totalTokens = GetInt(usage, "total_tokens");
|
|
return new UsageDetails
|
|
{
|
|
InputTokenCount = inputTokens,
|
|
OutputTokenCount = outputTokens,
|
|
TotalTokenCount = totalTokens
|
|
};
|
|
}
|
|
|
|
private static int EstimateTokens(JsonObject payload)
|
|
{
|
|
var json = payload.ToJsonString(JsonOptions);
|
|
return Math.Max(1, (int)Math.Ceiling(json.Length / 4.0));
|
|
}
|
|
|
|
private static int? GetInt(JsonElement element, string propertyName)
|
|
{
|
|
return element.TryGetProperty(propertyName, out var property) && property.TryGetInt32(out var value)
|
|
? value
|
|
: null;
|
|
}
|
|
|
|
private static string GetRequiredString(JsonElement element, string propertyName)
|
|
{
|
|
return GetString(element, propertyName)
|
|
?? throw new InvalidOperationException($"LiteLLM response item did not include '{propertyName}'.");
|
|
}
|
|
|
|
private static string? GetString(JsonElement element, string propertyName)
|
|
{
|
|
return element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String
|
|
? property.GetString()
|
|
: null;
|
|
}
|
|
|
|
private static DateTimeOffset? GetUnixTimestamp(JsonElement element, string propertyName)
|
|
{
|
|
return element.TryGetProperty(propertyName, out var property) && property.TryGetInt64(out var value)
|
|
? DateTimeOffset.FromUnixTimeSeconds(value)
|
|
: null;
|
|
}
|
|
|
|
private static Uri NormalizeEndpoint(Uri endpoint)
|
|
{
|
|
var value = endpoint.ToString().TrimEnd('/');
|
|
if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
value += "/v1";
|
|
}
|
|
|
|
return new Uri(value + "/");
|
|
}
|
|
|
|
private static string NormalizeRelativePath(string value)
|
|
{
|
|
return value.TrimStart('/');
|
|
}
|
|
}
|
|
#pragma warning restore MAAI001
|