Public Access
889 lines
30 KiB
C#
889 lines
30 KiB
C#
using System.Net.Http.Headers;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Nodes;
|
|
using Microsoft.Agents.AI.Compaction;
|
|
using Microsoft.Extensions.AI;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace MeetingAssistant.Summary;
|
|
|
|
#pragma warning disable MAAI001
|
|
public sealed class LiteLlmResponsesChatClient : IChatClient
|
|
{
|
|
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
|
|
|
private readonly HttpClient httpClient;
|
|
private readonly string model;
|
|
private readonly bool enableThinking;
|
|
private readonly string reasoningEffort;
|
|
private readonly int reconnectionAttempts;
|
|
private readonly TimeSpan reconnectionDelay;
|
|
private readonly LiteLlmResponsesCompactionOptions? compactionOptions;
|
|
private readonly ILogger? logger;
|
|
private readonly Action? retrying;
|
|
private readonly Action<string>? reasoningSummaryChanged;
|
|
private int responsesRequestCount;
|
|
|
|
public LiteLlmResponsesChatClient(
|
|
Uri endpoint,
|
|
string apiKey,
|
|
string model,
|
|
bool enableThinking,
|
|
string reasoningEffort,
|
|
int reconnectionAttempts,
|
|
TimeSpan reconnectionDelay,
|
|
LiteLlmResponsesCompactionOptions? compactionOptions = null,
|
|
ILogger? logger = null,
|
|
bool firstRequestIsUser = true,
|
|
Action? retrying = null,
|
|
Action<string>? reasoningSummaryChanged = null)
|
|
: this(
|
|
new HttpClient { BaseAddress = NormalizeEndpoint(endpoint) },
|
|
apiKey,
|
|
model,
|
|
enableThinking,
|
|
reasoningEffort,
|
|
reconnectionAttempts,
|
|
reconnectionDelay,
|
|
compactionOptions,
|
|
logger,
|
|
firstRequestIsUser,
|
|
retrying,
|
|
reasoningSummaryChanged)
|
|
{
|
|
}
|
|
|
|
internal LiteLlmResponsesChatClient(
|
|
HttpClient httpClient,
|
|
string apiKey,
|
|
string model,
|
|
bool enableThinking,
|
|
string reasoningEffort,
|
|
int reconnectionAttempts,
|
|
TimeSpan reconnectionDelay,
|
|
LiteLlmResponsesCompactionOptions? compactionOptions = null,
|
|
ILogger? logger = null,
|
|
bool firstRequestIsUser = true,
|
|
Action? retrying = null,
|
|
Action<string>? reasoningSummaryChanged = null)
|
|
{
|
|
this.httpClient = httpClient;
|
|
this.httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
|
|
this.model = model;
|
|
this.enableThinking = enableThinking;
|
|
this.reasoningEffort = reasoningEffort;
|
|
this.reconnectionAttempts = Math.Max(0, reconnectionAttempts);
|
|
this.reconnectionDelay = reconnectionDelay > TimeSpan.Zero ? reconnectionDelay : TimeSpan.Zero;
|
|
this.compactionOptions = compactionOptions;
|
|
this.logger = logger;
|
|
this.retrying = retrying;
|
|
this.reasoningSummaryChanged = reasoningSummaryChanged;
|
|
responsesRequestCount = firstRequestIsUser ? 0 : 1;
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
httpClient.Dispose();
|
|
}
|
|
|
|
public async Task<ChatResponse> GetResponseAsync(
|
|
IEnumerable<ChatMessage> messages,
|
|
ChatOptions? options = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var payload = await CreateCompactedPayloadAsync(messages.ToList(), options, cancellationToken).ConfigureAwait(false);
|
|
var responseJson = await PostWithRetryAsync(payload, cancellationToken).ConfigureAwait(false);
|
|
|
|
var response = ParseResponseJson(responseJson, out var reasoningSummaries);
|
|
ReportVisibleReasoningSummaries(reasoningSummaries);
|
|
LogResponseDiagnostics(responseJson, response);
|
|
ThrowIfResponseHasNoContent(responseJson, response);
|
|
LogResponseUsage(response.Usage);
|
|
return response;
|
|
}
|
|
|
|
public async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
|
|
IEnumerable<ChatMessage> messages,
|
|
ChatOptions? options = null,
|
|
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default)
|
|
{
|
|
var response = await GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false);
|
|
foreach (var message in response.Messages)
|
|
{
|
|
yield return new ChatResponseUpdate(message.Role, message.Contents);
|
|
}
|
|
}
|
|
|
|
public object? GetService(Type serviceType, object? serviceKey = null)
|
|
{
|
|
if (serviceKey is not null)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
return serviceType == typeof(ChatClientMetadata)
|
|
? new ChatClientMetadata("litellm-responses", httpClient.BaseAddress, model)
|
|
: null;
|
|
}
|
|
|
|
public static ChatResponse ParseResponseJson(string responseJson)
|
|
{
|
|
return ParseResponseJson(responseJson, out _);
|
|
}
|
|
|
|
private static ChatResponse ParseResponseJson(
|
|
string responseJson,
|
|
out IReadOnlyList<string> reasoningSummaries)
|
|
{
|
|
using var document = JsonDocument.Parse(responseJson);
|
|
var root = document.RootElement;
|
|
var contents = new List<AIContent>();
|
|
reasoningSummaries = ExtractVisibleReasoningSummaries(root);
|
|
|
|
if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array)
|
|
{
|
|
foreach (var item in output.EnumerateArray())
|
|
{
|
|
var type = GetString(item, "type");
|
|
if (type == "message")
|
|
{
|
|
AddMessageContent(contents, item);
|
|
}
|
|
else if (type == "function_call")
|
|
{
|
|
contents.Add(new FunctionCallContent(
|
|
GetRequiredString(item, "call_id"),
|
|
GetRequiredString(item, "name"),
|
|
ParseArguments(GetString(item, "arguments"))));
|
|
}
|
|
}
|
|
}
|
|
|
|
var message = new ChatMessage
|
|
{
|
|
Role = ChatRole.Assistant,
|
|
Contents = contents
|
|
};
|
|
|
|
return new ChatResponse(message)
|
|
{
|
|
ResponseId = GetString(root, "id"),
|
|
ModelId = GetString(root, "model"),
|
|
CreatedAt = GetUnixTimestamp(root, "created_at"),
|
|
Usage = ParseUsage(root),
|
|
RawRepresentation = responseJson
|
|
};
|
|
}
|
|
|
|
internal static IReadOnlyList<string> ExtractVisibleReasoningSummaries(string responseJson)
|
|
{
|
|
using var document = JsonDocument.Parse(responseJson);
|
|
return ExtractVisibleReasoningSummaries(document.RootElement);
|
|
}
|
|
|
|
private static IReadOnlyList<string> ExtractVisibleReasoningSummaries(JsonElement root)
|
|
{
|
|
var summaries = new List<string>();
|
|
|
|
if (!root.TryGetProperty("output", out var output) || output.ValueKind != JsonValueKind.Array)
|
|
{
|
|
return summaries;
|
|
}
|
|
|
|
foreach (var item in output.EnumerateArray())
|
|
{
|
|
if (GetString(item, "type") == "reasoning")
|
|
{
|
|
AddVisibleReasoningSummaryText(summaries, item);
|
|
}
|
|
}
|
|
|
|
return summaries;
|
|
}
|
|
|
|
private void ReportVisibleReasoningSummaries(IReadOnlyList<string> summaries)
|
|
{
|
|
if (reasoningSummaryChanged is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var summary in summaries)
|
|
{
|
|
try
|
|
{
|
|
reasoningSummaryChanged(summary);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
logger?.LogWarning(
|
|
exception,
|
|
"Reasoning summary callback failed; continuing with the LiteLLM response.");
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task<JsonObject> CreateCompactedPayloadAsync(
|
|
IReadOnlyList<ChatMessage> messages,
|
|
ChatOptions? options,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var payload = CreatePayload(messages, options);
|
|
var estimate = EstimateTokens(payload);
|
|
var triggerTokens = compactionOptions?.TriggerTokens ?? int.MaxValue;
|
|
if (compactionOptions?.Enabled != true || estimate < triggerTokens)
|
|
{
|
|
LogContextWindow(estimate, compacted: false, source: "none");
|
|
return payload;
|
|
}
|
|
|
|
LogContextWindow(estimate, compacted: false, source: "threshold");
|
|
|
|
var remotePayload = await TryCompactWithResponsesEndpointAsync(payload, estimate, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
if (remotePayload is not null)
|
|
{
|
|
LogContextWindow(EstimateTokens(remotePayload), compacted: true, source: "responses_compact");
|
|
return remotePayload;
|
|
}
|
|
|
|
if (compactionOptions.FallbackStrategy is null)
|
|
{
|
|
return payload;
|
|
}
|
|
|
|
try
|
|
{
|
|
var compactedMessages = await CompactionProvider.CompactAsync(
|
|
compactionOptions.FallbackStrategy,
|
|
messages,
|
|
logger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance,
|
|
cancellationToken)
|
|
.ConfigureAwait(false);
|
|
var fallbackPayload = CreatePayload(compactedMessages, options);
|
|
LogContextWindow(EstimateTokens(fallbackPayload), compacted: true, source: "agent_framework");
|
|
return fallbackPayload;
|
|
}
|
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
|
{
|
|
logger?.LogWarning(
|
|
exception,
|
|
"Agent Framework compaction fallback failed; sending un-compacted summary request");
|
|
return payload;
|
|
}
|
|
}
|
|
|
|
private async Task<JsonObject?> TryCompactWithResponsesEndpointAsync(
|
|
JsonObject payload,
|
|
int estimatedTokens,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
var request = new JsonObject
|
|
{
|
|
["model"] = payload["model"]?.DeepClone(),
|
|
["input"] = payload["input"]?.DeepClone(),
|
|
["instructions"] = payload["instructions"]?.DeepClone(),
|
|
["context_window_tokens"] = compactionOptions!.ContextWindowTokens,
|
|
["max_output_tokens"] = compactionOptions.MaxOutputTokens,
|
|
["estimated_input_tokens"] = estimatedTokens,
|
|
["target_input_tokens"] = compactionOptions.TriggerTokens,
|
|
["strategy"] = new JsonArray
|
|
{
|
|
"collapse_tool_results",
|
|
"summarize_older_spans",
|
|
"keep_last_4_user_turns",
|
|
"drop_oldest_groups"
|
|
}
|
|
};
|
|
|
|
using var requestMessage = CreateJsonRequest(
|
|
NormalizeRelativePath(compactionOptions.CompactPath),
|
|
request.ToJsonString(JsonOptions),
|
|
"agent");
|
|
using var response = await httpClient.SendAsync(requestMessage, cancellationToken).ConfigureAwait(false);
|
|
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
logger?.LogWarning(
|
|
"Responses compaction endpoint failed with {StatusCode} {ReasonPhrase}: {Body}",
|
|
(int)response.StatusCode,
|
|
response.ReasonPhrase,
|
|
responseJson);
|
|
return null;
|
|
}
|
|
|
|
return ApplyCompactedInput(payload, responseJson);
|
|
}
|
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
|
{
|
|
logger?.LogWarning(exception, "Responses compaction endpoint failed; falling back to Agent Framework compaction");
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static JsonObject? ApplyCompactedInput(JsonObject payload, string responseJson)
|
|
{
|
|
using var document = JsonDocument.Parse(responseJson);
|
|
var root = document.RootElement;
|
|
var input = FindCompactedInput(root);
|
|
if (input is null)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var compactedPayload = (JsonObject)payload.DeepClone();
|
|
compactedPayload["input"] = JsonNode.Parse(input.Value.GetRawText());
|
|
if (root.TryGetProperty("instructions", out var instructions) && instructions.ValueKind == JsonValueKind.String)
|
|
{
|
|
compactedPayload["instructions"] = instructions.GetString();
|
|
}
|
|
|
|
return compactedPayload;
|
|
}
|
|
|
|
private static JsonElement? FindCompactedInput(JsonElement root)
|
|
{
|
|
foreach (var propertyName in new[] { "input", "compacted_input", "messages", "output" })
|
|
{
|
|
if (root.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.Array)
|
|
{
|
|
return value;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private JsonObject CreatePayload(IEnumerable<ChatMessage> messages, ChatOptions? options)
|
|
{
|
|
var instructions = new StringBuilder();
|
|
if (!string.IsNullOrWhiteSpace(options?.Instructions))
|
|
{
|
|
instructions.AppendLine(options.Instructions);
|
|
}
|
|
|
|
var input = new JsonArray();
|
|
foreach (var message in messages)
|
|
{
|
|
AddInputItem(input, instructions, message);
|
|
}
|
|
|
|
var payload = new JsonObject
|
|
{
|
|
["model"] = options?.ModelId ?? model,
|
|
["input"] = input,
|
|
["store"] = false,
|
|
["tool_choice"] = "auto"
|
|
};
|
|
|
|
if (instructions.Length > 0)
|
|
{
|
|
payload["instructions"] = instructions.ToString().Trim();
|
|
}
|
|
|
|
if (enableThinking)
|
|
{
|
|
payload["reasoning"] = new JsonObject
|
|
{
|
|
["effort"] = reasoningEffort
|
|
};
|
|
}
|
|
|
|
if (options?.MaxOutputTokens is > 0)
|
|
{
|
|
payload["max_output_tokens"] = options.MaxOutputTokens.Value;
|
|
}
|
|
|
|
var tools = CreateTools(options?.Tools);
|
|
if (tools.Count > 0)
|
|
{
|
|
payload["tools"] = tools;
|
|
payload["parallel_tool_calls"] = true;
|
|
}
|
|
|
|
return payload;
|
|
}
|
|
|
|
private async Task<string> PostWithRetryAsync(JsonObject payload, CancellationToken cancellationToken)
|
|
{
|
|
var payloadJson = payload.ToJsonString(JsonOptions);
|
|
Exception? lastException = null;
|
|
|
|
for (var attempt = 0; attempt <= reconnectionAttempts; attempt++)
|
|
{
|
|
try
|
|
{
|
|
var initiator = NextInitiator();
|
|
LogRequestDiagnostics(payload, initiator, attempt);
|
|
using var request = CreateJsonRequest(
|
|
"responses",
|
|
payloadJson,
|
|
initiator);
|
|
using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false);
|
|
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
|
|
|
if (response.IsSuccessStatusCode)
|
|
{
|
|
return responseJson;
|
|
}
|
|
|
|
var exception = new InvalidOperationException(
|
|
$"LiteLLM Responses request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}");
|
|
if (!IsRetryableStatusCode((int)response.StatusCode) || attempt == reconnectionAttempts)
|
|
{
|
|
throw exception;
|
|
}
|
|
|
|
lastException = exception;
|
|
}
|
|
catch (Exception exception) when (IsRetryableException(exception) && attempt < reconnectionAttempts)
|
|
{
|
|
lastException = exception;
|
|
}
|
|
|
|
retrying?.Invoke();
|
|
await DelayBeforeRetryAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
throw lastException ?? new InvalidOperationException("LiteLLM Responses request failed.");
|
|
}
|
|
|
|
private HttpRequestMessage CreateJsonRequest(
|
|
string requestUri,
|
|
string payloadJson,
|
|
string? initiator = null)
|
|
{
|
|
var request = new HttpRequestMessage(HttpMethod.Post, requestUri)
|
|
{
|
|
Content = new StringContent(payloadJson, Encoding.UTF8, "application/json")
|
|
};
|
|
if (!string.IsNullOrWhiteSpace(initiator))
|
|
{
|
|
request.Headers.TryAddWithoutValidation("X-Initiator", initiator);
|
|
}
|
|
|
|
return request;
|
|
}
|
|
|
|
private string NextInitiator()
|
|
{
|
|
return Interlocked.Increment(ref responsesRequestCount) == 1 ? "user" : "agent";
|
|
}
|
|
|
|
private async Task DelayBeforeRetryAsync(CancellationToken cancellationToken)
|
|
{
|
|
if (reconnectionDelay > TimeSpan.Zero)
|
|
{
|
|
await Task.Delay(reconnectionDelay, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
private static bool IsRetryableStatusCode(int statusCode)
|
|
{
|
|
return statusCode is 408 or 409 or 425 or 429
|
|
|| statusCode >= 500;
|
|
}
|
|
|
|
private static bool IsRetryableException(Exception exception)
|
|
{
|
|
return exception is HttpRequestException or TaskCanceledException;
|
|
}
|
|
|
|
private void LogContextWindow(int estimatedTokens, bool compacted, string source)
|
|
{
|
|
if (compactionOptions is null || logger is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var contextWindow = Math.Max(1, compactionOptions.ContextWindowTokens);
|
|
var remainingTokens = Math.Max(0, contextWindow - estimatedTokens);
|
|
logger.LogInformation(
|
|
"Summary context estimate: {EstimatedTokens}/{ContextWindowTokens} tokens, {RemainingTokens} remaining, compacted={Compacted}, source={Source}",
|
|
estimatedTokens,
|
|
contextWindow,
|
|
remainingTokens,
|
|
compacted,
|
|
source);
|
|
}
|
|
|
|
private void LogResponseUsage(UsageDetails? usage)
|
|
{
|
|
if (usage is null || compactionOptions is null || logger is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var contextWindow = Math.Max(1, compactionOptions.ContextWindowTokens);
|
|
var inputTokens = usage.InputTokenCount ?? 0;
|
|
var outputTokens = usage.OutputTokenCount ?? 0;
|
|
var totalTokens = usage.TotalTokenCount ?? inputTokens + outputTokens;
|
|
logger.LogInformation(
|
|
"Summary response usage: input={InputTokens}, output={OutputTokens}, total={TotalTokens}, contextWindow={ContextWindowTokens}",
|
|
inputTokens,
|
|
outputTokens,
|
|
totalTokens,
|
|
contextWindow);
|
|
}
|
|
|
|
private void LogRequestDiagnostics(JsonObject payload, string initiator, int attempt)
|
|
{
|
|
if (logger is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var inputCount = payload.TryGetPropertyValue("input", out var input) && input is JsonArray inputArray
|
|
? inputArray.Count
|
|
: 0;
|
|
var tools = payload.TryGetPropertyValue("tools", out var toolsNode) && toolsNode is JsonArray toolsArray
|
|
? toolsArray
|
|
: [];
|
|
var toolNames = tools
|
|
.OfType<JsonObject>()
|
|
.Select(tool => tool.TryGetPropertyValue("name", out var name) ? name?.GetValue<string>() : null)
|
|
.Where(name => !string.IsNullOrWhiteSpace(name))
|
|
.ToArray();
|
|
var instructionsPreview = payload.TryGetPropertyValue("instructions", out var instructionsNode)
|
|
? Truncate(instructionsNode?.GetValue<string>() ?? "", maxLength: 500)
|
|
: "";
|
|
|
|
logger.LogInformation(
|
|
"LiteLLM Responses request: initiator={Initiator}, attempt={Attempt}, inputItems={InputItems}, tools={ToolCount}, toolNames={ToolNames}, instructionsPreview={InstructionsPreview}",
|
|
initiator,
|
|
attempt + 1,
|
|
inputCount,
|
|
toolNames.Length,
|
|
string.Join(", ", toolNames),
|
|
instructionsPreview);
|
|
}
|
|
|
|
private void LogResponseDiagnostics(string responseJson, ChatResponse response)
|
|
{
|
|
if (logger is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var outputTypes = GetOutputItemTypes(responseJson);
|
|
var parsedTypes = response.Messages
|
|
.SelectMany(message => message.Contents)
|
|
.Select(content => content.GetType().Name)
|
|
.ToArray();
|
|
var functionCalls = response.Messages
|
|
.SelectMany(message => message.Contents)
|
|
.OfType<FunctionCallContent>()
|
|
.Select(call => call.Name)
|
|
.ToArray();
|
|
|
|
logger.LogInformation(
|
|
"LiteLLM Responses response: responseId={ResponseId}, outputTypes={OutputTypes}, parsedContents={ParsedContents}, functionCalls={FunctionCalls}, textLength={TextLength}",
|
|
response.ResponseId,
|
|
string.Join(", ", outputTypes),
|
|
string.Join(", ", parsedTypes),
|
|
string.Join(", ", functionCalls),
|
|
response.Text?.Length ?? 0);
|
|
|
|
if (parsedTypes.Length == 0)
|
|
{
|
|
logger.LogWarning(
|
|
"LiteLLM Responses response contained no parseable message text or function calls. Raw response preview: {ResponsePreview}",
|
|
Truncate(responseJson, maxLength: 6000));
|
|
}
|
|
}
|
|
|
|
private static void ThrowIfResponseHasNoContent(string responseJson, ChatResponse response)
|
|
{
|
|
if (response.Messages.SelectMany(message => message.Contents).Any())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var outputTypes = GetOutputItemTypes(responseJson);
|
|
throw new InvalidOperationException(
|
|
"LiteLLM Responses returned no parseable message text or function calls. " +
|
|
$"ResponseId={response.ResponseId ?? "<none>"}, outputTypes=[{string.Join(", ", outputTypes)}].");
|
|
}
|
|
|
|
private static string[] GetOutputItemTypes(string responseJson)
|
|
{
|
|
try
|
|
{
|
|
using var document = JsonDocument.Parse(responseJson);
|
|
return document.RootElement.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array
|
|
? output
|
|
.EnumerateArray()
|
|
.Select(item => GetString(item, "type") ?? item.ValueKind.ToString())
|
|
.ToArray()
|
|
: [];
|
|
}
|
|
catch (JsonException)
|
|
{
|
|
return ["<invalid-json>"];
|
|
}
|
|
}
|
|
|
|
private static string Truncate(string value, int maxLength)
|
|
{
|
|
return value.Length <= maxLength
|
|
? value
|
|
: value[..maxLength] + "...";
|
|
}
|
|
|
|
private static void AddInputItem(JsonArray input, StringBuilder instructions, ChatMessage message)
|
|
{
|
|
var role = message.Role.Value;
|
|
var text = message.Text;
|
|
if (role == ChatRole.System.Value)
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(text))
|
|
{
|
|
instructions.AppendLine(text);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(text))
|
|
{
|
|
input.Add(new JsonObject
|
|
{
|
|
["role"] = role == ChatRole.Assistant.Value ? "assistant" : "user",
|
|
["content"] = text
|
|
});
|
|
}
|
|
|
|
foreach (var content in message.Contents)
|
|
{
|
|
if (content is FunctionCallContent call)
|
|
{
|
|
input.Add(new JsonObject
|
|
{
|
|
["type"] = "function_call",
|
|
["call_id"] = call.CallId,
|
|
["name"] = call.Name,
|
|
["arguments"] = JsonSerializer.Serialize(call.Arguments, JsonOptions)
|
|
});
|
|
}
|
|
else if (content is FunctionResultContent result)
|
|
{
|
|
input.Add(new JsonObject
|
|
{
|
|
["type"] = "function_call_output",
|
|
["call_id"] = result.CallId,
|
|
["output"] = ResultToString(result.Result)
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
private static JsonArray CreateTools(IList<AITool>? tools)
|
|
{
|
|
var result = new JsonArray();
|
|
if (tools is null)
|
|
{
|
|
return result;
|
|
}
|
|
|
|
foreach (var tool in tools.OfType<AIFunctionDeclaration>())
|
|
{
|
|
result.Add(new JsonObject
|
|
{
|
|
["type"] = "function",
|
|
["name"] = tool.Name,
|
|
["description"] = tool.Description ?? string.Empty,
|
|
["parameters"] = JsonNode.Parse(tool.JsonSchema.GetRawText())
|
|
});
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
private static void AddMessageContent(List<AIContent> contents, JsonElement item)
|
|
{
|
|
if (!item.TryGetProperty("content", out var messageContent) || messageContent.ValueKind != JsonValueKind.Array)
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var content in messageContent.EnumerateArray())
|
|
{
|
|
if (GetString(content, "type") == "output_text")
|
|
{
|
|
var text = GetString(content, "text");
|
|
if (!string.IsNullOrEmpty(text))
|
|
{
|
|
contents.Add(new TextContent(text));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void AddVisibleReasoningSummaryText(List<string> summaries, JsonElement item)
|
|
{
|
|
AddVisibleReasoningSummaryText(summaries, item, "summary");
|
|
AddVisibleReasoningSummaryText(summaries, item, "content");
|
|
}
|
|
|
|
private static void AddVisibleReasoningSummaryText(
|
|
List<string> summaries,
|
|
JsonElement item,
|
|
string propertyName)
|
|
{
|
|
if (!item.TryGetProperty(propertyName, out var property))
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (property.ValueKind == JsonValueKind.String)
|
|
{
|
|
AddNonEmpty(summaries, property.GetString());
|
|
return;
|
|
}
|
|
|
|
if (property.ValueKind != JsonValueKind.Array)
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var summaryItem in property.EnumerateArray())
|
|
{
|
|
if (summaryItem.ValueKind == JsonValueKind.String)
|
|
{
|
|
AddNonEmpty(summaries, summaryItem.GetString());
|
|
}
|
|
else if (summaryItem.ValueKind == JsonValueKind.Object)
|
|
{
|
|
if (propertyName == "content" && !IsSummaryContent(summaryItem))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
AddNonEmpty(summaries, GetString(summaryItem, "text"));
|
|
}
|
|
}
|
|
}
|
|
|
|
private static bool IsSummaryContent(JsonElement item)
|
|
{
|
|
var type = GetString(item, "type");
|
|
return !string.IsNullOrWhiteSpace(type)
|
|
&& type.Contains("summary", StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
private static void AddNonEmpty(List<string> values, string? value)
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(value))
|
|
{
|
|
values.Add(value.Trim());
|
|
}
|
|
}
|
|
|
|
private static Dictionary<string, object?> ParseArguments(string? arguments)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(arguments))
|
|
{
|
|
return [];
|
|
}
|
|
|
|
using var document = JsonDocument.Parse(arguments);
|
|
return document.RootElement.EnumerateObject()
|
|
.ToDictionary(property => property.Name, property => ConvertJsonValue(property.Value));
|
|
}
|
|
|
|
private static object? ConvertJsonValue(JsonElement element)
|
|
{
|
|
return element.ValueKind switch
|
|
{
|
|
JsonValueKind.String => element.GetString(),
|
|
JsonValueKind.Number when element.TryGetInt64(out var integer) => integer,
|
|
JsonValueKind.Number when element.TryGetDouble(out var number) => number,
|
|
JsonValueKind.True => true,
|
|
JsonValueKind.False => false,
|
|
JsonValueKind.Null => null,
|
|
_ => JsonSerializer.Deserialize<object>(element.GetRawText(), JsonOptions)
|
|
};
|
|
}
|
|
|
|
private static string ResultToString(object? result)
|
|
{
|
|
return result switch
|
|
{
|
|
null => string.Empty,
|
|
string text => text,
|
|
_ => JsonSerializer.Serialize(result, JsonOptions)
|
|
};
|
|
}
|
|
|
|
private static UsageDetails? ParseUsage(JsonElement root)
|
|
{
|
|
if (!root.TryGetProperty("usage", out var usage) || usage.ValueKind != JsonValueKind.Object)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var inputTokens = GetInt(usage, "input_tokens") ?? GetInt(usage, "prompt_tokens");
|
|
var outputTokens = GetInt(usage, "output_tokens") ?? GetInt(usage, "completion_tokens");
|
|
var totalTokens = GetInt(usage, "total_tokens");
|
|
return new UsageDetails
|
|
{
|
|
InputTokenCount = inputTokens,
|
|
OutputTokenCount = outputTokens,
|
|
TotalTokenCount = totalTokens
|
|
};
|
|
}
|
|
|
|
private static int EstimateTokens(JsonObject payload)
|
|
{
|
|
var json = payload.ToJsonString(JsonOptions);
|
|
return Math.Max(1, (int)Math.Ceiling(json.Length / 4.0));
|
|
}
|
|
|
|
private static int? GetInt(JsonElement element, string propertyName)
|
|
{
|
|
return element.TryGetProperty(propertyName, out var property) && property.TryGetInt32(out var value)
|
|
? value
|
|
: null;
|
|
}
|
|
|
|
private static string GetRequiredString(JsonElement element, string propertyName)
|
|
{
|
|
return GetString(element, propertyName)
|
|
?? throw new InvalidOperationException($"LiteLLM response item did not include '{propertyName}'.");
|
|
}
|
|
|
|
private static string? GetString(JsonElement element, string propertyName)
|
|
{
|
|
return element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String
|
|
? property.GetString()
|
|
: null;
|
|
}
|
|
|
|
private static DateTimeOffset? GetUnixTimestamp(JsonElement element, string propertyName)
|
|
{
|
|
return element.TryGetProperty(propertyName, out var property) && property.TryGetInt64(out var value)
|
|
? DateTimeOffset.FromUnixTimeSeconds(value)
|
|
: null;
|
|
}
|
|
|
|
private static Uri NormalizeEndpoint(Uri endpoint)
|
|
{
|
|
var value = endpoint.ToString().TrimEnd('/');
|
|
if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
value += "/v1";
|
|
}
|
|
|
|
return new Uri(value + "/");
|
|
}
|
|
|
|
private static string NormalizeRelativePath(string value)
|
|
{
|
|
return value.TrimStart('/');
|
|
}
|
|
}
|
|
#pragma warning restore MAAI001
|