Manuel/meeting-assistant
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity Open in Coder

Merge remote-tracking branch 'origin/main' into renovate/microsoftspeechversion

Browse Source
This commit is contained in:
codex
2026-08-05 13:00:56 +02:00
parent ada4a40f62 8dc1b70f8a
commit a087ff0cbf
29 changed files with 1315 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files
MeetingAssistant/Calendar/CalendarRecordingPromptScheduler.cs
+31 -3
View File
@@ -98,7 +98,9 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
meeting.Subject,
meeting.Start);
await promptService.ShowPromptAsync(
new MeetingStartPromptRequest(meeting),
new MeetingStartPromptRequest(
meeting,
recordingController.CurrentStatus.IsRecording && meeting.Metadata is not null),
(response, token) => HandlePromptResponseAsync(meeting, response, token),
cancellationToken);
}
@@ -139,6 +141,18 @@ public sealed class CalendarRecordingPromptScheduler : BackgroundService
MeetingStartPromptResponse response,
CancellationToken cancellationToken)
{
if (response == MeetingStartPromptResponse.AttachMetadataToCurrentMeeting)
{
if (meeting.Metadata is not null)
{
await recordingController.AttachMetadataToCurrentMeetingAsync(
meeting.Metadata,
cancellationToken);
}
return;
}
if (response != MeetingStartPromptResponse.Record)
{
return;
@@ -244,12 +258,15 @@ public interface IMeetingStartPromptService
CancellationToken cancellationToken);
}
public sealed record MeetingStartPromptRequest(CalendarMeeting Meeting);
public sealed record MeetingStartPromptRequest(
CalendarMeeting Meeting,
bool CanAttachToCurrentMeeting = false);
public enum MeetingStartPromptResponse
{
Record,
Skip
Skip,
AttachMetadataToCurrentMeeting
}
public interface IMeetingPromptRecordingController
@@ -262,6 +279,10 @@ public interface IMeetingPromptRecordingController
MeetingMetadata? metadata,
CancellationToken cancellationToken);
Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
MeetingMetadata metadata,
CancellationToken cancellationToken);
Task<RecordingStatus> StopAsync(CancellationToken cancellationToken);
}
@@ -288,6 +309,13 @@ public sealed class MeetingPromptRecordingController : IMeetingPromptRecordingCo
return coordinator.StartFromPromptAsync(metadata, cancellationToken);
}
public Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
MeetingMetadata metadata,
CancellationToken cancellationToken)
{
return coordinator.AttachMetadataToCurrentMeetingAsync(metadata, cancellationToken);
}
public Task<RecordingStatus> StopAsync(CancellationToken cancellationToken)
{
return coordinator.StopAsync(cancellationToken);
MeetingAssistant/Calendar/WindowsMeetingStartPromptService.Windows.cs
+35 -18
View File
@@ -118,21 +118,10 @@ public sealed class WindowsMeetingStartPromptService : IMeetingStartPromptServic
string promptId,
MeetingStartPromptRequest request)
{
var yesButton = new ToastButton()
.SetContent("Yes")
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", "record")
.SetBackgroundActivation();
var yesButton = BuildResponseButton("Yes", promptId, "record");
var noButton = BuildResponseButton("No", promptId, "skip");
var noButton = new ToastButton()
.SetContent("No")
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", "skip")
.SetBackgroundActivation();
return new ToastContentBuilder()
var notification = new ToastContentBuilder()
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.SetToastScenario(ToastScenario.Reminder)
@@ -141,6 +130,30 @@ public sealed class WindowsMeetingStartPromptService : IMeetingStartPromptServic
.AddText($"{request.Meeting.Subject} starts at {request.Meeting.Start.LocalDateTime:t}.")
.AddButton(yesButton)
.AddButton(noButton);
if (request.CanAttachToCurrentMeeting)
{
notification.AddButton(
BuildResponseButton(
"Add metadata to current meeting",
promptId,
"attach-metadata"));
}
return notification;
}
private static ToastButton BuildResponseButton(
string content,
string promptId,
string response)
{
return new ToastButton()
.SetContent(content)
.AddArgument("source", NotificationSource)
.AddArgument("promptId", promptId)
.AddArgument("response", response)
.SetBackgroundActivation();
}
private void OnNotificationInvoked(ToastNotificationActivatedEventArgsCompat args)
@@ -166,10 +179,14 @@ public sealed class WindowsMeetingStartPromptService : IMeetingStartPromptServic
return;
}
var response = TryGetArgument(arguments, "response", out var responseValue) &&
string.Equals(responseValue, "record", StringComparison.OrdinalIgnoreCase)
? MeetingStartPromptResponse.Record
: MeetingStartPromptResponse.Skip;
var response = TryGetArgument(arguments, "response", out var responseValue)
? responseValue.ToLowerInvariant() switch
{
"record" => MeetingStartPromptResponse.Record,
"attach-metadata" => MeetingStartPromptResponse.AttachMetadataToCurrentMeeting,
_ => MeetingStartPromptResponse.Skip
}
: MeetingStartPromptResponse.Skip;
_ = Task.Run(async () =>
{
try
MeetingAssistant/Recording/MeetingRecordingCoordinator.cs
+80 -23
View File
@@ -155,6 +155,44 @@ public sealed class MeetingRecordingCoordinator
return await StartAsync(null, metadata, suppressMetadataLookup: true, cancellationToken);
}
public async Task<RecordingStatus> AttachMetadataToCurrentMeetingAsync(
MeetingMetadata metadata,
CancellationToken cancellationToken)
{
await gate.WaitAsync(cancellationToken);
try
{
var run = currentRun;
if (run is null || run.IsCaptureStopping)
{
return CurrentStatus;
}
var meetingNote = await ApplyAndPersistMeetingMetadataAsync(
run.Artifacts,
await meetingNoteStore.ReadAsync(run.MeetingNotePath, cancellationToken),
metadata,
run.Options,
cancellationToken);
currentMeetingNote = meetingNote;
await transcriptStore.UpdateMetadataAsync(
run.Session,
run.Artifacts,
meetingNote,
cancellationToken);
run.MarkPromptMetadataAttached();
logger.LogInformation(
"Attached prompted calendar metadata to active meeting {MeetingNotePath}",
run.MeetingNotePath);
return CurrentStatus;
}
finally
{
gate.Release();
}
}
public async Task<RecordingStatus> StartAsync(
string? launchProfileName,
CancellationToken cancellationToken)
@@ -213,22 +251,12 @@ public sealed class MeetingRecordingCoordinator
currentMeetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
if (suppliedMetadata is not null)
{
await ApplyMeetingMetadataAsync(
currentMeetingNote = await ApplyAndPersistMeetingMetadataAsync(
currentArtifacts,
currentMeetingNote,
suppliedMetadata,
runOptions,
cancellationToken);
currentMeetingNote = await meetingNoteStore.SaveAsync(
currentMeetingNote,
runOptions,
cancellationToken);
await meetingArtifactStore.UpdateAssistantContextMetadataAsync(
currentArtifacts,
currentMeetingNote,
suppliedMetadata.Agenda,
suppliedMetadata.ScheduledEnd,
cancellationToken);
}
await meetingArtifactStore.UpdateAssistantContextMeetingAsync(
@@ -1021,7 +1049,7 @@ public sealed class MeetingRecordingCoordinator
return;
}
if (run.IsAborted)
if (run.IsAborted || run.HasAttachedPromptMetadata)
{
return;
}
@@ -1029,25 +1057,21 @@ public sealed class MeetingRecordingCoordinator
await gate.WaitAsync(CancellationToken.None);
try
{
if (run.IsAborted)
if (run.IsAborted || run.HasAttachedPromptMetadata)
{
return;
}
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, CancellationToken.None);
await ApplyMeetingMetadataAsync(run.Artifacts, meetingNote, metadata, run.Options, CancellationToken.None);
meetingNote = await meetingNoteStore.SaveAsync(meetingNote, run.Options, CancellationToken.None);
var meetingNote = await ApplyAndPersistMeetingMetadataAsync(
run.Artifacts,
await meetingNoteStore.ReadAsync(run.MeetingNotePath, CancellationToken.None),
metadata,
run.Options,
CancellationToken.None);
if (currentMeetingNote?.Path == meetingNote.Path)
{
currentMeetingNote = meetingNote;
}
await meetingArtifactStore.UpdateAssistantContextMetadataAsync(
run.Artifacts,
meetingNote,
metadata.Agenda,
metadata.ScheduledEnd,
CancellationToken.None);
logger.LogInformation(
"Applied Outlook meeting metadata to {MeetingNotePath}",
run.MeetingNotePath);
@@ -1119,6 +1143,32 @@ public sealed class MeetingRecordingCoordinator
}
}
private async Task<MeetingNote> ApplyAndPersistMeetingMetadataAsync(
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote,
MeetingMetadata metadata,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
await ApplyMeetingMetadataAsync(
artifacts,
meetingNote,
metadata,
options,
cancellationToken);
var savedMeetingNote = await meetingNoteStore.SaveAsync(
meetingNote,
options,
cancellationToken);
await meetingArtifactStore.UpdateAssistantContextMetadataAsync(
artifacts,
savedMeetingNote,
metadata.Agenda,
metadata.ScheduledEnd,
cancellationToken);
return savedMeetingNote;
}
private async Task<List<string>> TransformAttendeesAsync(
MeetingSessionArtifacts artifacts,
IReadOnlyList<string> attendees,
@@ -1908,6 +1958,8 @@ public sealed class MeetingRecordingCoordinator
public bool HasSwitchedProfile { get; private set; }
public bool HasAttachedPromptMetadata { get; private set; }
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
public DateTimeOffset? InferredEndTime
@@ -1926,6 +1978,11 @@ public sealed class MeetingRecordingCoordinator
CaptureCancellationSource.Cancel();
}
public void MarkPromptMetadataAttached()
{
HasAttachedPromptMetadata = true;
}
public void Abort()
{
IsAborted = true;
MeetingAssistant/Screenshots/LiteLlmScreenshotOcrClient.cs
+26 -98
View File
@@ -1,15 +1,13 @@
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.RegularExpressions;
using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Summary;
using Microsoft.Extensions.AI;
namespace MeetingAssistant.Screenshots;
public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
{
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
private readonly ILogger<LiteLlmScreenshotOcrClient> logger;
private readonly Func<HttpMessageHandler>? httpMessageHandlerFactory;
@@ -40,20 +38,30 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
: options.Agent.Model;
var key = ResolveApiKey(options);
var imageBytes = await File.ReadAllBytesAsync(screenshotPath, cancellationToken);
using var httpClient = CreateHttpClient();
httpClient.BaseAddress = NormalizeEndpoint(new Uri(endpoint));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", key);
var payload = CreatePayload(model, CreatePrompt(prompt, imageBytes), imageBytes);
using var content = new StringContent(payload.ToJsonString(JsonOptions), Encoding.UTF8, "application/json");
using var response = await httpClient.PostAsync("responses", content, cancellationToken);
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken);
if (!response.IsSuccessStatusCode)
{
throw new InvalidOperationException(
$"Screenshot OCR request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}");
}
var text = ParseOutputText(responseJson);
var httpClient = CreateHttpClient();
httpClient.BaseAddress = LiteLlmResponsesChatClient.NormalizeEndpoint(new Uri(endpoint));
using var chatClient = new LiteLlmResponsesChatClient(
httpClient,
key,
model,
enableThinking: false,
reasoningEffort: "none",
reconnectionAttempts: options.Agent.ReconnectionAttempts,
reconnectionDelay: options.Agent.ReconnectionDelay,
logger: logger,
firstRequestIsUser: false,
useStreaming: options.Agent.UseStreaming);
var response = await chatClient.GetResponseAsync(
[
new ChatMessage(
ChatRole.User,
[
new TextContent(CreatePrompt(prompt, imageBytes)),
new DataContent(imageBytes, "image/png")
])
],
cancellationToken: cancellationToken);
var text = response.Text ?? string.Empty;
logger.LogInformation("Screenshot OCR completed for {ScreenshotPath}", screenshotPath);
return ParseOcrResult(text);
}
@@ -65,35 +73,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
: new HttpClient(httpMessageHandlerFactory());
}
private static JsonObject CreatePayload(string model, string prompt, byte[] imageBytes)
{
return new JsonObject
{
["model"] = model,
["store"] = false,
["input"] = new JsonArray
{
new JsonObject
{
["role"] = "user",
["content"] = new JsonArray
{
new JsonObject
{
["type"] = "input_text",
["text"] = prompt
},
new JsonObject
{
["type"] = "input_image",
["image_url"] = "data:image/png;base64," + Convert.ToBase64String(imageBytes)
}
}
}
}
};
}
private static string CreatePrompt(string prompt, byte[] imageBytes)
{
return TryReadPngDimensions(imageBytes, out var width, out var height)
@@ -210,46 +189,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
bytes[offset + 3];
}
private static string ParseOutputText(string responseJson)
{
using var document = JsonDocument.Parse(responseJson);
var root = document.RootElement;
var parts = new List<string>();
if (root.TryGetProperty("output_text", out var outputText) &&
outputText.ValueKind == JsonValueKind.String &&
!string.IsNullOrWhiteSpace(outputText.GetString()))
{
parts.Add(outputText.GetString()!);
}
if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array)
{
foreach (var item in output.EnumerateArray())
{
if (!item.TryGetProperty("content", out var content) || content.ValueKind != JsonValueKind.Array)
{
continue;
}
foreach (var block in content.EnumerateArray())
{
if (block.TryGetProperty("type", out var type) &&
type.GetString() == "output_text" &&
block.TryGetProperty("text", out var text) &&
text.ValueKind == JsonValueKind.String &&
!string.IsNullOrWhiteSpace(text.GetString()))
{
parts.Add(text.GetString()!);
}
}
}
}
return parts.Count == 0
? ""
: string.Join(Environment.NewLine + Environment.NewLine, parts);
}
private static string ResolveApiKey(MeetingAssistantOptions options)
{
if (!string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Key))
@@ -278,17 +217,6 @@ public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
$"No screenshot OCR API key configured. Set MeetingAssistant:Screenshots:Ocr:Key or environment variable '{options.Screenshots.Ocr.KeyEnv}'.");
}
private static Uri NormalizeEndpoint(Uri endpoint)
{
var value = endpoint.ToString().TrimEnd('/');
if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase))
{
value += "/v1";
}
return new Uri(value + "/");
}
[GeneratedRegex("```json\\s*(?<json>.*?)\\s*```", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
private static partial Regex JsonCodeBlockRegex();
}
MeetingAssistant/Summary/LiteLlmResponsesChatClient.cs
+28 -12
View File
@@ -659,9 +659,9 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
private static void AddInputItem(JsonArray input, StringBuilder instructions, ChatMessage message)
{
var role = message.Role.Value;
var text = message.Text;
if (role == ChatRole.System.Value)
{
var text = message.Text;
if (!string.IsNullOrWhiteSpace(text))
{
instructions.AppendLine(text);
@@ -670,21 +670,37 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
return;
}
if (!string.IsNullOrWhiteSpace(text))
var isAssistant = role == ChatRole.Assistant.Value;
var messageContent = new JsonArray();
foreach (var content in message.Contents)
{
if (content is TextContent textContent && !string.IsNullOrWhiteSpace(textContent.Text))
{
messageContent.Add(new JsonObject
{
["type"] = isAssistant ? "output_text" : "input_text",
["text"] = textContent.Text
});
}
else if (!isAssistant &&
content is DataContent dataContent &&
dataContent.HasTopLevelMediaType("image"))
{
messageContent.Add(new JsonObject
{
["type"] = "input_image",
["image_url"] = dataContent.Uri.ToString()
});
}
}
if (messageContent.Count > 0)
{
var isAssistant = role == ChatRole.Assistant.Value;
input.Add(new JsonObject
{
["type"] = "message",
["role"] = isAssistant ? "assistant" : "user",
["content"] = new JsonArray
{
new JsonObject
{
["type"] = isAssistant ? "output_text" : "input_text",
["text"] = text
}
}
["content"] = messageContent
});
}
@@ -761,7 +777,7 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
return Math.Max(1, (int)Math.Ceiling(json.Length / 4.0));
}
private static Uri NormalizeEndpoint(Uri endpoint)
internal static Uri NormalizeEndpoint(Uri endpoint)
{
var value = endpoint.ToString().TrimEnd('/');
if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase))
MeetingAssistant/Taskbar/MeetingTaskbarMenu.cs
+31 -9
View File
@@ -26,7 +26,8 @@ public sealed record MeetingTaskbarMenuItem(
string? ProfileName = null,
string? MicrophoneDeviceId = null,
bool IsChecked = false,
IReadOnlyList<MeetingTaskbarMenuItem>? Items = null);
IReadOnlyList<MeetingTaskbarMenuItem>? Items = null,
bool StartsSection = false);
public static class MeetingTaskbarMenuBuilder
{
@@ -41,23 +42,29 @@ public static class MeetingTaskbarMenuBuilder
new("Open agent", MeetingTaskbarAction.EditRules)
};
if (status.IsRecording)
{
items.Add(new MeetingTaskbarMenuItem(
"Finish meeting",
MeetingTaskbarAction.StopRecording,
StartsSection: true));
}
var secondaryControls = new List<MeetingTaskbarMenuItem>();
if (microphones is { Count: > 0 })
{
items.Add(BuildMicrophoneMenu(microphones, currentMicrophoneDeviceId));
secondaryControls.Add(BuildMicrophoneMenu(microphones, currentMicrophoneDeviceId));
}
if (status.IsRecording)
{
items.Add(new MeetingTaskbarMenuItem(
"Stop meeting recording and transcribe",
MeetingTaskbarAction.StopRecording));
items.Add(new MeetingTaskbarMenuItem(
secondaryControls.Add(new MeetingTaskbarMenuItem(
"Cancel meeting recording and discard",
MeetingTaskbarAction.AbortRecording));
foreach (var profile in launchProfiles.Where(profile => !IsActiveProfile(profile, status)))
{
items.Add(new MeetingTaskbarMenuItem(
secondaryControls.Add(new MeetingTaskbarMenuItem(
AppendHotkey($"Switch to {profile.Name}", profile.Options.Hotkey.Toggle),
MeetingTaskbarAction.SwitchProfile,
profile.Name));
@@ -67,16 +74,18 @@ public static class MeetingTaskbarMenuBuilder
{
foreach (var profile in launchProfiles)
{
items.Add(new MeetingTaskbarMenuItem(
secondaryControls.Add(new MeetingTaskbarMenuItem(
AppendHotkey($"Start meeting recording ({profile.Name})", profile.Options.Hotkey.Toggle),
MeetingTaskbarAction.StartRecording,
profile.Name));
}
}
AddSection(items, secondaryControls);
items.Add(new MeetingTaskbarMenuItem(
"Exit",
MeetingTaskbarAction.Exit));
MeetingTaskbarAction.Exit,
StartsSection: true));
return new MeetingTaskbarMenu(
status.State,
@@ -102,6 +111,19 @@ public static class MeetingTaskbarMenuBuilder
Items: microphoneItems);
}
private static void AddSection(
List<MeetingTaskbarMenuItem> items,
IReadOnlyList<MeetingTaskbarMenuItem> section)
{
if (section.Count == 0)
{
return;
}
items.Add(section[0] with { StartsSection = true });
items.AddRange(section.Skip(1));
}
private static string BuildTooltip(RecordingStatus status)
{
return status.State switch
MeetingAssistant/Taskbar/UnoTaskbarIconService.Windows.cs
+3 -5
View File
@@ -196,14 +196,12 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
var popupMenu = new PopupMenu();
for (var index = 0; index < menu.Items.Count; index++)
{
if (index == 1 ||
(menu.Items[index].Action == MeetingTaskbarAction.Exit &&
menu.Items[index - 1].Action != MeetingTaskbarAction.EditRules))
var menuItem = menu.Items[index];
if (index > 0 && menuItem.StartsSection)
{
popupMenu.Items.Add(new PopupMenuSeparator());
}
var menuItem = menu.Items[index];
popupMenu.Items.Add(BuildPopupItem(menuItem));
}
@@ -290,7 +288,7 @@ public sealed class UnoTaskbarIconService : IHostedService, IDisposable
return string.Join(
"|",
FlattenMenuItems(menu.Items).Select(item =>
$"{item.Action}:{item.ProfileName}:{item.MicrophoneDeviceId}:{item.IsChecked}:{item.Text}"));
$"{item.Action}:{item.ProfileName}:{item.MicrophoneDeviceId}:{item.IsChecked}:{item.StartsSection}:{item.Text}"));
}
private static IEnumerable<MeetingTaskbarMenuItem> FlattenMenuItems(