4 Commits
Author SHA1 Message Date
renovate-bot 18bed0532a Update dependency NCalcSync to 6.3.3
PR and Push Build/Test / build-and-test (push) Successful in 9m54s
PR and Push Build/Test / build-and-test (pull_request) Successful in 10m1s
2026-07-02 02:32:16 +00:00
codex 8e2f266e66 Close stale transcript workflow follow-up
PR and Push Build/Test / build-and-test (push) Successful in 9m35s
2026-07-01 11:44:07 +02:00
codex 5c67738939 Add attendee transformation workflows 2026-07-01 11:10:36 +02:00
codex 92e359646b Update meeting summary agent UI 2026-07-01 10:30:28 +02:00
36 changed files with 1401 additions and 200 deletions
@@ -42,6 +42,99 @@ public sealed class LiteLlmResponsesChatClientTests
Assert.Equal("gpt-5.5-2026-04-23", response.ModelId); Assert.Equal("gpt-5.5-2026-04-23", response.ModelId);
} }
[Fact]
public async Task ClientReportsVisibleReasoningSummariesSeparatelyFromResponseText()
{
var handler = new SequencedHttpMessageHandler(
new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("""
{
"output": [
{
"type": "reasoning",
"summary": [
{
"type": "summary_text",
"text": "Checked the configured rules file."
},
{
"type": "summary_text",
"text": "Prepared a targeted update."
}
]
},
{
"type": "message",
"content": [
{
"type": "output_text",
"text": "Done."
}
]
}
]
}
""")
});
var reasoningSummaries = new List<string>();
using var client = CreateClient(
handler,
reconnectionAttempts: 0,
reasoningSummaryChanged: reasoningSummaries.Add);
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "update rules")]);
Assert.Equal("Done.", response.Text);
Assert.Equal(
[
"Checked the configured rules file.",
"Prepared a targeted update."
],
reasoningSummaries);
}
[Fact]
public async Task ClientDoesNotFailResponseWhenReasoningSummaryCallbackFails()
{
var handler = new SequencedHttpMessageHandler(
new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("""
{
"output": [
{
"type": "reasoning",
"summary": [
{
"type": "summary_text",
"text": "Checked the configured rules file."
}
]
},
{
"type": "message",
"content": [
{
"type": "output_text",
"text": "Done."
}
]
}
]
}
""")
});
using var client = CreateClient(
handler,
reconnectionAttempts: 0,
reasoningSummaryChanged: _ => throw new InvalidOperationException("UI failed"));
var response = await client.GetResponseAsync([new ChatMessage(ChatRole.User, "update rules")]);
Assert.Equal("Done.", response.Text);
}
[Fact] [Fact]
public void ParserReadsFunctionCalls() public void ParserReadsFunctionCalls()
{ {
@@ -306,7 +399,8 @@ public sealed class LiteLlmResponsesChatClientTests
HttpMessageHandler handler, HttpMessageHandler handler,
int reconnectionAttempts, int reconnectionAttempts,
LiteLlmResponsesCompactionOptions? compactionOptions = null, LiteLlmResponsesCompactionOptions? compactionOptions = null,
Action? retrying = null) Action? retrying = null,
Action<string>? reasoningSummaryChanged = null)
{ {
return new LiteLlmResponsesChatClient( return new LiteLlmResponsesChatClient(
new HttpClient(handler) new HttpClient(handler)
@@ -320,7 +414,8 @@ public sealed class LiteLlmResponsesChatClientTests
reconnectionAttempts, reconnectionAttempts,
TimeSpan.Zero, TimeSpan.Zero,
compactionOptions, compactionOptions,
retrying: retrying); retrying: retrying,
reasoningSummaryChanged: reasoningSummaryChanged);
} }
private sealed class SequencedHttpMessageHandler : HttpMessageHandler private sealed class SequencedHttpMessageHandler : HttpMessageHandler
@@ -1,6 +1,7 @@
using MeetingAssistant.MeetingNotes; using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Screenshots; using MeetingAssistant.Screenshots;
using MeetingAssistant.Speakers; using MeetingAssistant.Speakers;
using MeetingAssistant.Workflow;
using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using System.Drawing; using System.Drawing;
@@ -167,6 +168,35 @@ public sealed class MeetingScreenshotServiceTests
request.SequenceEqual(["Ada Lovelace", "Ada L.", "Grace Hopper", "Ada Lovelace"])); request.SequenceEqual(["Ada Lovelace", "Ada L.", "Grace Hopper", "Ada Lovelace"]));
} }
[Fact]
public async Task CaptureTransformsOcrAttendeesBeforeWritingMeetingNote()
{
var fixture = await ScreenshotFixture.CreateAsync(
options =>
{
options.Screenshots.Ocr.Enabled = true;
});
var workflowEngine = new TransformingAttendeeWorkflowEngine("Ada Lovelace (Contoso)", "Ada Lovelace");
var service = fixture.CreateService(
new FixedScreenshotCapture([1, 2, 3]),
new CapturingScreenshotOcrClient(
"Visible participant tile: Ada.",
attendees: ["Ada Lovelace (Contoso)"]),
meetingWorkflowEngine: workflowEngine);
await service.CaptureAsync(
fixture.Artifacts,
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
fixture.Options,
CancellationToken.None);
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
var meeting = await fixture.NoteStore.ReadAsync(fixture.Artifacts.MeetingNotePath, CancellationToken.None);
Assert.Equal(["Ada Lovelace"], meeting.Frontmatter.Attendees);
Assert.Equal(["Ada Lovelace (Contoso)"], workflowEngine.AttendeeRequests);
}
[Fact] [Fact]
public async Task CaptureWritesRetryLinkWhenOcrFails() public async Task CaptureWritesRetryLinkWhenOcrFails()
{ {
@@ -423,7 +453,8 @@ public sealed class MeetingScreenshotServiceTests
public MeetingScreenshotService CreateService( public MeetingScreenshotService CreateService(
IActiveWindowScreenshotCapture capture, IActiveWindowScreenshotCapture capture,
IScreenshotOcrClient ocrClient, IScreenshotOcrClient ocrClient,
ISpeakerIdentityAttendeeCanonicalizer? attendeeCanonicalizer = null) ISpeakerIdentityAttendeeCanonicalizer? attendeeCanonicalizer = null,
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
{ {
return new MeetingScreenshotService( return new MeetingScreenshotService(
capture, capture,
@@ -431,7 +462,8 @@ public sealed class MeetingScreenshotServiceTests
NoteStore, NoteStore,
attendeeCanonicalizer ?? PassthroughSpeakerIdentityAttendeeCanonicalizer.Instance, attendeeCanonicalizer ?? PassthroughSpeakerIdentityAttendeeCanonicalizer.Instance,
ocrClient, ocrClient,
NullLogger<MeetingScreenshotService>.Instance); NullLogger<MeetingScreenshotService>.Instance,
meetingWorkflowEngine);
} }
} }
@@ -293,6 +293,44 @@ public sealed class MeetingSummaryToolTests
Assert.Contains("transcript: \"[[../Transcripts/transcript|Transcript]]\"", meetingNote); Assert.Contains("transcript: \"[[../Transcripts/transcript|Transcript]]\"", meetingNote);
} }
[Fact]
public async Task ToolsTransformAddedAttendeeBeforeWritingMeetingNote()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var artifacts = new MeetingSessionArtifacts(
MeetingNotePath: Path.Combine(root, "Meetings", "Notes", "meeting.md"),
TranscriptPath: Path.Combine(root, "Meetings", "Transcripts", "transcript.md"),
AssistantContextPath: Path.Combine(root, "Meetings", "Assistant Context", "context.md"),
SummaryPath: Path.Combine(root, "Meetings", "Summaries", "summary.md"));
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
await File.WriteAllTextAsync(
artifacts.MeetingNotePath,
"""
---
title: Meeting
attendees: []
projects: []
transcript: "[[../Transcripts/transcript|Transcript]]"
assistant_context: "[[../Assistant Context/context|Assistant Context]]"
summary: "[[../Summaries/summary|Summary]]"
---
User note line.
""");
var workflowEngine = new TransformingAttendeeWorkflowEngine("Ada Lovelace (Contoso)", "Ada Lovelace");
var tools = new MeetingSummaryTools(
artifacts,
new MeetingAssistantOptions(),
meetingWorkflowEngine: workflowEngine);
Assert.Equal("Added attendee Ada Lovelace.", await tools.AddAttendee("Ada Lovelace (Contoso)"));
var meetingNote = await File.ReadAllTextAsync(artifacts.MeetingNotePath);
Assert.Contains("- Ada Lovelace", meetingNote);
Assert.DoesNotContain("Ada Lovelace (Contoso)", meetingNote);
Assert.Equal(["Ada Lovelace (Contoso)"], workflowEngine.AttendeeRequests);
}
[Fact] [Fact]
public async Task ToolsOverrideSpeakerInTranscriptAndRecordOverride() public async Task ToolsOverrideSpeakerInTranscriptAndRecordOverride()
{ {
@@ -799,4 +837,5 @@ public sealed class MeetingSummaryToolTests
Assert.False(tools.SummaryWasWritten); Assert.False(tools.SummaryWasWritten);
Assert.False(File.Exists(artifacts.SummaryPath)); Assert.False(File.Exists(artifacts.SummaryPath));
} }
} }
@@ -310,6 +310,10 @@ public sealed class MeetingWorkflowEngineTests
[InlineData("speaker_identified:\n name: ADA", "speaker", null, null, "ada", true)] [InlineData("speaker_identified:\n name: ADA", "speaker", null, null, "ada", true)]
[InlineData("speaker_identified:\n name: Ada", "speaker", null, null, "Grace", false)] [InlineData("speaker_identified:\n name: Ada", "speaker", null, null, "Grace", false)]
[InlineData("speaker_identified: {}", "speaker", null, null, "Grace", true)] [InlineData("speaker_identified: {}", "speaker", null, null, "Grace", true)]
[InlineData("attendee_added:\n equals: Ada Lovelace", "attendee", null, null, "ada lovelace", true)]
[InlineData("attendee_added:\n contains: contoso", "attendee", null, null, "Ada (Contoso)", true)]
[InlineData("attendee_added:\n regex: '^Ada .+Contoso\\)$'", "attendee", null, null, "Ada (Contoso)", true)]
[InlineData("attendee_added:\n regex: '^Grace'", "attendee", null, null, "Ada (Contoso)", false)]
public async Task TriggerMatchingScenarios( public async Task TriggerMatchingScenarios(
string triggerYaml, string triggerYaml,
string eventKind, string eventKind,
@@ -318,6 +322,16 @@ public sealed class MeetingWorkflowEngineTests
string? speaker, string? speaker,
bool shouldRun) bool shouldRun)
{ {
var stepYaml = eventKind == "attendee"
? """
- uses: set_property
property: attendee.name
value: Triggered
"""
: """
- uses: add_project
value: Triggered
""";
var fixture = await WorkflowFixture.CreateAsync( var fixture = await WorkflowFixture.CreateAsync(
$$""" $$"""
rules: rules:
@@ -325,14 +339,21 @@ public sealed class MeetingWorkflowEngineTests
on: on:
- {{triggerYaml}} - {{triggerYaml}}
steps: steps:
- uses: add_project {{stepYaml}}
value: Triggered
"""); """);
await fixture.Engine.RunAsync( var workflowEvent = CreateEvent(eventKind, fixture.Artifacts, from, to, speaker);
CreateEvent(eventKind, fixture.Artifacts, from, to, speaker), if (eventKind == "attendee")
{
var transformed = await fixture.Engine.TransformAttendeeAsync(
workflowEvent,
fixture.Options, fixture.Options,
CancellationToken.None); CancellationToken.None);
Assert.Equal(shouldRun ? "Triggered" : speaker, transformed);
return;
}
await fixture.Engine.RunAsync(workflowEvent, fixture.Options, CancellationToken.None);
var meeting = await fixture.ReadMeetingAsync(); var meeting = await fixture.ReadMeetingAsync();
Assert.Equal(shouldRun, meeting.Frontmatter.Projects.Contains("Triggered")); Assert.Equal(shouldRun, meeting.Frontmatter.Projects.Contains("Triggered"));
@@ -603,6 +624,34 @@ public sealed class MeetingWorkflowEngineTests
Assert.Equal(["Ada Lovelace"], meeting.Frontmatter.Attendees); Assert.Equal(["Ada Lovelace"], meeting.Frontmatter.Attendees);
} }
[Fact]
public async Task AttendeeAddedRuleCanTransformWorkflowAddedAttendee()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: add-raw-attendee
on:
- created: {}
steps:
- uses: add_attendee
value: 'Ada Lovelace (Contoso)'
- name: clean-contoso-attendee
on:
- attendee_added:
contains: 'Contoso'
steps:
- uses: set_property
property: attendee.name
value: '@Model.Attendee.Name.Replace(" (Contoso)", "")'
""");
await RunCreatedAsync(fixture);
var meeting = await fixture.ReadMeetingAsync();
Assert.Equal(["Ada Lovelace"], meeting.Frontmatter.Attendees);
}
[Fact] [Fact]
public async Task AddAttendeeDoesNotDuplicateExistingDisplayNameFromEmailAddress() public async Task AddAttendeeDoesNotDuplicateExistingDisplayNameFromEmailAddress()
{ {
@@ -792,6 +841,7 @@ public sealed class MeetingWorkflowEngineTests
{ {
"created" => MeetingWorkflowEvent.Created(artifacts), "created" => MeetingWorkflowEvent.Created(artifacts),
"speaker" => MeetingWorkflowEvent.SpeakerIdentified(artifacts, speaker ?? "Ada"), "speaker" => MeetingWorkflowEvent.SpeakerIdentified(artifacts, speaker ?? "Ada"),
"attendee" => MeetingWorkflowEvent.AttendeeAdded(artifacts, speaker ?? "Ada"),
"state" => MeetingWorkflowEvent.StateTransition( "state" => MeetingWorkflowEvent.StateTransition(
artifacts, artifacts,
ParseState(from ?? "collecting metadata"), ParseState(from ?? "collecting metadata"),
@@ -117,7 +117,7 @@ public sealed class RecordingCoordinatorTests
} }
[Fact] [Fact]
public async Task TranscriptLineWorkflowRuleTransformsLiveTranscriptBeforePersistingMarkdown() public async Task TranscriptLineWorkflowRuleTransformsLiveTranscriptAfterDurableAppend()
{ {
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N")); var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var rulesPath = Path.Combine(root, "rules.yaml"); var rulesPath = Path.Combine(root, "rules.yaml");
@@ -900,6 +900,39 @@ public sealed class RecordingCoordinatorTests
await coordinator.StopAsync(CancellationToken.None); await coordinator.StopAsync(CancellationToken.None);
} }
[Fact]
public async Task StartTransformsMetadataAttendeesBeforeWritingNote()
{
var audioSource = new ControlledAudioSource();
var noteStore = new InMemoryMeetingNoteStore("C:\\Vault\\Meetings\\Notes\\metadata-meeting.md");
var workflowEngine = new TransformingAttendeeWorkflowEngine("Ada Lovelace (Contoso)", "Ada Lovelace");
var coordinator = new MeetingRecordingCoordinator(
audioSource,
new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()),
new InMemoryTranscriptStore(),
noteStore,
new CapturingMeetingNoteOpener(),
new InMemoryMeetingArtifactStore(),
new InMemoryRecordedAudioStore(),
new CapturingMeetingSummaryPipeline(),
Options.Create(new MeetingAssistantOptions()),
NullLogger<MeetingRecordingCoordinator>.Instance,
meetingMetadataProvider: new FixedMeetingMetadataProvider(new MeetingMetadata(
"Architecture Sync",
["Ada Lovelace (Contoso)"],
"",
null)),
meetingWorkflowEngine: workflowEngine);
await coordinator.StartAsync(CancellationToken.None);
await WaitUntilAsync(() => noteStore.SavedNote?.Frontmatter.Attendees.Count > 0);
Assert.Equal(["Ada Lovelace"], noteStore.SavedNote?.Frontmatter.Attendees);
Assert.Equal(["Ada Lovelace (Contoso)"], workflowEngine.AttendeeRequests);
await coordinator.StopAsync(CancellationToken.None);
}
[Fact] [Fact]
public async Task StartSkipsOutlookMeetingAttendeesAboveConfiguredImportLimit() public async Task StartSkipsOutlookMeetingAttendeesAboveConfiguredImportLimit()
{ {
+1 -1
View File
@@ -18,7 +18,7 @@ public sealed class TaskbarIconTests
Assert.Equal(RecordingProcessState.Idle, menu.State); Assert.Equal(RecordingProcessState.Idle, menu.State);
Assert.Contains(menu.Items, item => Assert.Contains(menu.Items, item =>
item.Action == MeetingTaskbarAction.EditRules && item.Action == MeetingTaskbarAction.EditRules &&
item.Text == "Settings and logs"); item.Text == "Open agent");
Assert.Contains(menu.Items, item => Assert.Contains(menu.Items, item =>
item.Action == MeetingTaskbarAction.StartRecording && item.Action == MeetingTaskbarAction.StartRecording &&
item.ProfileName == "default" && item.ProfileName == "default" &&
@@ -0,0 +1,44 @@
using MeetingAssistant.Workflow;
namespace MeetingAssistant.Tests;
internal sealed class TransformingAttendeeWorkflowEngine : IMeetingWorkflowEngine
{
private readonly string source;
private readonly string replacement;
public TransformingAttendeeWorkflowEngine(string source, string replacement)
{
this.source = source;
this.replacement = replacement;
}
public List<string> AttendeeRequests { get; } = [];
public Task RunAsync(
MeetingWorkflowEvent workflowEvent,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
public Task<string> TransformTranscriptLineAsync(
MeetingWorkflowEvent workflowEvent,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return Task.FromResult(workflowEvent.TranscriptLineText ?? "");
}
public Task<string> TransformAttendeeAsync(
MeetingWorkflowEvent workflowEvent,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
AttendeeRequests.Add(workflowEvent.AttendeeName ?? "");
return Task.FromResult(string.Equals(workflowEvent.AttendeeName, source, StringComparison.OrdinalIgnoreCase)
? replacement
: workflowEvent.AttendeeName ?? "");
}
}
@@ -637,6 +637,28 @@ public sealed class WorkflowRulesEditorTests
Assert.Equal("rules: []", await File.ReadAllTextAsync(fixture.RulesPath)); Assert.Equal("rules: []", await File.ReadAllTextAsync(fixture.RulesPath));
} }
[Fact]
public async Task RulesEditorToolsRefuseSideEffectingAttendeeAddedRules()
{
var fixture = await CreateRulesEditorFixtureAsync();
var result = await fixture.Tools.WriteRules("""
rules:
- name: attendee-side-effect
on:
- attendee_added:
contains: Contoso
steps:
- uses: add_context
value: '@Model.Attendee.Name'
""", replace_file: true);
Assert.StartsWith("Refused: workflow rules are invalid.", result);
Assert.Contains("attendee-side-effect", result);
Assert.Contains("attendee.name", result);
Assert.Equal("rules: []", await File.ReadAllTextAsync(fixture.RulesPath));
}
[Fact] [Fact]
public async Task RulesEditorToolsAcceptAndParseMaskedProfanityRedactionRule() public async Task RulesEditorToolsAcceptAndParseMaskedProfanityRedactionRule()
{ {
@@ -857,15 +879,13 @@ public sealed class WorkflowRulesEditorTests
Assert.Equal("", viewModel.Draft); Assert.Equal("", viewModel.Draft);
Assert.Equal( Assert.Equal(
[new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, "Rename ABS Daily")], [new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, "Rename ABS Daily")],
viewModel.Messages.ToArray()); viewModel.Messages.Select(item => item.Message).OfType<WorkflowRulesEditorChatMessage>().ToArray());
pipeline.Release.SetResult(); pipeline.Release.SetResult();
await sendTask.WaitAsync(TimeSpan.FromSeconds(2)); await sendTask.WaitAsync(TimeSpan.FromSeconds(2));
Assert.False(viewModel.IsThinking); Assert.False(viewModel.IsThinking);
Assert.Equal(2, viewModel.Messages.Count); AssertCompletedActivity(viewModel, ["Thinking..."], "Updated rules.");
Assert.Equal(WorkflowRulesEditorChatRole.Agent, viewModel.Messages[1].Role);
Assert.Equal("Updated rules.", viewModel.Messages[1].Content);
} }
[Fact] [Fact]
@@ -881,20 +901,19 @@ public sealed class WorkflowRulesEditorTests
Assert.False(viewModel.IsThinking); Assert.False(viewModel.IsThinking);
Assert.Equal("", viewModel.Draft); Assert.Equal("", viewModel.Draft);
Assert.Equal( Assert.Equal(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, "check logs"), viewModel.Messages[0].Message);
[ AssertCompletedActivity(
new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, "check logs"), viewModel,
new WorkflowRulesEditorChatMessage( ["Thinking..."],
WorkflowRulesEditorChatRole.Agent, "Meeting Summary Agent failed: The request timed out.");
"Settings and logs failed: The request timed out.")
],
viewModel.Messages.ToArray());
} }
[Fact] [Fact]
public async Task ViewModelShowsReconnectStatusReportedByPipeline() public async Task ViewModelShowsReconnectStatusReportedByPipeline()
{ {
var pipeline = new StatusReportingRulesEditorPipeline("Updated rules."); var pipeline = new ReportingRulesEditorPipeline(
"Updated rules.",
WorkflowRulesEditorActivityUpdate.Status("Reconnecting..."));
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline) var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
{ {
Draft = "check logs" Draft = "check logs"
@@ -911,7 +930,146 @@ public sealed class WorkflowRulesEditorTests
Assert.False(viewModel.IsThinking); Assert.False(viewModel.IsThinking);
Assert.Equal("Thinking...", viewModel.ActivityMessage); Assert.Equal("Thinking...", viewModel.ActivityMessage);
Assert.Equal("Updated rules.", viewModel.Messages[1].Content); AssertCompletedActivity(viewModel, ["Thinking...", "Reconnecting..."], "Updated rules.");
}
[Fact]
public async Task ViewModelShowsToolCallsAbovePersistentThinkingLine()
{
var pipeline = new ReportingRulesEditorPipeline(
"Updated rules.",
WorkflowRulesEditorActivityUpdate.ToolCall("read_logs"));
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
{
Draft = "check logs"
};
var sendTask = viewModel.SendAsync();
await pipeline.Reported.Task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.True(viewModel.IsThinking);
Assert.Equal("Thinking...", viewModel.ActivityMessage);
Assert.Equal(["Called tool: read_logs"], viewModel.ActivityMessages.ToArray());
pipeline.Release.SetResult();
await sendTask.WaitAsync(TimeSpan.FromSeconds(2));
Assert.False(viewModel.IsThinking);
Assert.Empty(viewModel.ActivityMessages);
AssertCompletedActivity(viewModel, ["Thinking...", "Called tool: read_logs"], "Updated rules.");
}
[Fact]
public async Task ViewModelShowsVisibleThinkingOutputInActivityExpander()
{
var pipeline = new ReportingRulesEditorPipeline(
"Updated rules.",
WorkflowRulesEditorActivityUpdate.Thinking("Checked the recent logs."),
WorkflowRulesEditorActivityUpdate.ToolCall("read_logs"),
WorkflowRulesEditorActivityUpdate.Thinking("Matched the rule to the user request."));
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
{
Draft = "check logs"
};
var sendTask = viewModel.SendAsync();
await pipeline.Reported.Task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.True(viewModel.IsThinking);
Assert.Equal("Thinking...", viewModel.ActivityMessage);
Assert.Equal(
[
"Checked the recent logs.",
"Called tool: read_logs",
"Matched the rule to the user request."
],
viewModel.ActivityMessages.ToArray());
pipeline.Release.SetResult();
await sendTask.WaitAsync(TimeSpan.FromSeconds(2));
Assert.False(viewModel.IsThinking);
Assert.Empty(viewModel.ActivityMessages);
AssertCompletedActivity(
viewModel,
[
"Checked the recent logs.",
"Called tool: read_logs",
"Matched the rule to the user request."
],
"Updated rules.");
}
[Fact]
public async Task ViewModelMarshalsBackgroundActivityUpdatesToCapturedContext()
{
var previousContext = SynchronizationContext.Current;
var context = new RecordingSynchronizationContext();
var pipeline = new BackgroundReportingRulesEditorPipeline(
"Updated rules.",
WorkflowRulesEditorActivityUpdate.Thinking("Checked the recent logs."));
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
{
Draft = "check logs"
};
SynchronizationContext.SetSynchronizationContext(context);
var sendTask = viewModel.SendAsync();
SynchronizationContext.SetSynchronizationContext(previousContext);
await pipeline.Reported.Task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.True(viewModel.IsThinking);
Assert.Equal(["Checked the recent logs."], viewModel.ActivityMessages.ToArray());
Assert.Equal(1, context.SendCount);
pipeline.Release.SetResult();
await sendTask.WaitAsync(TimeSpan.FromSeconds(2));
AssertCompletedActivity(viewModel, ["Checked the recent logs."], "Updated rules.");
}
[Fact]
public async Task ViewModelKeepsActivityItemsOutOfModelConversation()
{
var pipeline = new BlockingRulesEditorPipeline("Updated rules.");
var viewModel = new WorkflowRulesEditorChatViewModel(pipeline)
{
Draft = "first"
};
var firstSend = viewModel.SendAsync();
await pipeline.Started.Task.WaitAsync(TimeSpan.FromSeconds(2));
pipeline.Release.SetResult();
await firstSend.WaitAsync(TimeSpan.FromSeconds(2));
pipeline.Reset("Second response.");
viewModel.Draft = "second";
var secondSend = viewModel.SendAsync();
await pipeline.Started.Task.WaitAsync(TimeSpan.FromSeconds(2));
Assert.Equal(
[
new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, "first"),
new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.Agent, "Updated rules.")
],
pipeline.LastConversation);
pipeline.Release.SetResult();
await secondSend.WaitAsync(TimeSpan.FromSeconds(2));
}
private static void AssertCompletedActivity(
WorkflowRulesEditorChatViewModel viewModel,
IReadOnlyList<string> expectedActivityLines,
string expectedAgentMessage)
{
Assert.Equal(3, viewModel.Messages.Count);
Assert.Equal(WorkflowRulesEditorConversationItemKind.Activity, viewModel.Messages[1].Kind);
Assert.StartsWith("Worked for ", viewModel.Messages[1].Content);
Assert.Equal(expectedActivityLines, viewModel.Messages[1].ActivityLines);
Assert.Equal(WorkflowRulesEditorConversationItemKind.Message, viewModel.Messages[2].Kind);
Assert.Equal(WorkflowRulesEditorChatRole.Agent, viewModel.Messages[2].Message?.Role);
Assert.Equal(expectedAgentMessage, viewModel.Messages[2].Message?.Content);
} }
[Theory] [Theory]
@@ -1196,31 +1354,36 @@ public sealed class WorkflowRulesEditorTests
private sealed class BlockingRulesEditorPipeline : IWorkflowRulesEditorChatPipeline private sealed class BlockingRulesEditorPipeline : IWorkflowRulesEditorChatPipeline
{ {
private readonly string response; private string response;
public BlockingRulesEditorPipeline(string response) public BlockingRulesEditorPipeline(string response)
{ {
this.response = response; this.response = response;
} }
public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource Started { get; private set; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource Release { get; private set; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public IReadOnlyList<WorkflowRulesEditorChatMessage> LastConversation { get; private set; } = [];
public void Reset(string nextResponse)
{
response = nextResponse;
Started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
Release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
}
public async Task<WorkflowRulesEditorChatResult> SendAsync( public async Task<WorkflowRulesEditorChatResult> SendAsync(
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation, IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
string userMessage, string userMessage,
CancellationToken cancellationToken, CancellationToken cancellationToken,
Action<string>? statusChanged = null) Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null)
{ {
LastConversation = conversation;
Started.SetResult(); Started.SetResult();
await Release.Task.WaitAsync(cancellationToken); await Release.Task.WaitAsync(cancellationToken);
return new WorkflowRulesEditorChatResult( return new WorkflowRulesEditorChatResult(response);
response,
conversation
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, userMessage))
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.Agent, response))
.ToList());
} }
} }
@@ -1237,19 +1400,23 @@ public sealed class WorkflowRulesEditorTests
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation, IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
string userMessage, string userMessage,
CancellationToken cancellationToken, CancellationToken cancellationToken,
Action<string>? statusChanged = null) Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null)
{ {
throw exception; throw exception;
} }
} }
private sealed class StatusReportingRulesEditorPipeline : IWorkflowRulesEditorChatPipeline private sealed class ReportingRulesEditorPipeline : IWorkflowRulesEditorChatPipeline
{ {
private readonly string response; private readonly string response;
private readonly IReadOnlyList<WorkflowRulesEditorActivityUpdate> updates;
public StatusReportingRulesEditorPipeline(string response) public ReportingRulesEditorPipeline(
string response,
params WorkflowRulesEditorActivityUpdate[] updates)
{ {
this.response = response; this.response = response;
this.updates = updates;
} }
public TaskCompletionSource Reported { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource Reported { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
@@ -1260,17 +1427,57 @@ public sealed class WorkflowRulesEditorTests
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation, IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
string userMessage, string userMessage,
CancellationToken cancellationToken, CancellationToken cancellationToken,
Action<string>? statusChanged = null) Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null)
{ {
statusChanged?.Invoke("Reconnecting..."); foreach (var update in updates)
{
activityChanged?.Invoke(update);
}
Reported.SetResult(); Reported.SetResult();
await Release.Task.WaitAsync(cancellationToken); await Release.Task.WaitAsync(cancellationToken);
return new WorkflowRulesEditorChatResult( return new WorkflowRulesEditorChatResult(response);
response, }
conversation }
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, userMessage))
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.Agent, response)) private sealed class BackgroundReportingRulesEditorPipeline : IWorkflowRulesEditorChatPipeline
.ToList()); {
private readonly string response;
private readonly WorkflowRulesEditorActivityUpdate update;
public BackgroundReportingRulesEditorPipeline(
string response,
WorkflowRulesEditorActivityUpdate update)
{
this.response = response;
this.update = update;
}
public TaskCompletionSource Reported { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public async Task<WorkflowRulesEditorChatResult> SendAsync(
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
string userMessage,
CancellationToken cancellationToken,
Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null)
{
await Task.Run(() => activityChanged?.Invoke(update), cancellationToken);
Reported.SetResult();
await Release.Task.WaitAsync(cancellationToken);
return new WorkflowRulesEditorChatResult(response);
}
}
private sealed class RecordingSynchronizationContext : SynchronizationContext
{
public int SendCount { get; private set; }
public override void Send(SendOrPostCallback callback, object? state)
{
SendCount++;
callback(state);
} }
} }
@@ -213,7 +213,12 @@ public sealed class MeetingRecordingCoordinator
currentMeetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken); currentMeetingNote = await meetingNoteStore.ReadAsync(currentMeetingNote.Path, cancellationToken);
if (suppliedMetadata is not null) if (suppliedMetadata is not null)
{ {
await ApplyMeetingMetadataAsync(currentMeetingNote, suppliedMetadata, runOptions, cancellationToken); await ApplyMeetingMetadataAsync(
currentArtifacts,
currentMeetingNote,
suppliedMetadata,
runOptions,
cancellationToken);
currentMeetingNote = await meetingNoteStore.SaveAsync( currentMeetingNote = await meetingNoteStore.SaveAsync(
currentMeetingNote, currentMeetingNote,
runOptions, runOptions,
@@ -1030,7 +1035,7 @@ public sealed class MeetingRecordingCoordinator
} }
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, CancellationToken.None); var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, CancellationToken.None);
await ApplyMeetingMetadataAsync(meetingNote, metadata, run.Options, CancellationToken.None); await ApplyMeetingMetadataAsync(run.Artifacts, meetingNote, metadata, run.Options, CancellationToken.None);
meetingNote = await meetingNoteStore.SaveAsync(meetingNote, run.Options, CancellationToken.None); meetingNote = await meetingNoteStore.SaveAsync(meetingNote, run.Options, CancellationToken.None);
if (currentMeetingNote?.Path == meetingNote.Path) if (currentMeetingNote?.Path == meetingNote.Path)
{ {
@@ -1082,6 +1087,7 @@ public sealed class MeetingRecordingCoordinator
} }
private async Task ApplyMeetingMetadataAsync( private async Task ApplyMeetingMetadataAsync(
MeetingSessionArtifacts artifacts,
MeetingNote meetingNote, MeetingNote meetingNote,
MeetingMetadata metadata, MeetingMetadata metadata,
MeetingAssistantOptions options, MeetingAssistantOptions options,
@@ -1104,10 +1110,39 @@ public sealed class MeetingRecordingCoordinator
return; return;
} }
meetingNote.Frontmatter.Attendees = await CanonicalizeAttendeesAsync(metadata.Attendees, cancellationToken); var canonicalized = await CanonicalizeAttendeesAsync(metadata.Attendees, cancellationToken);
meetingNote.Frontmatter.Attendees = await TransformAttendeesAsync(
artifacts,
canonicalized,
options,
cancellationToken);
} }
} }
private async Task<List<string>> TransformAttendeesAsync(
MeetingSessionArtifacts artifacts,
IReadOnlyList<string> attendees,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var transformed = new List<string>();
foreach (var attendee in attendees)
{
var value = await TransformAttendeeAsync(artifacts, attendee, options, cancellationToken);
var storageValue = string.Equals(value, attendee, StringComparison.Ordinal)
? attendee.Trim()
: NormalizeAttendeeName(value);
var normalized = NormalizeAttendeeName(storageValue);
if (!string.IsNullOrWhiteSpace(normalized) &&
!transformed.Contains(normalized, StringComparer.OrdinalIgnoreCase))
{
transformed.Add(storageValue);
}
}
return transformed;
}
private async Task<List<string>> CanonicalizeAttendeesAsync( private async Task<List<string>> CanonicalizeAttendeesAsync(
IReadOnlyList<string> attendees, IReadOnlyList<string> attendees,
CancellationToken cancellationToken) CancellationToken cancellationToken)
@@ -1422,8 +1457,17 @@ public sealed class MeetingRecordingCoordinator
continue; continue;
} }
var transformedDisplayName = await TransformAttendeeAsync(
artifacts,
match.DisplayName.Trim(),
runOptions,
cancellationToken);
var displayName = string.Equals(transformedDisplayName, match.DisplayName.Trim(), StringComparison.Ordinal)
? match.DisplayName.Trim()
: NormalizeAttendeeName(transformedDisplayName);
var acceptedNames = match.AcceptedNames var acceptedNames = match.AcceptedNames
.Append(match.DisplayName) .Append(match.DisplayName)
.Append(displayName)
.Select(NormalizeAttendeeName) .Select(NormalizeAttendeeName)
.Where(name => !string.IsNullOrWhiteSpace(name)) .Where(name => !string.IsNullOrWhiteSpace(name))
.Distinct(StringComparer.OrdinalIgnoreCase) .Distinct(StringComparer.OrdinalIgnoreCase)
@@ -1442,7 +1486,6 @@ public sealed class MeetingRecordingCoordinator
continue; continue;
} }
var displayName = match.DisplayName.Trim();
meetingNote.Frontmatter.Attendees.Add(displayName); meetingNote.Frontmatter.Attendees.Add(displayName);
existingNames.Add(NormalizeAttendeeName(displayName)); existingNames.Add(NormalizeAttendeeName(displayName));
changed = true; changed = true;
@@ -1502,6 +1545,18 @@ public sealed class MeetingRecordingCoordinator
return MeetingAttendeeNames.NormalizeDisplayName(attendee); return MeetingAttendeeNames.NormalizeDisplayName(attendee);
} }
private async Task<string> TransformAttendeeAsync(
MeetingSessionArtifacts artifacts,
string attendee,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return await meetingWorkflowEngine.TransformAttendeeAsync(
MeetingWorkflowEvent.AttendeeAdded(artifacts, attendee),
options,
cancellationToken);
}
private static bool RemoveDuplicateAcceptedAliases( private static bool RemoveDuplicateAcceptedAliases(
List<string> attendees, List<string> attendees,
string displayName, string displayName,
@@ -5,6 +5,7 @@ using System.Runtime.InteropServices;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using MeetingAssistant.MeetingNotes; using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Speakers; using MeetingAssistant.Speakers;
using MeetingAssistant.Workflow;
namespace MeetingAssistant.Screenshots; namespace MeetingAssistant.Screenshots;
@@ -149,6 +150,7 @@ public sealed partial class MeetingScreenshotService :
private readonly IMeetingArtifactStore artifactStore; private readonly IMeetingArtifactStore artifactStore;
private readonly IMeetingNoteStore meetingNoteStore; private readonly IMeetingNoteStore meetingNoteStore;
private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer; private readonly ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer;
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
private readonly IScreenshotOcrClient ocrClient; private readonly IScreenshotOcrClient ocrClient;
private readonly ILogger<MeetingScreenshotService> logger; private readonly ILogger<MeetingScreenshotService> logger;
private readonly ConcurrentDictionary<string, List<PendingOcrTask>> pendingOcrByContext = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary<string, List<PendingOcrTask>> pendingOcrByContext = new(StringComparer.OrdinalIgnoreCase);
@@ -160,12 +162,14 @@ public sealed partial class MeetingScreenshotService :
IMeetingNoteStore meetingNoteStore, IMeetingNoteStore meetingNoteStore,
ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer, ISpeakerIdentityAttendeeCanonicalizer attendeeCanonicalizer,
IScreenshotOcrClient ocrClient, IScreenshotOcrClient ocrClient,
ILogger<MeetingScreenshotService> logger) ILogger<MeetingScreenshotService> logger,
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
{ {
this.screenshotCapture = screenshotCapture; this.screenshotCapture = screenshotCapture;
this.artifactStore = artifactStore; this.artifactStore = artifactStore;
this.meetingNoteStore = meetingNoteStore; this.meetingNoteStore = meetingNoteStore;
this.attendeeCanonicalizer = attendeeCanonicalizer; this.attendeeCanonicalizer = attendeeCanonicalizer;
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
this.ocrClient = ocrClient; this.ocrClient = ocrClient;
this.logger = logger; this.logger = logger;
} }
@@ -434,6 +438,7 @@ public sealed partial class MeetingScreenshotService :
await AddOcrAttendeesAsync( await AddOcrAttendeesAsync(
artifacts, artifacts,
result.Attendees, result.Attendees,
options,
CancellationToken.None); CancellationToken.None);
} }
} }
@@ -477,6 +482,7 @@ public sealed partial class MeetingScreenshotService :
private async Task AddOcrAttendeesAsync( private async Task AddOcrAttendeesAsync(
MeetingSessionArtifacts artifacts, MeetingSessionArtifacts artifacts,
IReadOnlyList<string> attendees, IReadOnlyList<string> attendees,
MeetingAssistantOptions options,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var additions = MeetingAttendeeNames.NormalizeDistinct(attendees); var additions = MeetingAttendeeNames.NormalizeDistinct(attendees);
@@ -488,8 +494,13 @@ public sealed partial class MeetingScreenshotService :
try try
{ {
var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken); var meetingNote = await meetingNoteStore.ReadAsync(artifacts.MeetingNotePath, cancellationToken);
var transformedAdditions = await TransformAttendeesAsync(
artifacts,
additions,
options,
cancellationToken);
var canonicalized = await attendeeCanonicalizer.CanonicalizeAsync( var canonicalized = await attendeeCanonicalizer.CanonicalizeAsync(
meetingNote.Frontmatter.Attendees.Concat(additions).ToList(), meetingNote.Frontmatter.Attendees.Concat(transformedAdditions).ToList(),
cancellationToken); cancellationToken);
if (meetingNote.Frontmatter.Attendees.SequenceEqual(canonicalized, StringComparer.Ordinal)) if (meetingNote.Frontmatter.Attendees.SequenceEqual(canonicalized, StringComparer.Ordinal))
{ {
@@ -512,6 +523,30 @@ public sealed partial class MeetingScreenshotService :
} }
} }
private async Task<IReadOnlyList<string>> TransformAttendeesAsync(
MeetingSessionArtifacts artifacts,
IReadOnlyList<string> attendees,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
var transformed = new List<string>();
foreach (var attendee in attendees)
{
var value = await meetingWorkflowEngine.TransformAttendeeAsync(
MeetingWorkflowEvent.AttendeeAdded(artifacts, attendee),
options,
cancellationToken);
var normalized = MeetingAttendeeNames.NormalizeDisplayName(value);
if (!string.IsNullOrWhiteSpace(normalized) &&
!transformed.Contains(normalized, StringComparer.OrdinalIgnoreCase))
{
transformed.Add(normalized);
}
}
return transformed;
}
private async Task<bool> ReplaceOcrPlaceholderAsync( private async Task<bool> ReplaceOcrPlaceholderAsync(
string assistantContextPath, string assistantContextPath,
string screenshotId, string screenshotId,
@@ -22,6 +22,7 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
private readonly LiteLlmResponsesCompactionOptions? compactionOptions; private readonly LiteLlmResponsesCompactionOptions? compactionOptions;
private readonly ILogger? logger; private readonly ILogger? logger;
private readonly Action? retrying; private readonly Action? retrying;
private readonly Action<string>? reasoningSummaryChanged;
private int responsesRequestCount; private int responsesRequestCount;
public LiteLlmResponsesChatClient( public LiteLlmResponsesChatClient(
@@ -35,7 +36,8 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
LiteLlmResponsesCompactionOptions? compactionOptions = null, LiteLlmResponsesCompactionOptions? compactionOptions = null,
ILogger? logger = null, ILogger? logger = null,
bool firstRequestIsUser = true, bool firstRequestIsUser = true,
Action? retrying = null) Action? retrying = null,
Action<string>? reasoningSummaryChanged = null)
: this( : this(
new HttpClient { BaseAddress = NormalizeEndpoint(endpoint) }, new HttpClient { BaseAddress = NormalizeEndpoint(endpoint) },
apiKey, apiKey,
@@ -47,7 +49,8 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
compactionOptions, compactionOptions,
logger, logger,
firstRequestIsUser, firstRequestIsUser,
retrying) retrying,
reasoningSummaryChanged)
{ {
} }
@@ -62,7 +65,8 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
LiteLlmResponsesCompactionOptions? compactionOptions = null, LiteLlmResponsesCompactionOptions? compactionOptions = null,
ILogger? logger = null, ILogger? logger = null,
bool firstRequestIsUser = true, bool firstRequestIsUser = true,
Action? retrying = null) Action? retrying = null,
Action<string>? reasoningSummaryChanged = null)
{ {
this.httpClient = httpClient; this.httpClient = httpClient;
this.httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey); this.httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
@@ -74,6 +78,7 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
this.compactionOptions = compactionOptions; this.compactionOptions = compactionOptions;
this.logger = logger; this.logger = logger;
this.retrying = retrying; this.retrying = retrying;
this.reasoningSummaryChanged = reasoningSummaryChanged;
responsesRequestCount = firstRequestIsUser ? 0 : 1; responsesRequestCount = firstRequestIsUser ? 0 : 1;
} }
@@ -90,7 +95,8 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
var payload = await CreateCompactedPayloadAsync(messages.ToList(), options, cancellationToken).ConfigureAwait(false); var payload = await CreateCompactedPayloadAsync(messages.ToList(), options, cancellationToken).ConfigureAwait(false);
var responseJson = await PostWithRetryAsync(payload, cancellationToken).ConfigureAwait(false); var responseJson = await PostWithRetryAsync(payload, cancellationToken).ConfigureAwait(false);
var response = ParseResponseJson(responseJson); var response = ParseResponseJson(responseJson, out var reasoningSummaries);
ReportVisibleReasoningSummaries(reasoningSummaries);
LogResponseDiagnostics(responseJson, response); LogResponseDiagnostics(responseJson, response);
ThrowIfResponseHasNoContent(responseJson, response); ThrowIfResponseHasNoContent(responseJson, response);
LogResponseUsage(response.Usage); LogResponseUsage(response.Usage);
@@ -122,10 +128,18 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
} }
public static ChatResponse ParseResponseJson(string responseJson) public static ChatResponse ParseResponseJson(string responseJson)
{
return ParseResponseJson(responseJson, out _);
}
private static ChatResponse ParseResponseJson(
string responseJson,
out IReadOnlyList<string> reasoningSummaries)
{ {
using var document = JsonDocument.Parse(responseJson); using var document = JsonDocument.Parse(responseJson);
var root = document.RootElement; var root = document.RootElement;
var contents = new List<AIContent>(); var contents = new List<AIContent>();
reasoningSummaries = ExtractVisibleReasoningSummaries(root);
if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array) if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array)
{ {
@@ -162,6 +176,54 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
}; };
} }
internal static IReadOnlyList<string> ExtractVisibleReasoningSummaries(string responseJson)
{
using var document = JsonDocument.Parse(responseJson);
return ExtractVisibleReasoningSummaries(document.RootElement);
}
private static IReadOnlyList<string> ExtractVisibleReasoningSummaries(JsonElement root)
{
var summaries = new List<string>();
if (!root.TryGetProperty("output", out var output) || output.ValueKind != JsonValueKind.Array)
{
return summaries;
}
foreach (var item in output.EnumerateArray())
{
if (GetString(item, "type") == "reasoning")
{
AddVisibleReasoningSummaryText(summaries, item);
}
}
return summaries;
}
private void ReportVisibleReasoningSummaries(IReadOnlyList<string> summaries)
{
if (reasoningSummaryChanged is null)
{
return;
}
foreach (var summary in summaries)
{
try
{
reasoningSummaryChanged(summary);
}
catch (Exception exception)
{
logger?.LogWarning(
exception,
"Reasoning summary callback failed; continuing with the LiteLLM response.");
}
}
}
private async Task<JsonObject> CreateCompactedPayloadAsync( private async Task<JsonObject> CreateCompactedPayloadAsync(
IReadOnlyList<ChatMessage> messages, IReadOnlyList<ChatMessage> messages,
ChatOptions? options, ChatOptions? options,
@@ -660,6 +722,66 @@ public sealed class LiteLlmResponsesChatClient : IChatClient
} }
} }
private static void AddVisibleReasoningSummaryText(List<string> summaries, JsonElement item)
{
AddVisibleReasoningSummaryText(summaries, item, "summary");
AddVisibleReasoningSummaryText(summaries, item, "content");
}
private static void AddVisibleReasoningSummaryText(
List<string> summaries,
JsonElement item,
string propertyName)
{
if (!item.TryGetProperty(propertyName, out var property))
{
return;
}
if (property.ValueKind == JsonValueKind.String)
{
AddNonEmpty(summaries, property.GetString());
return;
}
if (property.ValueKind != JsonValueKind.Array)
{
return;
}
foreach (var summaryItem in property.EnumerateArray())
{
if (summaryItem.ValueKind == JsonValueKind.String)
{
AddNonEmpty(summaries, summaryItem.GetString());
}
else if (summaryItem.ValueKind == JsonValueKind.Object)
{
if (propertyName == "content" && !IsSummaryContent(summaryItem))
{
continue;
}
AddNonEmpty(summaries, GetString(summaryItem, "text"));
}
}
}
private static bool IsSummaryContent(JsonElement item)
{
var type = GetString(item, "type");
return !string.IsNullOrWhiteSpace(type)
&& type.Contains("summary", StringComparison.OrdinalIgnoreCase);
}
private static void AddNonEmpty(List<string> values, string? value)
{
if (!string.IsNullOrWhiteSpace(value))
{
values.Add(value.Trim());
}
}
private static Dictionary<string, object?> ParseArguments(string? arguments) private static Dictionary<string, object?> ParseArguments(string? arguments)
{ {
if (string.IsNullOrWhiteSpace(arguments)) if (string.IsNullOrWhiteSpace(arguments))
@@ -1,6 +1,7 @@
using MeetingAssistant.MeetingNotes; using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Speakers; using MeetingAssistant.Speakers;
using MeetingAssistant.Transcription; using MeetingAssistant.Transcription;
using MeetingAssistant.Workflow;
using YamlDotNet.Serialization; using YamlDotNet.Serialization;
namespace MeetingAssistant.Summary; namespace MeetingAssistant.Summary;
@@ -13,6 +14,7 @@ public sealed class MeetingSummaryTools
private readonly MeetingAssistantOptions options; private readonly MeetingAssistantOptions options;
private readonly IDictationWordStore? dictationWordStore; private readonly IDictationWordStore? dictationWordStore;
private readonly SummaryAgentWriteAudit? writeAudit; private readonly SummaryAgentWriteAudit? writeAudit;
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
private readonly BoundMeetingProjectResolver projectResolver; private readonly BoundMeetingProjectResolver projectResolver;
public MeetingSummaryTools(MeetingSessionArtifacts artifacts) public MeetingSummaryTools(MeetingSessionArtifacts artifacts)
@@ -24,12 +26,14 @@ public sealed class MeetingSummaryTools
MeetingSessionArtifacts artifacts, MeetingSessionArtifacts artifacts,
MeetingAssistantOptions options, MeetingAssistantOptions options,
IDictationWordStore? dictationWordStore = null, IDictationWordStore? dictationWordStore = null,
SummaryAgentWriteAudit? writeAudit = null) SummaryAgentWriteAudit? writeAudit = null,
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
{ {
this.artifacts = artifacts; this.artifacts = artifacts;
this.options = options; this.options = options;
this.dictationWordStore = dictationWordStore; this.dictationWordStore = dictationWordStore;
this.writeAudit = writeAudit; this.writeAudit = writeAudit;
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
projectResolver = new BoundMeetingProjectResolver(options); projectResolver = new BoundMeetingProjectResolver(options);
} }
@@ -98,7 +102,11 @@ public sealed class MeetingSummaryTools
return "Refused: attendee must not be empty."; return "Refused: attendee must not be empty.";
} }
var displayName = attendee.Trim(); var displayName = await meetingWorkflowEngine.TransformAttendeeAsync(
MeetingWorkflowEvent.AttendeeAdded(artifacts, attendee.Trim()),
options,
CancellationToken.None);
displayName = displayName.Trim();
var normalized = MeetingAttendeeNames.NormalizeDisplayName(displayName); var normalized = MeetingAttendeeNames.NormalizeDisplayName(displayName);
if (string.IsNullOrWhiteSpace(normalized)) if (string.IsNullOrWhiteSpace(normalized))
{ {
@@ -1,5 +1,6 @@
using MeetingAssistant.MeetingNotes; using MeetingAssistant.MeetingNotes;
using MeetingAssistant.Transcription; using MeetingAssistant.Transcription;
using MeetingAssistant.Workflow;
using Microsoft.Agents.AI; using Microsoft.Agents.AI;
using Microsoft.Agents.AI.Compaction; using Microsoft.Agents.AI.Compaction;
using Microsoft.Extensions.AI; using Microsoft.Extensions.AI;
@@ -17,6 +18,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
private readonly IMeetingSummaryFailureWriter failureWriter; private readonly IMeetingSummaryFailureWriter failureWriter;
private readonly IMeetingSummaryInstructionBuilder instructionBuilder; private readonly IMeetingSummaryInstructionBuilder instructionBuilder;
private readonly IDictationWordStore dictationWordStore; private readonly IDictationWordStore dictationWordStore;
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
public OpenAiMeetingSummaryAgentPipeline( public OpenAiMeetingSummaryAgentPipeline(
IOptions<MeetingAssistantOptions> options, IOptions<MeetingAssistantOptions> options,
@@ -25,7 +27,8 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
IServiceProvider services, IServiceProvider services,
IMeetingSummaryFailureWriter failureWriter, IMeetingSummaryFailureWriter failureWriter,
IMeetingSummaryInstructionBuilder instructionBuilder, IMeetingSummaryInstructionBuilder instructionBuilder,
IDictationWordStore dictationWordStore) IDictationWordStore dictationWordStore,
IMeetingWorkflowEngine meetingWorkflowEngine)
{ {
this.options = options.Value; this.options = options.Value;
this.loggerFactory = loggerFactory; this.loggerFactory = loggerFactory;
@@ -34,6 +37,7 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
this.failureWriter = failureWriter; this.failureWriter = failureWriter;
this.instructionBuilder = instructionBuilder; this.instructionBuilder = instructionBuilder;
this.dictationWordStore = dictationWordStore; this.dictationWordStore = dictationWordStore;
this.meetingWorkflowEngine = meetingWorkflowEngine;
} }
public async Task<MeetingSummaryRunResult> RunAsync( public async Task<MeetingSummaryRunResult> RunAsync(
@@ -51,7 +55,12 @@ public sealed class OpenAiMeetingSummaryAgentPipeline : IMeetingSummaryPipeline
var agentOptions = options.Agent; var agentOptions = options.Agent;
var key = ResolveApiKey(agentOptions); var key = ResolveApiKey(agentOptions);
var writeAudit = new SummaryAgentWriteAudit(artifacts); var writeAudit = new SummaryAgentWriteAudit(artifacts);
var meetingTools = new MeetingSummaryTools(artifacts, options, dictationWordStore, writeAudit); var meetingTools = new MeetingSummaryTools(
artifacts,
options,
dictationWordStore,
writeAudit,
meetingWorkflowEngine);
var tools = CreateTools(meetingTools); var tools = CreateTools(meetingTools);
var instructions = await instructionBuilder.BuildAsync(artifacts, options, cancellationToken); var instructions = await instructionBuilder.BuildAsync(artifacts, options, cancellationToken);
using var compactionSummaryClient = new LiteLlmResponsesChatClient( using var compactionSummaryClient = new LiteLlmResponsesChatClient(
@@ -38,7 +38,7 @@ public static class MeetingTaskbarMenuBuilder
{ {
var items = new List<MeetingTaskbarMenuItem> var items = new List<MeetingTaskbarMenuItem>
{ {
new("Settings and logs", MeetingTaskbarAction.EditRules) new("Open agent", MeetingTaskbarAction.EditRules)
}; };
if (microphones is { Count: > 0 }) if (microphones is { Count: > 0 })
@@ -17,6 +17,14 @@ public interface IMeetingWorkflowEngine
MeetingWorkflowEvent workflowEvent, MeetingWorkflowEvent workflowEvent,
MeetingAssistantOptions options, MeetingAssistantOptions options,
CancellationToken cancellationToken); CancellationToken cancellationToken);
Task<string> TransformAttendeeAsync(
MeetingWorkflowEvent workflowEvent,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return Task.FromResult(workflowEvent.AttendeeName ?? "");
}
} }
public sealed class NoopMeetingWorkflowEngine : IMeetingWorkflowEngine public sealed class NoopMeetingWorkflowEngine : IMeetingWorkflowEngine
@@ -38,6 +46,14 @@ public sealed class NoopMeetingWorkflowEngine : IMeetingWorkflowEngine
{ {
return Task.FromResult(workflowEvent.TranscriptLineText ?? ""); return Task.FromResult(workflowEvent.TranscriptLineText ?? "");
} }
public Task<string> TransformAttendeeAsync(
MeetingWorkflowEvent workflowEvent,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
return Task.FromResult(workflowEvent.AttendeeName ?? "");
}
} }
public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
@@ -54,7 +70,8 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
"state.to", "state.to",
"speaker.name", "speaker.name",
MeetingWorkflowRuleSchema.PropertyTranscriptLine, MeetingWorkflowRuleSchema.PropertyTranscriptLine,
MeetingWorkflowRuleSchema.ParameterTranscriptSpeaker MeetingWorkflowRuleSchema.ParameterTranscriptSpeaker,
MeetingWorkflowRuleSchema.PropertyAttendeeName
]; ];
private readonly IMeetingWorkflowRulesProvider rulesProvider; private readonly IMeetingWorkflowRulesProvider rulesProvider;
@@ -98,6 +115,21 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
?? ""; ?? "";
} }
public async Task<string> TransformAttendeeAsync(
MeetingWorkflowEvent workflowEvent,
MeetingAssistantOptions options,
CancellationToken cancellationToken)
{
if (workflowEvent.Type != MeetingWorkflowEventType.AttendeeAdded)
{
throw new ArgumentException("Workflow event must be an attendee added event.", nameof(workflowEvent));
}
return await RunCoreAsync(workflowEvent, options, cancellationToken)
?? workflowEvent.AttendeeName
?? "";
}
private async Task<string?> RunCoreAsync( private async Task<string?> RunCoreAsync(
MeetingWorkflowEvent workflowEvent, MeetingWorkflowEvent workflowEvent,
MeetingAssistantOptions options, MeetingAssistantOptions options,
@@ -107,7 +139,7 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
var context = new MeetingWorkflowExecutionContext(workflowEvent); var context = new MeetingWorkflowExecutionContext(workflowEvent);
if (rules.Count == 0) if (rules.Count == 0)
{ {
return context.TranscriptLine; return context.ResultValue;
} }
var meeting = await meetingNoteStore.ReadAsync( var meeting = await meetingNoteStore.ReadAsync(
@@ -134,10 +166,18 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
var model = CreateTemplateModel(meeting, context); var model = CreateTemplateModel(meeting, context);
foreach (var step in rule.Steps) foreach (var step in rule.Steps)
{ {
if (workflowEvent.Type == MeetingWorkflowEventType.AttendeeAdded &&
!IsAttendeeTransformStep(step))
{
throw new InvalidOperationException(
"attendee_added workflow rules can only transform attendee.name with set_property.");
}
var stepChanged = await ApplyStepAsync( var stepChanged = await ApplyStepAsync(
step, step,
meeting, meeting,
context, context,
options,
model, model,
cancellationToken); cancellationToken);
ruleNoteChanged |= stepChanged; ruleNoteChanged |= stepChanged;
@@ -176,7 +216,7 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
cancellationToken); cancellationToken);
} }
return context.TranscriptLine; return context.ResultValue;
} }
private static bool MatchesTrigger( private static bool MatchesTrigger(
@@ -229,9 +269,41 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
StringComparison.OrdinalIgnoreCase)); StringComparison.OrdinalIgnoreCase));
} }
if (trigger.AttendeeAdded is not null)
{
return workflowEvent.Type == MeetingWorkflowEventType.AttendeeAdded &&
MatchesAttendeeAddedTrigger(trigger.AttendeeAdded, workflowEvent.AttendeeName);
}
return false; return false;
} }
private static bool MatchesAttendeeAddedTrigger(
MeetingWorkflowAttendeeAddedTrigger trigger,
string? attendeeName)
{
var attendee = attendeeName ?? "";
if (!string.IsNullOrWhiteSpace(trigger.EqualsValue) &&
!string.Equals(attendee, trigger.EqualsValue.Trim(), StringComparison.OrdinalIgnoreCase))
{
return false;
}
if (!string.IsNullOrWhiteSpace(trigger.Contains) &&
!attendee.Contains(trigger.Contains.Trim(), StringComparison.OrdinalIgnoreCase))
{
return false;
}
if (!string.IsNullOrWhiteSpace(trigger.Regex) &&
!Regex.IsMatch(attendee, trigger.Regex, RegexOptions.IgnoreCase))
{
return false;
}
return true;
}
private bool EvaluateConditions( private bool EvaluateConditions(
IReadOnlyList<MeetingWorkflowCondition> conditions, IReadOnlyList<MeetingWorkflowCondition> conditions,
MeetingNote meeting, MeetingNote meeting,
@@ -298,6 +370,7 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
MeetingWorkflowStep step, MeetingWorkflowStep step,
MeetingNote meeting, MeetingNote meeting,
MeetingWorkflowExecutionContext context, MeetingWorkflowExecutionContext context,
MeetingAssistantOptions options,
MeetingWorkflowTemplateModel model, MeetingWorkflowTemplateModel model,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
@@ -305,7 +378,11 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
switch (step.Uses.Trim().ToLowerInvariant()) switch (step.Uses.Trim().ToLowerInvariant())
{ {
case MeetingWorkflowRuleSchema.StepAddAttendee: case MeetingWorkflowRuleSchema.StepAddAttendee:
return AddUnique(meeting.Frontmatter.Attendees, value, MeetingAttendeeNames.NormalizeDisplayName); var attendee = await TransformAttendeeAsync(
MeetingWorkflowEvent.AttendeeAdded(context.Event.Artifacts, value),
options,
cancellationToken);
return AddUnique(meeting.Frontmatter.Attendees, attendee, MeetingAttendeeNames.NormalizeDisplayName);
case MeetingWorkflowRuleSchema.StepRemoveAttendee: case MeetingWorkflowRuleSchema.StepRemoveAttendee:
return RemoveValue(meeting.Frontmatter.Attendees, value); return RemoveValue(meeting.Frontmatter.Attendees, value);
case MeetingWorkflowRuleSchema.StepAddProject: case MeetingWorkflowRuleSchema.StepAddProject:
@@ -319,6 +396,9 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
case MeetingWorkflowRuleSchema.StepSetProperty when IsTranscriptLineProperty(step.Property ?? step.Name): case MeetingWorkflowRuleSchema.StepSetProperty when IsTranscriptLineProperty(step.Property ?? step.Name):
SetTranscriptLine(context, value); SetTranscriptLine(context, value);
return false; return false;
case MeetingWorkflowRuleSchema.StepSetProperty when IsAttendeeNameProperty(step.Property ?? step.Name):
SetAttendeeName(context, value);
return false;
case MeetingWorkflowRuleSchema.StepSetProperty: case MeetingWorkflowRuleSchema.StepSetProperty:
return SetProperty(meeting, step.Property ?? step.Name, value); return SetProperty(meeting, step.Property ?? step.Name, value);
default: default:
@@ -370,11 +450,39 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
context.TranscriptLine = value; context.TranscriptLine = value;
} }
private static void SetAttendeeName(
MeetingWorkflowExecutionContext context,
string value)
{
if (context.Event.Type != MeetingWorkflowEventType.AttendeeAdded)
{
throw new InvalidOperationException("The attendee.name property can only be set during attendee_added events.");
}
if (string.Equals(context.AttendeeName, value, StringComparison.Ordinal))
{
return;
}
context.AttendeeName = value;
}
private static bool IsTranscriptLineProperty(string? property) private static bool IsTranscriptLineProperty(string? property)
{ {
return MeetingWorkflowRuleSchema.IsProperty(property, MeetingWorkflowRuleSchema.PropertyTranscriptLine); return MeetingWorkflowRuleSchema.IsProperty(property, MeetingWorkflowRuleSchema.PropertyTranscriptLine);
} }
private static bool IsAttendeeNameProperty(string? property)
{
return MeetingWorkflowRuleSchema.IsProperty(property, MeetingWorkflowRuleSchema.PropertyAttendeeName);
}
private static bool IsAttendeeTransformStep(MeetingWorkflowStep step)
{
return MeetingWorkflowRuleSchema.IsStep(step.Uses, MeetingWorkflowRuleSchema.StepSetProperty) &&
IsAttendeeNameProperty(step.Property ?? step.Name);
}
private static bool AddUnique( private static bool AddUnique(
List<string> values, List<string> values,
string value, string value,
@@ -456,7 +564,8 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
["state.to"] = workflowEvent.ToState is { } to ? MeetingWorkflowStateNames.ToRuleName(to) : "", ["state.to"] = workflowEvent.ToState is { } to ? MeetingWorkflowStateNames.ToRuleName(to) : "",
["speaker.name"] = workflowEvent.SpeakerName, ["speaker.name"] = workflowEvent.SpeakerName,
[MeetingWorkflowRuleSchema.PropertyTranscriptLine] = context.TranscriptLine ?? "", [MeetingWorkflowRuleSchema.PropertyTranscriptLine] = context.TranscriptLine ?? "",
[MeetingWorkflowRuleSchema.ParameterTranscriptSpeaker] = workflowEvent.SpeakerName ?? "" [MeetingWorkflowRuleSchema.ParameterTranscriptSpeaker] = workflowEvent.SpeakerName ?? "",
[MeetingWorkflowRuleSchema.PropertyAttendeeName] = context.AttendeeName ?? ""
}; };
} }
@@ -480,7 +589,10 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
: new MeetingWorkflowSpeakerModel(workflowEvent.SpeakerName), : new MeetingWorkflowSpeakerModel(workflowEvent.SpeakerName),
new MeetingWorkflowTranscriptModel( new MeetingWorkflowTranscriptModel(
context.TranscriptLine ?? "", context.TranscriptLine ?? "",
workflowEvent.SpeakerName ?? "")); workflowEvent.SpeakerName ?? ""),
string.IsNullOrWhiteSpace(context.AttendeeName)
? null
: new MeetingWorkflowAttendeeModel(context.AttendeeName));
} }
private sealed class MeetingWorkflowExecutionContext private sealed class MeetingWorkflowExecutionContext
@@ -489,10 +601,23 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
{ {
Event = workflowEvent; Event = workflowEvent;
TranscriptLine = workflowEvent.TranscriptLineText; TranscriptLine = workflowEvent.TranscriptLineText;
AttendeeName = workflowEvent.AttendeeName;
} }
public MeetingWorkflowEvent Event { get; } public MeetingWorkflowEvent Event { get; }
public string? TranscriptLine { get; set; } public string? TranscriptLine { get; set; }
public string? AttendeeName { get; set; }
public string? ResultValue
{
get
{
return Event.Type == MeetingWorkflowEventType.AttendeeAdded
? AttendeeName
: TranscriptLine;
}
}
} }
} }
@@ -7,7 +7,8 @@ public enum MeetingWorkflowEventType
Created, Created,
StateTransition, StateTransition,
SpeakerIdentified, SpeakerIdentified,
TranscriptLine TranscriptLine,
AttendeeAdded
} }
public sealed record MeetingWorkflowEvent( public sealed record MeetingWorkflowEvent(
@@ -16,7 +17,8 @@ public sealed record MeetingWorkflowEvent(
AssistantContextState? FromState = null, AssistantContextState? FromState = null,
AssistantContextState? ToState = null, AssistantContextState? ToState = null,
string? SpeakerName = null, string? SpeakerName = null,
string? TranscriptLineText = null) string? TranscriptLineText = null,
string? AttendeeName = null)
{ {
public static MeetingWorkflowEvent Created(MeetingSessionArtifacts artifacts) public static MeetingWorkflowEvent Created(MeetingSessionArtifacts artifacts)
{ {
@@ -49,4 +51,14 @@ public sealed record MeetingWorkflowEvent(
SpeakerName: speakerName, SpeakerName: speakerName,
TranscriptLineText: line); TranscriptLineText: line);
} }
public static MeetingWorkflowEvent AttendeeAdded(
MeetingSessionArtifacts artifacts,
string attendeeName)
{
return new MeetingWorkflowEvent(
MeetingWorkflowEventType.AttendeeAdded,
artifacts,
AttendeeName: attendeeName);
}
} }
@@ -37,6 +37,9 @@ public sealed class MeetingWorkflowTrigger
[YamlMember(Alias = "transcript_line")] [YamlMember(Alias = "transcript_line")]
public MeetingWorkflowTranscriptLineTrigger? TranscriptLine { get; set; } public MeetingWorkflowTranscriptLineTrigger? TranscriptLine { get; set; }
[YamlMember(Alias = "attendee_added")]
public MeetingWorkflowAttendeeAddedTrigger? AttendeeAdded { get; set; }
} }
public sealed class MeetingWorkflowStateTransitionTrigger public sealed class MeetingWorkflowStateTransitionTrigger
@@ -60,6 +63,18 @@ public sealed class MeetingWorkflowTranscriptLineTrigger
public string? Speaker { get; set; } public string? Speaker { get; set; }
} }
public sealed class MeetingWorkflowAttendeeAddedTrigger
{
[YamlMember(Alias = "equals")]
public string? EqualsValue { get; set; }
[YamlMember(Alias = "contains")]
public string? Contains { get; set; }
[YamlMember(Alias = "regex")]
public string? Regex { get; set; }
}
public sealed class MeetingWorkflowCondition public sealed class MeetingWorkflowCondition
{ {
[YamlMember(Alias = "condition")] [YamlMember(Alias = "condition")]
@@ -94,7 +109,8 @@ public sealed record MeetingWorkflowTemplateModel(
MeetingWorkflowMeetingModel Meeting, MeetingWorkflowMeetingModel Meeting,
MeetingWorkflowEventModel Event, MeetingWorkflowEventModel Event,
MeetingWorkflowSpeakerModel? Speaker, MeetingWorkflowSpeakerModel? Speaker,
MeetingWorkflowTranscriptModel? Transcript); MeetingWorkflowTranscriptModel? Transcript,
MeetingWorkflowAttendeeModel? Attendee);
public sealed record MeetingWorkflowMeetingModel( public sealed record MeetingWorkflowMeetingModel(
string Title, string Title,
@@ -111,6 +127,8 @@ public sealed record MeetingWorkflowSpeakerModel(string Name);
public sealed record MeetingWorkflowTranscriptModel(string Line, string Speaker); public sealed record MeetingWorkflowTranscriptModel(string Line, string Speaker);
public sealed record MeetingWorkflowAttendeeModel(string Name);
internal static class MeetingWorkflowStateNames internal static class MeetingWorkflowStateNames
{ {
public static string ToRuleName(AssistantContextState state) public static string ToRuleName(AssistantContextState state)
@@ -12,13 +12,15 @@ internal static class MeetingWorkflowRuleSchema
public const string PropertyMeetingTitle = "meeting.title"; public const string PropertyMeetingTitle = "meeting.title";
public const string PropertyTranscriptLine = "transcript.line"; public const string PropertyTranscriptLine = "transcript.line";
public const string ParameterTranscriptSpeaker = "transcript.speaker"; public const string ParameterTranscriptSpeaker = "transcript.speaker";
public const string PropertyAttendeeName = "attendee.name";
public static readonly string[] SupportedTriggerKeys = public static readonly string[] SupportedTriggerKeys =
[ [
"created", "created",
"state_transition", "state_transition",
"speaker_identified", "speaker_identified",
"transcript_line" "transcript_line",
"attendee_added"
]; ];
public static readonly string[] SupportedConditionKeys = public static readonly string[] SupportedConditionKeys =
@@ -42,7 +44,8 @@ internal static class MeetingWorkflowRuleSchema
[ [
PropertyTitle, PropertyTitle,
PropertyMeetingTitle, PropertyMeetingTitle,
PropertyTranscriptLine PropertyTranscriptLine,
PropertyAttendeeName
]; ];
public static bool IsStep(string? value, string step) public static bool IsStep(string? value, string step)
@@ -59,7 +59,8 @@ internal static class MeetingWorkflowRulesValidator
trigger.Created is not null, trigger.Created is not null,
trigger.StateTransition is not null, trigger.StateTransition is not null,
trigger.SpeakerIdentified is not null, trigger.SpeakerIdentified is not null,
trigger.TranscriptLine is not null); trigger.TranscriptLine is not null,
trigger.AttendeeAdded is not null);
if (configuredCount == 0) if (configuredCount == 0)
{ {
errors.Add($"{ruleName} on[{triggerIndex + 1}] must contain one supported trigger: {JoinAllowed(MeetingWorkflowRuleSchema.SupportedTriggerKeys)}."); errors.Add($"{ruleName} on[{triggerIndex + 1}] must contain one supported trigger: {JoinAllowed(MeetingWorkflowRuleSchema.SupportedTriggerKeys)}.");
@@ -137,6 +138,12 @@ internal static class MeetingWorkflowRulesValidator
return; return;
} }
if (HasAttendeeAddedTrigger(rule) &&
!IsAttendeeNameSetPropertyStep(step))
{
errors.Add($"{stepPath} on attendee_added rules can only transform the attendee with 'set_property' property 'attendee.name'.");
}
if (MeetingWorkflowRuleSchema.IsStep(uses, MeetingWorkflowRuleSchema.StepSetProperty)) if (MeetingWorkflowRuleSchema.IsStep(uses, MeetingWorkflowRuleSchema.StepSetProperty))
{ {
var property = step.Property ?? step.Name; var property = step.Property ?? step.Name;
@@ -153,6 +160,11 @@ internal static class MeetingWorkflowRulesValidator
{ {
errors.Add($"{stepPath} can set 'transcript.line' only on rules with a transcript_line trigger."); errors.Add($"{stepPath} can set 'transcript.line' only on rules with a transcript_line trigger.");
} }
else if (MeetingWorkflowRuleSchema.IsProperty(property, MeetingWorkflowRuleSchema.PropertyAttendeeName) &&
!HasAttendeeAddedTrigger(rule))
{
errors.Add($"{stepPath} can set 'attendee.name' only on rules with an attendee_added trigger.");
}
} }
if (step.Value is { } value && if (step.Value is { } value &&
@@ -181,6 +193,17 @@ internal static class MeetingWorkflowRulesValidator
return rule.On.Any(static trigger => trigger.TranscriptLine is not null); return rule.On.Any(static trigger => trigger.TranscriptLine is not null);
} }
private static bool HasAttendeeAddedTrigger(MeetingWorkflowRule rule)
{
return rule.On.Any(static trigger => trigger.AttendeeAdded is not null);
}
private static bool IsAttendeeNameSetPropertyStep(MeetingWorkflowStep step)
{
return MeetingWorkflowRuleSchema.IsStep(step.Uses, MeetingWorkflowRuleSchema.StepSetProperty) &&
MeetingWorkflowRuleSchema.IsProperty(step.Property ?? step.Name, MeetingWorkflowRuleSchema.PropertyAttendeeName);
}
private static int CountConfigured(params bool[] values) private static int CountConfigured(params bool[] values)
{ {
return values.Count(static value => value); return values.Count(static value => value);
@@ -206,7 +229,8 @@ internal static class MeetingWorkflowRulesValidator
new MeetingWorkflowSpeakerModel("Ada Lovelace"), new MeetingWorkflowSpeakerModel("Ada Lovelace"),
new MeetingWorkflowTranscriptModel( new MeetingWorkflowTranscriptModel(
"[00:00:01] Ada Lovelace: Validation line.", "[00:00:01] Ada Lovelace: Validation line.",
"Ada Lovelace")); "Ada Lovelace"),
new MeetingWorkflowAttendeeModel("Ada Lovelace"));
} }
private static string FormatRazorValidationMessage(Exception exception) private static string FormatRazorValidationMessage(Exception exception)
@@ -10,9 +10,66 @@ public sealed record WorkflowRulesEditorChatMessage(
WorkflowRulesEditorChatRole Role, WorkflowRulesEditorChatRole Role,
string Content); string Content);
public sealed record WorkflowRulesEditorChatResult( public enum WorkflowRulesEditorConversationItemKind
string Response, {
IReadOnlyList<WorkflowRulesEditorChatMessage> Conversation); Message,
Activity
}
public sealed record WorkflowRulesEditorConversationItem(
WorkflowRulesEditorConversationItemKind Kind,
WorkflowRulesEditorChatMessage? Message,
string Content,
IReadOnlyList<string>? ActivityLines = null)
{
public bool IsExpanded { get; set; }
public static WorkflowRulesEditorConversationItem ChatMessage(WorkflowRulesEditorChatMessage message)
{
return new WorkflowRulesEditorConversationItem(
WorkflowRulesEditorConversationItemKind.Message,
message,
message.Content);
}
public static WorkflowRulesEditorConversationItem Activity(string content, IReadOnlyList<string> activityLines)
{
return new WorkflowRulesEditorConversationItem(
WorkflowRulesEditorConversationItemKind.Activity,
null,
content,
activityLines);
}
}
public sealed record WorkflowRulesEditorChatResult(string Response);
public enum WorkflowRulesEditorActivityKind
{
Status,
ToolCall,
Thinking
}
public sealed record WorkflowRulesEditorActivityUpdate(
WorkflowRulesEditorActivityKind Kind,
string Text)
{
public static WorkflowRulesEditorActivityUpdate Status(string text)
{
return new WorkflowRulesEditorActivityUpdate(WorkflowRulesEditorActivityKind.Status, text);
}
public static WorkflowRulesEditorActivityUpdate ToolCall(string toolName)
{
return new WorkflowRulesEditorActivityUpdate(WorkflowRulesEditorActivityKind.ToolCall, toolName);
}
public static WorkflowRulesEditorActivityUpdate Thinking(string text)
{
return new WorkflowRulesEditorActivityUpdate(WorkflowRulesEditorActivityKind.Thinking, text);
}
}
public interface IWorkflowRulesEditorChatPipeline public interface IWorkflowRulesEditorChatPipeline
{ {
@@ -20,5 +77,5 @@ public interface IWorkflowRulesEditorChatPipeline
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation, IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
string userMessage, string userMessage,
CancellationToken cancellationToken, CancellationToken cancellationToken,
Action<string>? statusChanged = null); Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null);
} }
@@ -59,11 +59,11 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
IReadOnlyList<WorkflowRulesEditorChatMessage> conversation, IReadOnlyList<WorkflowRulesEditorChatMessage> conversation,
string userMessage, string userMessage,
CancellationToken cancellationToken, CancellationToken cancellationToken,
Action<string>? statusChanged = null) Action<WorkflowRulesEditorActivityUpdate>? activityChanged = null)
{ {
if (string.IsNullOrWhiteSpace(userMessage)) if (string.IsNullOrWhiteSpace(userMessage))
{ {
return new WorkflowRulesEditorChatResult("", conversation); return new WorkflowRulesEditorChatResult("");
} }
var agentOptions = options.WorkflowRulesEditor.ToEffectiveAgentOptions(options.Agent); var agentOptions = options.WorkflowRulesEditor.ToEffectiveAgentOptions(options.Agent);
@@ -108,10 +108,18 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
compactionOptions, compactionOptions,
logger, logger,
firstRequestIsUser: true, firstRequestIsUser: true,
retrying: () => statusChanged?.Invoke("Reconnecting...")); retrying: () => activityChanged?.Invoke(WorkflowRulesEditorActivityUpdate.Status("Reconnecting...")),
reasoningSummaryChanged: text => activityChanged?.Invoke(WorkflowRulesEditorActivityUpdate.Thinking(text)));
var functionClient = chatClient var functionClient = chatClient
.AsBuilder() .AsBuilder()
.UseFunctionInvocation(loggerFactory) .UseFunctionInvocation(loggerFactory, client =>
{
client.FunctionInvoker = async (context, token) =>
{
activityChanged?.Invoke(WorkflowRulesEditorActivityUpdate.ToolCall(context.Function.Name));
return await context.Function.InvokeAsync(context.Arguments, token);
};
})
.Build(); .Build();
var response = await functionClient.GetResponseAsync( var response = await functionClient.GetResponseAsync(
@@ -121,11 +129,7 @@ public sealed class WorkflowRulesEditorChatPipeline : IWorkflowRulesEditorChatPi
var responseText = string.IsNullOrWhiteSpace(response.Text) var responseText = string.IsNullOrWhiteSpace(response.Text)
? "(No response text returned.)" ? "(No response text returned.)"
: response.Text.Trim(); : response.Text.Trim();
var nextConversation = conversation return new WorkflowRulesEditorChatResult(responseText);
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, userMessage.Trim()))
.Append(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.Agent, responseText))
.ToList();
return new WorkflowRulesEditorChatResult(responseText, nextConversation);
} }
private static ChatMessage ToChatMessage(WorkflowRulesEditorChatMessage message) private static ChatMessage ToChatMessage(WorkflowRulesEditorChatMessage message)
@@ -1,9 +1,11 @@
using System.Collections.ObjectModel; using System.Collections.ObjectModel;
using System.Diagnostics;
namespace MeetingAssistant.Workflow; namespace MeetingAssistant.Workflow;
public sealed class WorkflowRulesEditorChatViewModel public sealed class WorkflowRulesEditorChatViewModel
{ {
private const string ThinkingPlaceholder = "Thinking...";
private readonly IWorkflowRulesEditorChatPipeline pipeline; private readonly IWorkflowRulesEditorChatPipeline pipeline;
public WorkflowRulesEditorChatViewModel(IWorkflowRulesEditorChatPipeline pipeline) public WorkflowRulesEditorChatViewModel(IWorkflowRulesEditorChatPipeline pipeline)
@@ -11,13 +13,15 @@ public sealed class WorkflowRulesEditorChatViewModel
this.pipeline = pipeline; this.pipeline = pipeline;
} }
public ObservableCollection<WorkflowRulesEditorChatMessage> Messages { get; } = []; public ObservableCollection<WorkflowRulesEditorConversationItem> Messages { get; } = [];
public ObservableCollection<string> ActivityMessages { get; } = [];
public string Draft { get; set; } = ""; public string Draft { get; set; } = "";
public bool IsThinking { get; private set; } public bool IsThinking { get; private set; }
public string ActivityMessage { get; private set; } = "Thinking..."; public string ActivityMessage { get; private set; } = ThinkingPlaceholder;
public event EventHandler? Changed; public event EventHandler? Changed;
@@ -30,10 +34,19 @@ public sealed class WorkflowRulesEditorChatViewModel
} }
Draft = ""; Draft = "";
var priorConversation = Messages.ToList(); var startedAt = Stopwatch.GetTimestamp();
Messages.Add(new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, prompt)); var activityLines = new List<string>();
var hasVisibleThinking = false;
var activityContext = SynchronizationContext.Current;
var priorConversation = Messages
.Select(item => item.Message)
.OfType<WorkflowRulesEditorChatMessage>()
.ToList();
Messages.Add(WorkflowRulesEditorConversationItem.ChatMessage(
new WorkflowRulesEditorChatMessage(WorkflowRulesEditorChatRole.User, prompt)));
ActivityMessages.Clear();
IsThinking = true; IsThinking = true;
ActivityMessage = "Thinking..."; ActivityMessage = ThinkingPlaceholder;
OnChanged(); OnChanged();
try try
@@ -42,36 +55,121 @@ public sealed class WorkflowRulesEditorChatViewModel
priorConversation, priorConversation,
prompt, prompt,
cancellationToken, cancellationToken,
SetActivityMessage); update => hasVisibleThinking |= ApplyActivityUpdate(activityContext, update, activityLines));
Messages.Clear(); AddCompletedActivity(startedAt, activityLines, hasVisibleThinking);
foreach (var message in result.Conversation) Messages.Add(WorkflowRulesEditorConversationItem.ChatMessage(
{ new WorkflowRulesEditorChatMessage(
Messages.Add(message); WorkflowRulesEditorChatRole.Agent,
} string.IsNullOrWhiteSpace(result.Response)
? "(No response text returned.)"
: result.Response.Trim())));
} }
catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested) catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{ {
Messages.Add(new WorkflowRulesEditorChatMessage( AddCompletedActivity(startedAt, activityLines, hasVisibleThinking);
Messages.Add(WorkflowRulesEditorConversationItem.ChatMessage(
new WorkflowRulesEditorChatMessage(
WorkflowRulesEditorChatRole.Agent, WorkflowRulesEditorChatRole.Agent,
$"Settings and logs failed: {exception.Message}")); $"Meeting Summary Agent failed: {exception.Message}")));
} }
finally finally
{ {
IsThinking = false; IsThinking = false;
ActivityMessage = "Thinking..."; ActivityMessages.Clear();
ActivityMessage = ThinkingPlaceholder;
OnChanged(); OnChanged();
} }
} }
private void SetActivityMessage(string message) private bool ApplyActivityUpdate(
SynchronizationContext? context,
WorkflowRulesEditorActivityUpdate update,
List<string> activityLines)
{ {
if (!IsThinking || string.IsNullOrWhiteSpace(message)) if (context is null || SynchronizationContext.Current == context)
{ {
return; return ApplyActivityUpdate(update, activityLines);
}
var hasVisibleThinking = false;
Exception? exception = null;
context.Send(
_ =>
{
try
{
hasVisibleThinking = ApplyActivityUpdate(update, activityLines);
}
catch (Exception caught)
{
exception = caught;
}
},
null);
if (exception is not null)
{
throw exception;
}
return hasVisibleThinking;
}
private bool ApplyActivityUpdate(
WorkflowRulesEditorActivityUpdate update,
List<string> activityLines)
{
if (!IsThinking || string.IsNullOrWhiteSpace(update.Text))
{
return false;
}
var hasVisibleThinking = false;
if (update.Kind == WorkflowRulesEditorActivityKind.ToolCall)
{
var line = $"Called tool: {update.Text.Trim()}";
ActivityMessages.Add(line);
activityLines.Add(line);
}
else if (update.Kind == WorkflowRulesEditorActivityKind.Thinking)
{
var line = update.Text.Trim();
ActivityMessages.Add(line);
activityLines.Add(line);
hasVisibleThinking = true;
}
else
{
var line = update.Text.Trim();
ActivityMessage = line;
activityLines.Add(line);
} }
ActivityMessage = message;
OnChanged(); OnChanged();
return hasVisibleThinking;
}
private void AddCompletedActivity(
long startedAt,
IReadOnlyList<string> activityLines,
bool hasVisibleThinking)
{
var completedActivityLines = hasVisibleThinking
? activityLines.ToArray()
: new[] { ThinkingPlaceholder }.Concat(activityLines).ToArray();
Messages.Add(WorkflowRulesEditorConversationItem.Activity(
$"Worked for {FormatDuration(Stopwatch.GetElapsedTime(startedAt))}",
completedActivityLines));
}
private static string FormatDuration(TimeSpan duration)
{
if (duration.TotalMinutes >= 1)
{
return $"{(int)duration.TotalMinutes}m {duration.Seconds}s";
}
return $"{Math.Max(0, (int)Math.Round(duration.TotalSeconds))}s";
} }
private void OnChanged() private void OnChanged()
@@ -11,7 +11,7 @@ namespace MeetingAssistant.Workflow;
internal sealed class WpfWorkflowRulesEditorWindowService : IWorkflowRulesEditorWindowService internal sealed class WpfWorkflowRulesEditorWindowService : IWorkflowRulesEditorWindowService
{ {
internal const string WindowTitle = "Settings and logs"; internal const string WindowTitle = "Meeting Summary Agent";
private readonly IServiceProvider services; private readonly IServiceProvider services;
private readonly WorkflowRulesEditorMarkdownLinkResolver linkResolver; private readonly WorkflowRulesEditorMarkdownLinkResolver linkResolver;
@@ -288,19 +288,21 @@ internal sealed class WorkflowRulesEditorWpfWindow : Window
conversationPanel.ActualHeight); conversationPanel.ActualHeight);
conversationPanel.Children.Clear(); conversationPanel.Children.Clear();
foreach (var message in viewModel.Messages) foreach (var item in viewModel.Messages)
{ {
conversationPanel.Children.Add(CreateMessageCard(message)); conversationPanel.Children.Add(item.Kind == WorkflowRulesEditorConversationItemKind.Activity
? CreateActivityExpander(item)
: CreateMessageCard(item.Message!));
} }
if (viewModel.IsThinking) if (viewModel.IsThinking)
{ {
conversationPanel.Children.Add(new TextBlock foreach (var activity in viewModel.ActivityMessages)
{ {
Text = viewModel.ActivityMessage, conversationPanel.Children.Add(CreateActivityLine(activity));
Foreground = MutedText, }
Margin = new Thickness(4, 2, 4, 2)
}); conversationPanel.Children.Add(CreateActivityLine(viewModel.ActivityMessage));
} }
sendButton.IsEnabled = !viewModel.IsThinking; sendButton.IsEnabled = !viewModel.IsThinking;
@@ -338,6 +340,43 @@ internal sealed class WorkflowRulesEditorWpfWindow : Window
return card; return card;
} }
private static Expander CreateActivityExpander(WorkflowRulesEditorConversationItem item)
{
var details = new StackPanel
{
Orientation = Orientation.Vertical,
Margin = new Thickness(16, 2, 4, 6)
};
foreach (var line in item.ActivityLines ?? [])
{
details.Children.Add(CreateActivityLine(line));
}
var expander = new Expander
{
Header = item.Content,
Content = details,
IsExpanded = item.IsExpanded,
Foreground = MutedText,
Background = Brushes.Transparent,
Margin = new Thickness(4, 0, 4, 8)
};
expander.Expanded += (_, _) => item.IsExpanded = true;
expander.Collapsed += (_, _) => item.IsExpanded = false;
return expander;
}
private static TextBlock CreateActivityLine(string text)
{
return new TextBlock
{
Text = text,
Foreground = MutedText,
Margin = new Thickness(4, 2, 4, 2)
};
}
private static Style CreateSendButtonStyle() private static Style CreateSendButtonStyle()
{ {
var style = new Style(typeof(Button)); var style = new Style(typeof(Button));
+4 -4
View File
@@ -108,7 +108,7 @@ Important settings:
- `CalendarRecordingPrompts`: enables Outlook Classic Teams-start prompts on Windows. - `CalendarRecordingPrompts`: enables Outlook Classic Teams-start prompts on Windows.
- `Screenshots`: controls the capture hotkey, attachment folder, and optional OCR/vision model. - `Screenshots`: controls the capture hotkey, attachment folder, and optional OCR/vision model.
- `Agent`: configures the OpenAI-compatible summary/project agent endpoint, model, retries, output limits, and compaction. - `Agent`: configures the OpenAI-compatible summary/project agent endpoint, model, retries, output limits, and compaction.
- `WorkflowRulesEditor`: optionally overrides the agent settings for the tray-launched settings/logs assistant. - `WorkflowRulesEditor`: optionally overrides the agent settings for the tray-launched assistant.
Required or commonly used secrets: Required or commonly used secrets:
@@ -125,14 +125,14 @@ See `docs/meeting-assistant-configuration.md` for the full configuration referen
- **Azure AI Speech**: default checked-in ASR path, live diarized conversation transcription, and speaker identity matching. - **Azure AI Speech**: default checked-in ASR path, live diarized conversation transcription, and speaker identity matching.
- **FunASR**: optional WebSocket streaming ASR. When managed backend startup is enabled, the app pulls and runs the configured Docker image as `meeting-assistant-funasr` on port `10095`. - **FunASR**: optional WebSocket streaming ASR. When managed backend startup is enabled, the app pulls and runs the configured Docker image as `meeting-assistant-funasr` on port `10095`.
- **Whisper.NET plus pyannote**: optional local Whisper fallback and Docker-backed final diarization. - **Whisper.NET plus pyannote**: optional local Whisper fallback and Docker-backed final diarization.
- **LiteLLM/OpenAI-compatible Responses endpoint**: summary generation, screenshot OCR fallback, project tools, settings/logs assistant, and retry flows. - **LiteLLM/OpenAI-compatible Responses endpoint**: summary generation, screenshot OCR fallback, project tools, the tray-launched assistant, and retry flows.
- **Docker Desktop or compatible Docker CLI**: required only for managed FunASR and pyannote paths. - **Docker Desktop or compatible Docker CLI**: required only for managed FunASR and pyannote paths.
## Workflow Rules And Agents ## Workflow Rules And Agents
Meeting-specific automation lives in a local YAML file, not in committed personal rules. Rules can trigger on meeting creation, assistant-context state transitions, identified speakers, and transcript-line writes. They can add/remove attendees, set supported properties, add context, add projects, and rewrite a transcript line before persistence. Meeting-specific automation lives in a local YAML file, not in committed personal rules. Rules can trigger on meeting creation, assistant-context state transitions, identified speakers, transcript-line writes, and added attendees. They can add/remove attendees, set supported properties, add context, add projects, rewrite a transcript line, and transform an attendee name before it is stored.
The tray menu exposes the settings/logs assistant. It can edit workflow rules with validation, inspect logs and health/status, manage speaker identities and samples, run ASR diagnostics, and read/write scoped meeting/project artifacts through explicit tools. The tray menu exposes `Open agent`, which opens the `Meeting Summary Agent` window. It can edit workflow rules with validation, inspect logs and health/status, manage speaker identities and samples, run ASR diagnostics, and read/write scoped meeting/project artifacts through explicit tools.
Detailed workflow syntax and extension guidance live in `docs/meeting-workflow-engine.md`. Detailed workflow syntax and extension guidance live in `docs/meeting-workflow-engine.md`.
+2 -2
View File
@@ -358,7 +358,7 @@ The summary agent can add and remove meeting-note attendees when transcript or O
## Workflow Rules Editor ## Workflow Rules Editor
`WorkflowRulesEditor` configures the tray-launched settings/logs assistant. Blank values inherit from `Agent`, so it uses the summarizer endpoint, key, model, reasoning, retry, output, and compaction settings unless explicitly overridden. `WorkflowRulesEditor` configures the tray-launched `Meeting Summary Agent` window. Blank values inherit from `Agent`, so it uses the summarizer endpoint, key, model, reasoning, retry, output, and compaction settings unless explicitly overridden.
The overridable fields are `Endpoint`, `Key`, `KeyEnv`, `Model`, `EnableThinking`, `ReasoningEffort`, `ReconnectionAttempts`, `ReconnectionDelay`, `ContextWindowTokens`, `MaxOutputTokens`, `EnableCompaction`, `CompactionRemainingRatio`, `ResponsesCompactPath`, and `InitialPrompt`. The overridable fields are `Endpoint`, `Key`, `KeyEnv`, `Model`, `EnableThinking`, `ReasoningEffort`, `ReconnectionAttempts`, `ReconnectionDelay`, `ContextWindowTokens`, `MaxOutputTokens`, `EnableCompaction`, `CompactionRemainingRatio`, `ResponsesCompactPath`, and `InitialPrompt`.
@@ -378,7 +378,7 @@ Independently from stdout and stderr redirection, Meeting Assistant writes an ap
%TEMP%\MeetingAssistant\Logs\meeting-assistant.log %TEMP%\MeetingAssistant\Logs\meeting-assistant.log
``` ```
On startup it rotates the previous current file to `meeting-assistant.log.1` and keeps up to `.4`. The settings/logs assistant reads and searches these current and rotated files. If another app instance or test host still has the file open, rotation is skipped and logging appends to the current file so startup is not blocked. On startup it rotates the previous current file to `meeting-assistant.log.1` and keeps up to `.4`. The tray-launched assistant reads and searches these current and rotated files. If another app instance or test host still has the file open, rotation is skipped and logging appends to the current file so startup is not blocked.
## API ## API
+53 -2
View File
@@ -22,7 +22,7 @@ The rules file path is configured through `MeetingAssistant:Automation:RulesPath
If the configured path is empty, missing, or points to a blank file, the workflow engine does nothing. If the configured path is empty, missing, or points to a blank file, the workflow engine does nothing.
The tray menu includes `Settings and logs`, which opens a small MewUI chat assistant for this configured rules file, the local speaker identity database, appsettings configuration, and application logs. The assistant uses the summarizer agent configuration by default and can be overridden through `MeetingAssistant:WorkflowRulesEditor`. The tray menu includes `Open agent`, which opens the `Meeting Summary Agent` chat window for this configured rules file, the local speaker identity database, appsettings configuration, and application logs. The assistant uses the summarizer agent configuration by default and can be overridden through `MeetingAssistant:WorkflowRulesEditor`.
```json ```json
{ {
@@ -45,6 +45,8 @@ The tray menu includes `Settings and logs`, which opens a small MewUI chat assis
Blank editor values inherit from `MeetingAssistant:Agent`. This keeps the editor on the same LiteLLM Responses endpoint and model as the summarizer unless a value is explicitly configured. Each user-submitted editor turn sends its first model request with `X-Initiator: user`; follow-up model requests in the same turn, such as tool-call continuations, use `X-Initiator: agent`. Blank editor values inherit from `MeetingAssistant:Agent`. This keeps the editor on the same LiteLLM Responses endpoint and model as the summarizer unless a value is explicitly configured. Each user-submitted editor turn sends its first model request with `X-Initiator: user`; follow-up model requests in the same turn, such as tool-call continuations, use `X-Initiator: agent`.
While the agent is working, the window shows visible reasoning summaries returned by the Responses endpoint and called tool names as lightweight activity lines above the bottom activity line. After the turn completes, those lines move into a collapsible `Worked for <duration>` expander between the user and assistant messages.
The editor agent prompt includes this document and is restricted to these tools: The editor agent prompt includes this document and is restricted to these tools:
- `read_rules`: read the configured rules file, optionally by inclusive 1-based line range. - `read_rules`: read the configured rules file, optionally by inclusive 1-based line range.
@@ -69,6 +71,12 @@ The engine runs when `MeetingRecordingCoordinator` emits a meeting workflow even
- `state_transition`: after the assistant context state moves forward. - `state_transition`: after the assistant context state moves forward.
- `speaker_identified`: when live or final speaker identification reports a display name. - `speaker_identified`: when live or final speaker identification reports a display name.
- `transcript_line`: after a formatted live transcript line is durably appended, before any changed line is rewritten in place, and before lines are used in full transcript rewrites. - `transcript_line`: after a formatted live transcript line is durably appended, before any changed line is rewritten in place, and before lines are used in full transcript rewrites.
- `attendee_added`: before a newly added attendee is stored in the meeting note.
`attendee_added` is emitted only by Meeting Assistant code paths that intentionally add an attendee
through the workflow engine, notification actions, OCR/screenshot processing, summarizer tools, or
other attendee-add operations. Direct edits to a meeting note file, including direct artifact/frontmatter
repair writes by an agent, are left verbatim and do not trigger attendee automation.
When a recording is started by accepting a calendar notification, the cached appointment metadata is applied before the `created` event. Rules still receive the normal `created` event and the normal `collecting metadata` to `transcribing` state transition, but they see the accepted appointment title, attendees, agenda, and scheduled end instead of a generated placeholder or a separate Outlook current-meeting lookup result. When a recording is started by accepting a calendar notification, the cached appointment metadata is applied before the `created` event. Rules still receive the normal `created` event and the normal `collecting metadata` to `transcribing` state transition, but they see the accepted appointment title, attendees, agenda, and scheduled end instead of a generated placeholder or a separate Outlook current-meeting lookup result.
@@ -160,6 +168,22 @@ on:
speaker: Guest-1 speaker: Guest-1
``` ```
### `attendee_added`
Runs before a newly added attendee is stored. `equals`, `contains`, and `regex` filters are optional; omitted filters match any attendee. `equals` and `contains` are case-insensitive.
```yaml
on:
- attendee_added:
contains: Contoso
```
```yaml
on:
- attendee_added:
regex: '@contoso\.com>$'
```
## Conditions ## Conditions
Conditions use NCalc expressions. Dotted workflow variables may be written directly; the engine rewrites them into NCalc parameters internally. Conditions use NCalc expressions. Dotted workflow variables may be written directly; the engine rewrites them into NCalc parameters internally.
@@ -182,6 +206,7 @@ Available condition variables:
- `speaker.name` - `speaker.name`
- `transcript.line` - `transcript.line`
- `transcript.speaker` - `transcript.speaker`
- `attendee.name`
Available helper functions: Available helper functions:
@@ -215,6 +240,7 @@ Step `value` fields can be plain strings or Razor templates. Razor templates rec
- `Model.Speaker.Name` - `Model.Speaker.Name`
- `Model.Transcript.Line` - `Model.Transcript.Line`
- `Model.Transcript.Speaker` - `Model.Transcript.Speaker`
- `Model.Attendee.Name`
Example: Example:
@@ -257,7 +283,11 @@ steps:
### `set_property` ### `set_property`
Sets a supported property. Supported meeting properties are `title` and `meeting.title`. During `transcript_line` events, `transcript.line` is also supported. For live transcription, changing `transcript.line` rewrites the referenced formatted line after the workflow task completes. Sets a supported property. Supported meeting properties are `title` and `meeting.title`. During `transcript_line` events, `transcript.line` is also supported. During `attendee_added` events, `attendee.name` is supported. For live transcription, changing `transcript.line` rewrites the referenced formatted line after the workflow task completes. For attendee additions, changing `attendee.name` stores the transformed attendee instead of the original added value.
Rules triggered by `attendee_added` are transformation-only: their steps must use `set_property`
with `property: attendee.name`. They can still use trigger filters, conditions, and Razor templates
to decide and compute the transformed value, but they cannot perform note/context side effects.
```yaml ```yaml
steps: steps:
@@ -273,6 +303,13 @@ steps:
value: '@Model.Transcript.Line.Replace("*****", "[redacted]")' value: '@Model.Transcript.Line.Replace("*****", "[redacted]")'
``` ```
```yaml
steps:
- uses: set_property
property: attendee.name
value: '@Model.Attendee.Name.Replace(" (Contoso)", "")'
```
### `add_context` ### `add_context`
Appends rendered text to the assistant context artifact body. This step does not count as a meeting-note mutation and does not cause a meeting-note save by itself. Appends rendered text to the assistant context artifact body. This step does not count as a meeting-note mutation and does not cause a meeting-note save by itself.
@@ -368,6 +405,20 @@ rules:
value: '@Model.Transcript.Line.Replace("*****", "[redacted]")' value: '@Model.Transcript.Line.Replace("*****", "[redacted]")'
``` ```
Clean attendee names as they are added:
```yaml
rules:
- name: clean-contoso-attendees
on:
- attendee_added:
contains: Contoso
steps:
- uses: set_property
property: attendee.name
value: '@Model.Attendee.Name.Replace(" (Contoso)", "")'
```
## Implementation Notes ## Implementation Notes
Core files: Core files:
@@ -6,4 +6,4 @@
- [x] Add workflow rule completion/error logging. - [x] Add workflow rule completion/error logging.
- [x] Update workflow engine documentation. - [x] Update workflow engine documentation.
- [x] Run focused tests and `openspec validate resilient-transcript-workflow --strict`. - [x] Run focused tests and `openspec validate resilient-transcript-workflow --strict`.
- [ ] Follow up later: investigate anomalous temporary-recording disk-full `IOException` despite small expected WAV size. - [x] Follow up later: investigate anomalous temporary-recording disk-full `IOException` despite small expected WAV size. Closed for now because the anomaly has not recurred.
@@ -0,0 +1,2 @@
schema: spec-driven
created: 2026-07-01
@@ -0,0 +1,30 @@
# Design
## Event Model
Add a workflow event type, `attendee_added`, that carries the attendee string being added. The event is used as a transformation hook before a caller stores the attendee in the meeting note. Rules can match the event with `equals`, `contains`, or `regex` trigger filters over the current attendee value.
## Transformation Contract
The workflow engine exposes `TransformAttendeeAsync`, mirroring `TransformTranscriptLineAsync`. During `attendee_added` events:
- conditions can read `attendee.name`,
- Razor templates can read `Model.Attendee.Name`,
- `set_property attendee.name` mutates the attendee value returned to the caller.
Other workflow steps keep their existing semantics, but the intended transformation path is `set_property attendee.name`.
Rules triggered by `attendee_added` are limited to that transformation path so the value transform does not hide unrelated note or context side effects.
Direct meeting note file edits remain outside the workflow event model. If a user or agent writes a meeting note file/frontmatter directly, Meeting Assistant preserves that content verbatim and does not emit `attendee_added`.
## Call Sites
Callers transform attendee candidates before writing them:
- metadata and accepted prompt metadata after canonicalization,
- workflow `add_attendee` steps before de-duplication,
- screenshot OCR candidates before canonicalization and merge,
- summary-agent `add_attendee` tool calls before duplicate checks and note writes,
- speaker identity attendee additions before adding matched names.
Unchanged metadata attendees preserve the existing canonicalizer output, including email display strings.
@@ -0,0 +1,17 @@
## Why
Attendees enter Meeting Assistant from several paths: calendar metadata, accepted recording prompts, workflow actions, screenshot OCR, speaker identity updates, and summarizer add-attendee tools. Today those paths can normalize display names, but they cannot apply local transformation rules such as trimming company suffixes, replacing aliases, or cleaning attendee strings consistently.
## What Changes
- Add an `attendee_added` workflow trigger that runs whenever Meeting Assistant adds an attendee to a meeting note.
- Allow `attendee_added` triggers to filter the added attendee with full equality, substring, or regex matching.
- Expose `attendee.name` to workflow conditions and Razor templates.
- Allow `set_property` to update `attendee.name` during `attendee_added` events, mirroring transcript-line transformation.
- Apply attendee transformations to attendees added by metadata import, prompted-start metadata, workflow `add_attendee`, screenshot OCR, and summary/workflow editor agent tools.
## Impact
- Workflow engine, rules schema validation, and workflow documentation.
- Attendee write paths in recording metadata, screenshot OCR, summary tools, and workflow actions.
- Behavior tests for workflow transformation and representative attendee-add sources.
@@ -0,0 +1,29 @@
## MODIFIED Requirements
### Requirement: Meeting automation rules support lifecycle triggers
Meeting Assistant SHALL support rule triggers for `created`, `state_transition`, `speaker_identified`, `transcript_line`, and `attendee_added`.
An `attendee_added` trigger MAY filter by `equals`, `contains`, or `regex` against the attendee value being added. `equals` and `contains` filters SHALL match case-insensitively.
Meeting Assistant SHALL apply attendee-added transformations before storing attendees added from meeting metadata, accepted recording-prompt metadata, workflow `add_attendee` steps, screenshot OCR, speaker identity updates, and summarizer add-attendee tools.
#### Scenario: Attendee added rule filters and transforms a metadata attendee
- **GIVEN** a configured rule that triggers on added attendees containing `@contoso.com`
- **AND** the rule sets `attendee.name` to a display name derived from the added attendee
- **WHEN** Meeting Assistant adds attendee `Ada Lovelace <ada@contoso.com>` from meeting metadata
- **THEN** the stored meeting attendee is the transformed attendee name
- **AND** the original attendee string is not stored
### Requirement: Meeting automation rules support conditions and steps
Meeting Assistant SHALL expose the added attendee name to `attendee_added` rule conditions and Razor step templates as `attendee.name`.
The `set_property` step SHALL support setting `attendee.name` during `attendee_added` events.
Rules with an `attendee_added` trigger SHALL be limited to transforming `attendee.name` with `set_property`.
Direct meeting note file writes SHALL NOT emit `attendee_added` or transform attendee values.
#### Scenario: Attendee added rules can use regex and Razor
- **GIVEN** a configured `attendee_added` rule with a regex filter over the added attendee
- **WHEN** Meeting Assistant adds an attendee matching the regex
- **THEN** the rule can set `attendee.name` with a Razor template using `Model.Attendee.Name`
@@ -0,0 +1,8 @@
# Tasks
- [x] Add OpenSpec scenarios for attendee-added trigger filtering and transformation.
- [x] Add a failing workflow-engine behavior test for transforming an added attendee through `attendee_added`.
- [x] Implement attendee-added workflow event, trigger filters, `attendee.name` condition/template value, and `set_property attendee.name`.
- [x] Apply attendee transformations from metadata/prompted starts, workflow `add_attendee`, screenshot OCR, and agent attendee tools.
- [x] Update workflow engine documentation.
- [x] Run focused tests plus `openspec validate transform-added-attendees --strict`.
+31 -76
View File
@@ -221,95 +221,43 @@ Meeting Assistant SHALL expose a diagnostic endpoint that reloads application co
- **AND** future workflow events use the reloaded workflow automation configuration - **AND** future workflow events use the reloaded workflow automation configuration
### Requirement: Meeting automation rules support lifecycle triggers ### Requirement: Meeting automation rules support lifecycle triggers
Meeting Assistant SHALL support rule triggers for `created`, `state_transition`, `speaker_identified`, and `transcript_line`. Meeting Assistant SHALL support rule triggers for `created`, `state_transition`, `speaker_identified`, `transcript_line`, and `attendee_added`.
A `state_transition` trigger MAY filter by `from`, `to`, or both state values. An `attendee_added` trigger MAY filter by `equals`, `contains`, or `regex` against the attendee value being added. `equals` and `contains` filters SHALL match case-insensitively.
A `speaker_identified` trigger MAY filter by speaker name. Meeting Assistant SHALL apply attendee-added transformations before storing attendees added from meeting metadata, accepted recording-prompt metadata, workflow `add_attendee` steps, screenshot OCR, speaker identity updates, and summarizer add-attendee tools.
A `transcript_line` trigger MAY filter by speaker name. #### Scenario: Attendee added rule filters and transforms a metadata attendee
- **GIVEN** a configured rule that triggers on added attendees containing `@contoso.com`
#### Scenario: State transition rule matches from and to - **AND** the rule sets `attendee.name` to a display name derived from the added attendee
- **GIVEN** a configured rule that triggers on a state transition from `collecting metadata` to `transcribing` - **WHEN** Meeting Assistant adds attendee `Ada Lovelace <ada@contoso.com>` from meeting metadata
- **WHEN** Meeting Assistant transitions that meeting from `collecting metadata` to `transcribing` - **THEN** the stored meeting attendee is the transformed attendee name
- **THEN** it applies the rule steps - **AND** the original attendee string is not stored
#### Scenario: Speaker identified rule filters by name
- **GIVEN** a configured rule that triggers when speaker `Ada` is identified
- **WHEN** Meeting Assistant identifies speaker `Grace`
- **THEN** it does not apply the rule
- **WHEN** Meeting Assistant identifies speaker `Ada`
- **THEN** it applies the rule steps
#### Scenario: Transcript line rule rewrites masked profanity before persistence
- **GIVEN** a configured rule that triggers on transcript line writes and sets `transcript.line` by replacing `*****` with `[redacted]`
- **WHEN** Meeting Assistant writes a transcript line for speaker `Guest-1` containing `*****`
- **THEN** the written transcript line contains `[redacted]`
- **AND** the written transcript line does not contain `*****`
### Requirement: Meeting automation rules support conditions and steps ### Requirement: Meeting automation rules support conditions and steps
Meeting Assistant SHALL support rule conditions using an expression engine. Meeting Assistant SHALL expose the added attendee name to `attendee_added` rule conditions and Razor step templates as `attendee.name`.
Rules SHALL support nested `and`, `or`, and `not` condition groups. The `set_property` step SHALL support setting `attendee.name` during `attendee_added` events.
Step values SHALL support Razor syntax against the current meeting event model. Rules with an `attendee_added` trigger SHALL be limited to transforming `attendee.name` with `set_property`.
Step values SHALL treat `@` characters inside valid email address tokens as literal text rather than Razor transitions. Direct meeting note file writes SHALL NOT emit `attendee_added` or transform attendee values.
Meeting Assistant SHALL expose the formatted transcript line and transcript speaker to `transcript_line` rule conditions and Razor step templates. #### Scenario: Attendee added rules can use regex and Razor
- **GIVEN** a configured `attendee_added` rule with a regex filter over the added attendee
Meeting Assistant SHALL support these initial rule steps: - **WHEN** Meeting Assistant adds an attendee matching the regex
- **THEN** the rule can set `attendee.name` with a Razor template using `Model.Attendee.Name`
- `add_attendee`
- `remove_attendee`
- `set_property`
- `add_context`
- `add_project`
The `set_property` step SHALL support setting `transcript.line` during `transcript_line` events.
#### Scenario: Nested conditions choose a matching rule
- **GIVEN** a configured rule with nested `and`, `or`, and `not` conditions over meeting title, attendees, and event data
- **WHEN** the condition evaluates to true
- **THEN** Meeting Assistant applies the rule
- **WHEN** the condition evaluates to false
- **THEN** Meeting Assistant skips the rule
#### Scenario: Templated context mentions identified speaker
- **GIVEN** a configured `speaker_identified` rule with an `add_context` step using Razor syntax
- **WHEN** Meeting Assistant identifies matching speaker `Ada`
- **THEN** it appends rendered context text containing `Ada` to the assistant context note
#### Scenario: Email addresses do not trigger Razor templating
- **GIVEN** a configured rule step value containing `Support@contoso.com`
- **WHEN** the rule runs
- **THEN** Meeting Assistant preserves the email address as literal text
#### Scenario: Email addresses can appear beside Razor templating
- **GIVEN** a configured rule step value containing both `Support@contoso.com` and a Razor expression
- **WHEN** the rule runs
- **THEN** Meeting Assistant renders the Razor expression and preserves the email address as literal text
#### Scenario: Rule can clean a meeting title
- **GIVEN** a configured state-transition rule that matches a title containing a configured marker
- **WHEN** the rule runs
- **THEN** Meeting Assistant can update the meeting title through `set_property`
#### Scenario: Transcript line conditions can use the written line and speaker
- **GIVEN** a configured `transcript_line` rule with conditions over `transcript.line` and `transcript.speaker`
- **WHEN** Meeting Assistant writes a transcript line that matches both conditions
- **THEN** it applies the rule steps before the line is persisted
### Requirement: Workflow rules and speaker identities can be edited through a tray-launched assistant ### Requirement: Workflow rules and speaker identities can be edited through a tray-launched assistant
Meeting Assistant SHALL expose an `Edit rules and identities` item from the tray icon menu. Meeting Assistant SHALL expose an `Open agent` item from the tray icon menu.
The tray icon SHALL be implemented through the Uno notification icon stack. The tray icon SHALL be implemented through the Uno notification icon stack.
The tray icon menu SHALL show every configured launch profile when recording can be started and SHALL include each profile's configured toggle hotkey in the corresponding start or switch menu item. The tray icon menu SHALL show every configured launch profile when recording can be started and SHALL include each profile's configured toggle hotkey in the corresponding start or switch menu item.
When the user selects `Edit rules and identities`, Meeting Assistant SHALL open a basic MewUI chat window for editing the configured workflow rules file and speaker identities. When the user selects `Open agent`, Meeting Assistant SHALL open a basic MewUI chat window for editing the configured workflow rules file and speaker identities.
The chat window SHALL be titled `Edit rules and identities`, SHALL display user and assistant messages as visually distinct cards, SHALL display basic markdown emphasis, inline code, fenced code blocks, pipe tables rendered as grid views, and line breaks in agent responses, SHALL display a plain `Thinking...` line while the agent is working, MAY replace that line with `Reconnecting...` while retrying a transient agent request, SHALL provide a multiline text input at the bottom with placeholder text for asking to make a rule or list identities, SHALL send on Enter, SHALL insert a newline on Shift+Enter, and SHALL provide an explicit Send button. The chat window SHALL be titled `Meeting Summary Agent`, SHALL display user and assistant messages as visually distinct cards, SHALL display basic markdown emphasis, inline code, fenced code blocks, pipe tables rendered as grid views, and line breaks in agent responses, SHALL display lightweight visible reasoning summary lines and tool-call lines that include only the called tool name while the agent is working, SHALL display a plain `Thinking...` line as the bottom activity line while the agent is working, MAY replace that bottom line with `Reconnecting...` while retrying a transient agent request, SHALL replace per-turn activity after completion with a collapsible `Worked for <duration>` expander between the user and assistant cards, SHALL show visible reasoning summary lines, tool-call lines, and status lines in order inside the expander when expanded, SHALL collapse those details again when the expander is closed, SHALL provide a multiline text input at the bottom with placeholder text for asking to make a rule or list identities, SHALL send on Enter, SHALL insert a newline on Shift+Enter, and SHALL provide an explicit Send button.
When a new chat message or thinking state is appended, the chat window SHALL scroll to the bottom of the newly rendered content if the conversation was already near the bottom or did not need scrolling before the append. When a new chat message or thinking state is appended, the chat window SHALL scroll to the bottom of the newly rendered content if the conversation was already near the bottom or did not need scrolling before the append.
@@ -335,8 +283,8 @@ Meeting Assistant SHALL provide a diagnostic endpoint that opens the workflow ru
#### Scenario: Tray menu opens the editor #### Scenario: Tray menu opens the editor
- **WHEN** the user opens the tray icon menu - **WHEN** the user opens the tray icon menu
- **THEN** the menu includes `Edit rules and identities` - **THEN** the menu includes `Open agent`
- **WHEN** the user selects `Edit rules and identities` - **WHEN** the user selects `Open agent`
- **THEN** Meeting Assistant opens the workflow rules and identities editor chat window - **THEN** Meeting Assistant opens the workflow rules and identities editor chat window
#### Scenario: Tray menu shows configured profile hotkeys #### Scenario: Tray menu shows configured profile hotkeys
@@ -385,9 +333,16 @@ Meeting Assistant SHALL provide a diagnostic endpoint that opens the workflow ru
- **GIVEN** the rules editor chat window is open - **GIVEN** the rules editor chat window is open
- **WHEN** the user types a prompt and presses Enter - **WHEN** the user types a prompt and presses Enter
- **THEN** the user message appears in the conversation - **THEN** the user message appears in the conversation
- **AND** a plain `Thinking...` line appears while the agent is running - **AND** a plain `Thinking...` line appears at the bottom while the agent is running
- **WHEN** the agent emits visible reasoning summaries or calls tools
- **THEN** lightweight activity lines appear above the bottom activity line with the visible reasoning summary text or only the called tool names
- **WHEN** the agent turn completes
- **THEN** the activity is represented between the user and assistant cards as a collapsed `Worked for <duration>` expander
- **AND** expanding it shows the visible reasoning summary, status, and tool-call lines in order
- **AND** collapsing it hides those details again
- **AND** if no visible reasoning summary was emitted, the expander includes the fallback `Thinking...` line
- **AND** the first model request for that turn is marked as user-initiated - **AND** the first model request for that turn is marked as user-initiated
- **AND** the final assistant response replaces the thinking line - **AND** the final assistant response replaces the live activity lines
#### Scenario: Rules editor displays agent request failures #### Scenario: Rules editor displays agent request failures
- **GIVEN** the rules editor chat window is open - **GIVEN** the rules editor chat window is open