Strengthen workflow automation diagnostics

This commit is contained in:
2026-05-27 12:55:19 +02:00
parent b114071957
commit 2422236ef7
8 changed files with 629 additions and 6 deletions
@@ -0,0 +1,62 @@
using System.Net;
using System.Net.Http.Json;
using System.Text.Json;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.Extensions.Configuration;
namespace MeetingAssistant.Tests;
public sealed class MeetingWorkflowDiagnosticEndpointTests
{
[Fact]
public async Task ReloadEndpointReloadsAutomationRulesPathFromConfigurationFile()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(root);
var configPath = Path.Combine(root, "workflow-config.json");
var firstRulesPath = Path.Combine(root, "first-rules.yaml");
var secondRulesPath = Path.Combine(root, "second-rules.yaml");
await File.WriteAllTextAsync(configPath, CreateConfigJson(firstRulesPath));
await using var factory = new WebApplicationFactory<Program>()
.WithWebHostBuilder(builder =>
{
builder.ConfigureAppConfiguration((_, configuration) =>
{
configuration.AddJsonFile(configPath, optional: false, reloadOnChange: false);
configuration.AddInMemoryCollection(new Dictionary<string, string?>
{
["MeetingAssistant:FunAsr:Backend:Enabled"] = "false"
});
});
});
using var client = factory.CreateClient();
using var firstResponse = await client.PostAsync("/diagnostics/workflow/reload", null);
var firstBody = await firstResponse.Content.ReadFromJsonAsync<WorkflowReloadResponse>();
await File.WriteAllTextAsync(configPath, CreateConfigJson(secondRulesPath));
using var secondResponse = await client.PostAsync("/diagnostics/workflow/reload", null);
var secondBody = await secondResponse.Content.ReadFromJsonAsync<WorkflowReloadResponse>();
Assert.Equal(HttpStatusCode.OK, firstResponse.StatusCode);
Assert.Equal(firstRulesPath, firstBody?.RulesPath);
Assert.Equal(HttpStatusCode.OK, secondResponse.StatusCode);
Assert.Equal(secondRulesPath, secondBody?.RulesPath);
}
private static string CreateConfigJson(string rulesPath)
{
return JsonSerializer.Serialize(new
{
MeetingAssistant = new
{
Automation = new
{
RulesPath = rulesPath
}
}
});
}
private sealed record WorkflowReloadResponse(string? RulesPath);
}
@@ -137,6 +137,466 @@ public sealed class MeetingWorkflowEngineTests
Assert.Equal(["Ada"], meeting.Frontmatter.Attendees);
}
[Theory]
[InlineData("created: {}", "created", null, null, null, true)]
[InlineData("created: {}", "speaker", null, null, "Ada", false)]
[InlineData("state_transition:\n from: collecting metadata", "state", "collecting metadata", "speaker recognition", null, true)]
[InlineData("state_transition:\n to: transcribing", "state", "collecting metadata", "transcribing", null, true)]
[InlineData("state_transition:\n from: collecting metadata\n to: transcribing", "state", "collecting metadata", "transcribing", null, true)]
[InlineData("state_transition:\n from: transcribing\n to: summarizing", "state", "collecting metadata", "transcribing", null, false)]
[InlineData("state_transition:\n from: COLLECTING METADATA\n to: TRANSCRIBING", "state", "collecting metadata", "transcribing", null, true)]
[InlineData("speaker_identified:\n name: Ada", "speaker", null, null, "Ada", true)]
[InlineData("speaker_identified:\n name: ADA", "speaker", null, null, "ada", true)]
[InlineData("speaker_identified:\n name: Ada", "speaker", null, null, "Grace", false)]
[InlineData("speaker_identified: {}", "speaker", null, null, "Grace", true)]
public async Task TriggerMatchingScenarios(
string triggerYaml,
string eventKind,
string? from,
string? to,
string? speaker,
bool shouldRun)
{
var fixture = await WorkflowFixture.CreateAsync(
$$"""
rules:
- name: trigger-scenario
on:
- {{triggerYaml}}
steps:
- uses: add_project
value: Triggered
""");
await fixture.Engine.RunAsync(
CreateEvent(eventKind, fixture.Artifacts, from, to, speaker),
fixture.Options,
CancellationToken.None);
var meeting = await fixture.ReadMeetingAsync();
Assert.Equal(shouldRun, meeting.Frontmatter.Projects.Contains("Triggered"));
}
[Theory]
[InlineData("event.type = 'SpeakerIdentified'", "speaker", null, null, "Ada", true)]
[InlineData("state.from = 'collecting metadata' and state.to = 'transcribing'", "state", "collecting metadata", "transcribing", null, true)]
[InlineData("meeting.state = 'transcribing'", "state", "collecting metadata", "transcribing", null, true)]
[InlineData("speaker.name = 'Ada'", "speaker", null, null, "Ada", true)]
[InlineData("speaker.name = 'Ada'", "speaker", null, null, "Grace", false)]
[InlineData("starts_with(meeting.title, 'planning')", "created", null, null, null, true)]
[InlineData("ends_with(meeting.title, 'sync')", "created", null, null, null, true)]
[InlineData("contains(meeting.title, 'ANNING S')", "created", null, null, null, true)]
[InlineData("contains(meeting.attendees, 'Ada')", "created", null, null, null, true)]
[InlineData("contains(meeting.projects, 'Architecture')", "created", null, null, null, true)]
[InlineData("meeting.attendees.count >= 2", "created", null, null, null, true)]
public async Task ExpressionConditionParameterScenarios(
string condition,
string eventKind,
string? from,
string? to,
string? speaker,
bool shouldRun)
{
var fixture = await WorkflowFixture.CreateAsync(
$$"""
rules:
- name: condition-scenario
on:
- created: {}
- state_transition: {}
- speaker_identified: {}
if:
- condition: {{condition}}
steps:
- uses: add_project
value: ConditionMatched
""",
title: "Planning Sync",
attendees: ["Ada", "Grace"],
projects: ["Architecture"]);
await fixture.Engine.RunAsync(
CreateEvent(eventKind, fixture.Artifacts, from, to, speaker),
fixture.Options,
CancellationToken.None);
var meeting = await fixture.ReadMeetingAsync();
Assert.Equal(shouldRun, meeting.Frontmatter.Projects.Contains("ConditionMatched"));
}
[Fact]
public async Task TopLevelConditionsAreCombinedWithAnd()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: all-top-level-conditions
on:
- created: {}
if:
- condition: contains(meeting.title, 'Planning')
- condition: contains(meeting.attendees, 'Ada')
- condition: contains(meeting.projects, 'Missing')
steps:
- uses: add_project
value: ShouldNotRun
""",
attendees: ["Ada"],
projects: ["Architecture"]);
await RunCreatedAsync(fixture);
var meeting = await fixture.ReadMeetingAsync();
Assert.DoesNotContain("ShouldNotRun", meeting.Frontmatter.Projects);
}
[Fact]
public async Task OrGroupSkipsRuleWhenAllBranchesAreFalse()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: false-or
on:
- created: {}
if:
- or:
- condition: contains(meeting.title, 'Budget')
- condition: contains(meeting.attendees, 'Grace')
steps:
- uses: add_project
value: ShouldNotRun
""",
attendees: ["Ada"]);
await RunCreatedAsync(fixture);
var meeting = await fixture.ReadMeetingAsync();
Assert.DoesNotContain("ShouldNotRun", meeting.Frontmatter.Projects);
}
[Fact]
public async Task EmptyConditionListRunsWhenTriggerMatches()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: no-if
on:
- created: {}
steps:
- uses: add_project
value: NoConditionNeeded
""");
await RunCreatedAsync(fixture);
var meeting = await fixture.ReadMeetingAsync();
Assert.Contains("NoConditionNeeded", meeting.Frontmatter.Projects);
}
[Fact]
public async Task RuleWithMultipleTriggersRunsWhenAnyTriggerMatches()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: multi-trigger
on:
- created: {}
- speaker_identified:
name: Ada
steps:
- uses: add_project
value: MultiTrigger
""");
await fixture.Engine.RunAsync(
MeetingWorkflowEvent.SpeakerIdentified(fixture.Artifacts, "Ada"),
fixture.Options,
CancellationToken.None);
var meeting = await fixture.ReadMeetingAsync();
Assert.Contains("MultiTrigger", meeting.Frontmatter.Projects);
}
[Fact]
public async Task MultipleRulesRunInFileOrderAndLaterRulesSeeEarlierMutations()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: first
on:
- created: {}
steps:
- uses: add_attendee
value: Ada
- name: second
on:
- created: {}
if:
- condition: contains(meeting.attendees, 'Ada')
steps:
- uses: add_project
value: Architecture
""");
await RunCreatedAsync(fixture);
var meeting = await fixture.ReadMeetingAsync();
Assert.Equal(["Ada"], meeting.Frontmatter.Attendees);
Assert.Equal(["Architecture"], meeting.Frontmatter.Projects);
}
[Fact]
public async Task LaterStepsInSameRuleSeeEarlierStepMutationsInRazorModel()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: step-order
on:
- created: {}
steps:
- uses: add_attendee
value: Ada
- uses: add_context
value: 'Attendees: @Model.Meeting.Attendees.Count'
""");
await RunCreatedAsync(fixture);
var context = await fixture.ReadContextAsync();
Assert.Contains("Attendees: 1", context);
}
[Fact]
public async Task SpeakerIdentifiedTemplateCanUseRecognizedSpeakerName()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: speaker-template
on:
- speaker_identified: {}
steps:
- uses: add_context
value: '@Model.Speaker.Name joined @Model.Meeting.Title'
""",
title: "Design Review");
await fixture.Engine.RunAsync(
MeetingWorkflowEvent.SpeakerIdentified(fixture.Artifacts, "Manuel"),
fixture.Options,
CancellationToken.None);
var context = await fixture.ReadContextAsync();
Assert.Contains("Manuel joined Design Review", context);
}
[Fact]
public async Task StateTransitionTemplateCanUseFromAndToStates()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: transition-template
on:
- state_transition:
to: transcribing
steps:
- uses: add_context
value: '@Model.Event.From -> @Model.Event.To'
""");
await fixture.Engine.RunAsync(
MeetingWorkflowEvent.StateTransition(
fixture.Artifacts,
AssistantContextState.CollectingMetadata,
AssistantContextState.Transcribing),
fixture.Options,
CancellationToken.None);
var context = await fixture.ReadContextAsync();
Assert.Contains("collecting metadata -> transcribing", context);
}
[Fact]
public async Task AddAttendeeNormalizesDisplayNameFromEmailAddress()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: normalize-attendee
on:
- created: {}
steps:
- uses: add_attendee
value: 'Ada Lovelace <ada@example.com>'
""");
await RunCreatedAsync(fixture);
var meeting = await fixture.ReadMeetingAsync();
Assert.Equal(["Ada Lovelace"], meeting.Frontmatter.Attendees);
}
[Fact]
public async Task AddAttendeeDoesNotDuplicateExistingDisplayNameFromEmailAddress()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: dedupe-attendee
on:
- created: {}
steps:
- uses: add_attendee
value: Ada
""",
attendees: ["Ada <ada@example.com>"]);
await RunCreatedAsync(fixture);
var meeting = await fixture.ReadMeetingAsync();
Assert.Equal(["Ada <ada@example.com>"], meeting.Frontmatter.Attendees);
}
[Fact]
public async Task RemoveAttendeeMatchesExistingDisplayNameFromEmailAddress()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: remove-email-attendee
on:
- created: {}
steps:
- uses: remove_attendee
value: Ada
""",
attendees: ["Ada <ada@example.com>", "Grace"]);
await RunCreatedAsync(fixture);
var meeting = await fixture.ReadMeetingAsync();
Assert.Equal(["Grace"], meeting.Frontmatter.Attendees);
}
[Fact]
public async Task AddProjectDoesNotDuplicateCaseInsensitiveProject()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: dedupe-project
on:
- created: {}
steps:
- uses: add_project
value: architecture
""",
projects: ["Architecture"]);
await RunCreatedAsync(fixture);
var meeting = await fixture.ReadMeetingAsync();
Assert.Equal(["Architecture"], meeting.Frontmatter.Projects);
}
[Fact]
public async Task SetPropertyCanUseNameAliasForTitle()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: title-by-name
on:
- created: {}
steps:
- uses: set_property
name: meeting.title
value: Updated Title
""");
await RunCreatedAsync(fixture);
var meeting = await fixture.ReadMeetingAsync();
Assert.Equal("Updated Title", meeting.Frontmatter.Title);
}
[Fact]
public async Task AddContextDoesNotRequireMeetingNoteMutationToPersist()
{
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: context-only
on:
- created: {}
steps:
- uses: add_context
value: Context-only note.
""");
await RunCreatedAsync(fixture);
var context = await fixture.ReadContextAsync();
Assert.Contains("Context-only note.", context);
}
[Fact]
public async Task RulesFileChangesAreReadForTheNextWorkflowEvent()
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
var rulesPath = Path.Combine(root, "rules.yaml");
Directory.CreateDirectory(root);
var fixture = await WorkflowFixture.CreateAsync(
"""
rules:
- name: first-version
on:
- created: {}
steps:
- uses: add_project
value: First
""",
rulesPath: rulesPath);
await RunCreatedAsync(fixture);
await File.WriteAllTextAsync(
rulesPath,
"""
rules:
- name: second-version
on:
- speaker_identified: {}
steps:
- uses: add_project
value: Second
""");
await fixture.Engine.RunAsync(
MeetingWorkflowEvent.SpeakerIdentified(fixture.Artifacts, "Ada"),
fixture.Options,
CancellationToken.None);
var meeting = await fixture.ReadMeetingAsync();
Assert.Equal(["First", "Second"], meeting.Frontmatter.Projects);
}
[Fact]
public async Task BlankRulesFileDoesNothing()
{
var fixture = await WorkflowFixture.CreateAsync(" ");
await RunCreatedAsync(fixture);
var meeting = await fixture.ReadMeetingAsync();
Assert.Empty(meeting.Frontmatter.Attendees);
Assert.Empty(meeting.Frontmatter.Projects);
}
[Fact]
public async Task MissingRulesFileDoesNothing()
@@ -152,6 +612,47 @@ public sealed class MeetingWorkflowEngineTests
Assert.Empty(meeting.Frontmatter.Attendees);
}
private static Task RunCreatedAsync(WorkflowFixture fixture)
{
return fixture.Engine.RunAsync(
MeetingWorkflowEvent.Created(fixture.Artifacts),
fixture.Options,
CancellationToken.None);
}
private static MeetingWorkflowEvent CreateEvent(
string eventKind,
MeetingSessionArtifacts artifacts,
string? from,
string? to,
string? speaker)
{
return eventKind switch
{
"created" => MeetingWorkflowEvent.Created(artifacts),
"speaker" => MeetingWorkflowEvent.SpeakerIdentified(artifacts, speaker ?? "Ada"),
"state" => MeetingWorkflowEvent.StateTransition(
artifacts,
ParseState(from ?? "collecting metadata"),
ParseState(to ?? "transcribing")),
_ => throw new InvalidOperationException($"Unknown workflow event kind '{eventKind}'.")
};
}
private static AssistantContextState ParseState(string state)
{
return state.Trim().ToLowerInvariant() switch
{
"collecting metadata" => AssistantContextState.CollectingMetadata,
"transcribing" => AssistantContextState.Transcribing,
"speaker recognition" => AssistantContextState.SpeakerRecognition,
"summarizing" => AssistantContextState.Summarizing,
"finished" => AssistantContextState.Finished,
"error" => AssistantContextState.Error,
_ => throw new InvalidOperationException($"Unknown workflow state '{state}'.")
};
}
private sealed class WorkflowFixture
{
private WorkflowFixture(
@@ -178,6 +679,7 @@ public sealed class MeetingWorkflowEngineTests
string rulesYaml,
string title = "Planning Sync",
IReadOnlyList<string>? attendees = null,
IReadOnlyList<string>? projects = null,
string? rulesPath = null)
{
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
@@ -219,6 +721,7 @@ public sealed class MeetingWorkflowEngineTests
title,
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
attendees: attendees ?? [],
projects: projects ?? [],
transcriptPath: artifacts.TranscriptPath,
assistantContextPath: artifacts.AssistantContextPath,
summaryPath: artifacts.SummaryPath) with
@@ -242,5 +745,15 @@ public sealed class MeetingWorkflowEngineTests
NullLogger<MeetingWorkflowEngine>.Instance);
return new WorkflowFixture(options, noteStore, artifacts, engine);
}
public Task<MeetingNote> ReadMeetingAsync()
{
return NoteStore.ReadAsync(Artifacts.MeetingNotePath, CancellationToken.None);
}
public Task<string> ReadContextAsync()
{
return File.ReadAllTextAsync(Artifacts.AssistantContextPath);
}
}
}