Public Access
Archive meeting workflow and screenshot changes
This commit is contained in:
@@ -71,6 +71,8 @@ For Meeting Assistant, prefer verification through:
|
||||
- transcription and summarization behavior tests
|
||||
- application logs proving the end-to-end flow
|
||||
|
||||
Before restarting the local Meeting Assistant application, check whether a meeting is currently being recorded, transcribed, finalized, or summarized through the recording status endpoint and/or logs. Do not restart the application while an active meeting run is recording, transcribing, finalizing speaker recognition, waiting on OCR, or summarizing unless the user explicitly tells you to restart anyway.
|
||||
|
||||
If production or end-to-end verification is not possible, state exactly why and what lower-level verification was performed instead.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
@@ -61,6 +61,56 @@ public sealed class LaunchProfileOptionsProviderTests
|
||||
Assert.Contains("Ctrl+Alt+M", exception.Message, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScreenshotHotkeysAreRegisteredForDefaultAndExplicitProfileOverrides()
|
||||
{
|
||||
var provider = CreateProvider(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:Hotkey:Toggle"] = "Ctrl+Alt+M",
|
||||
["MeetingAssistant:Hotkey:Abort"] = "Ctrl+Alt+D",
|
||||
["MeetingAssistant:Screenshots:Hotkey"] = "Ctrl+Alt+S",
|
||||
["MeetingAssistant:LaunchProfiles:english:Hotkey:Toggle"] = "Ctrl+Alt+E",
|
||||
["MeetingAssistant:LaunchProfiles:english:Hotkey:Abort"] = "Ctrl+Alt+Shift+D",
|
||||
["MeetingAssistant:LaunchProfiles:english:Screenshots:Hotkey"] = "Ctrl+Alt+Shift+S"
|
||||
});
|
||||
|
||||
var hotkeys = provider.GetHotkeys();
|
||||
|
||||
Assert.Contains(hotkeys, hotkey =>
|
||||
hotkey.ProfileName == "default" &&
|
||||
hotkey.Hotkey == "Ctrl+Alt+S" &&
|
||||
hotkey.Action == LaunchProfileHotkeyAction.CaptureScreenshot);
|
||||
Assert.Contains(hotkeys, hotkey =>
|
||||
hotkey.ProfileName == "english" &&
|
||||
hotkey.Hotkey == "Ctrl+Alt+Shift+S" &&
|
||||
hotkey.Action == LaunchProfileHotkeyAction.CaptureScreenshot);
|
||||
Assert.Contains(hotkeys, hotkey =>
|
||||
hotkey.ProfileName == "default" &&
|
||||
hotkey.Hotkey == "Ctrl+Alt+D" &&
|
||||
hotkey.Action == LaunchProfileHotkeyAction.AbortRecording);
|
||||
Assert.Contains(hotkeys, hotkey =>
|
||||
hotkey.ProfileName == "english" &&
|
||||
hotkey.Hotkey == "Ctrl+Alt+Shift+D" &&
|
||||
hotkey.Action == LaunchProfileHotkeyAction.AbortRecording);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NamedProfilesDoNotInheritDefaultHotkeysForRegistration()
|
||||
{
|
||||
var provider = CreateProvider(new Dictionary<string, string?>
|
||||
{
|
||||
["MeetingAssistant:Hotkey:Toggle"] = "Ctrl+Alt+M",
|
||||
["MeetingAssistant:Hotkey:Abort"] = "Ctrl+Alt+D",
|
||||
["MeetingAssistant:Screenshots:Hotkey"] = "Ctrl+Alt+S",
|
||||
["MeetingAssistant:LaunchProfiles:english:AzureSpeech:Language"] = "en-US"
|
||||
});
|
||||
|
||||
var hotkeys = provider.GetHotkeys();
|
||||
|
||||
Assert.Equal(3, hotkeys.Count);
|
||||
Assert.DoesNotContain(hotkeys, hotkey => hotkey.ProfileName == "english");
|
||||
}
|
||||
|
||||
private static ConfigurationLaunchProfileOptionsProvider CreateProvider(
|
||||
IReadOnlyDictionary<string, string?> values)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
#pragma warning disable CA1416
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class LiteLlmScreenshotOcrClientTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task ExtractUsesAgentEndpointAndModelWhenOcrEndpointAndModelAreBlank()
|
||||
{
|
||||
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
|
||||
await File.WriteAllBytesAsync(screenshotPath, [1, 2, 3]);
|
||||
var handler = new RecordingHandler("""
|
||||
{
|
||||
"output": [
|
||||
{
|
||||
"content": [
|
||||
{ "type": "output_text", "text": "Visible slide text" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
""");
|
||||
var client = new LiteLlmScreenshotOcrClient(
|
||||
() => handler,
|
||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||
var options = new MeetingAssistantOptions
|
||||
{
|
||||
Agent =
|
||||
{
|
||||
Endpoint = "https://summary.local",
|
||||
Model = "summary-model",
|
||||
Key = "agent-key"
|
||||
},
|
||||
Screenshots =
|
||||
{
|
||||
Ocr =
|
||||
{
|
||||
Endpoint = "",
|
||||
Model = "",
|
||||
Key = "ocr-key"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var result = await client.ExtractAsync(
|
||||
screenshotPath,
|
||||
"Extract screenshot.",
|
||||
options,
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Visible slide text", result.Text);
|
||||
Assert.Null(result.Crop);
|
||||
Assert.Equal(new Uri("https://summary.local/v1/responses"), handler.RequestUri);
|
||||
Assert.Equal("Bearer", handler.Authorization?.Scheme);
|
||||
Assert.Equal("ocr-key", handler.Authorization?.Parameter);
|
||||
using var payload = JsonDocument.Parse(handler.RequestBody!);
|
||||
Assert.Equal("summary-model", payload.RootElement.GetProperty("model").GetString());
|
||||
var content = payload.RootElement
|
||||
.GetProperty("input")[0]
|
||||
.GetProperty("content");
|
||||
Assert.Equal("Extract screenshot.", content[0].GetProperty("text").GetString());
|
||||
Assert.Equal("data:image/png;base64,AQID", content[1].GetProperty("image_url").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractUsesScreenshotOcrEndpointAndModelWhenConfigured()
|
||||
{
|
||||
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
|
||||
await File.WriteAllBytesAsync(screenshotPath, [4, 5, 6]);
|
||||
var handler = new RecordingHandler("""{ "output_text": "OCR result" }""");
|
||||
var client = new LiteLlmScreenshotOcrClient(
|
||||
() => handler,
|
||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||
var options = new MeetingAssistantOptions
|
||||
{
|
||||
Agent =
|
||||
{
|
||||
Endpoint = "https://summary.local",
|
||||
Model = "summary-model",
|
||||
Key = "agent-key"
|
||||
},
|
||||
Screenshots =
|
||||
{
|
||||
Ocr =
|
||||
{
|
||||
Endpoint = "https://vision.local/v1",
|
||||
Model = "vision-model",
|
||||
Key = "ocr-key"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var result = await client.ExtractAsync(
|
||||
screenshotPath,
|
||||
"Extract screenshot.",
|
||||
options,
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("OCR result", result.Text);
|
||||
Assert.Equal(new Uri("https://vision.local/v1/responses"), handler.RequestUri);
|
||||
using var payload = JsonDocument.Parse(handler.RequestBody!);
|
||||
Assert.Equal("vision-model", payload.RootElement.GetProperty("model").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExtractParsesCropMetadataAndOmitsMetadataFromReturnedText()
|
||||
{
|
||||
var screenshotPath = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N") + ".png");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
|
||||
await File.WriteAllBytesAsync(screenshotPath, CreatePngBytes(8, 6));
|
||||
var handler = new RecordingHandler("""
|
||||
{
|
||||
"output_text": "Slide text\n\n```json\n{ \"crop\": { \"x\": 1, \"y\": 2, \"width\": 3, \"height\": 4 } }\n```"
|
||||
}
|
||||
""");
|
||||
var client = new LiteLlmScreenshotOcrClient(
|
||||
() => handler,
|
||||
NullLogger<LiteLlmScreenshotOcrClient>.Instance);
|
||||
var options = new MeetingAssistantOptions
|
||||
{
|
||||
Agent =
|
||||
{
|
||||
Key = "agent-key"
|
||||
}
|
||||
};
|
||||
|
||||
var result = await client.ExtractAsync(
|
||||
screenshotPath,
|
||||
"Extract screenshot.",
|
||||
options,
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("Slide text", result.Text);
|
||||
Assert.Equal(new ScreenshotCropCoordinates(1, 2, 3, 4), result.Crop);
|
||||
using var payload = JsonDocument.Parse(handler.RequestBody!);
|
||||
Assert.Contains(
|
||||
"Original screenshot dimensions: 8x6 pixels.",
|
||||
payload.RootElement.GetProperty("input")[0].GetProperty("content")[0].GetProperty("text").GetString(),
|
||||
StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private sealed class RecordingHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly string responseBody;
|
||||
|
||||
public RecordingHandler(string responseBody)
|
||||
{
|
||||
this.responseBody = responseBody;
|
||||
}
|
||||
|
||||
public Uri? RequestUri { get; private set; }
|
||||
|
||||
public AuthenticationHeaderValue? Authorization { get; private set; }
|
||||
|
||||
public string? RequestBody { get; private set; }
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
RequestUri = request.RequestUri;
|
||||
Authorization = request.Headers.Authorization;
|
||||
RequestBody = request.Content is null
|
||||
? null
|
||||
: await request.Content.ReadAsStringAsync(cancellationToken);
|
||||
return new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(responseBody, Encoding.UTF8, "application/json")
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] CreatePngBytes(int width, int height)
|
||||
{
|
||||
using var bitmap = new Bitmap(width, height);
|
||||
using (var graphics = Graphics.FromImage(bitmap))
|
||||
{
|
||||
graphics.Clear(Color.White);
|
||||
}
|
||||
|
||||
using var stream = new MemoryStream();
|
||||
bitmap.Save(stream, ImageFormat.Png);
|
||||
return stream.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
#pragma warning restore CA1416
|
||||
@@ -0,0 +1,65 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Recording;
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MeetingRunArtifactCleanerTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task DeleteRunArtifactsDeletesGeneratedScreenshotAttachmentsOnly()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var artifacts = new MeetingSessionArtifacts(
|
||||
Path.Combine(root, "Notes", "meeting.md"),
|
||||
Path.Combine(root, "Transcripts", "transcript.md"),
|
||||
Path.Combine(root, "Context", "context.md"),
|
||||
Path.Combine(root, "Summaries", "summary.md"));
|
||||
var attachments = Path.Combine(Path.GetDirectoryName(artifacts.AssistantContextPath)!, "Attachments");
|
||||
Directory.CreateDirectory(attachments);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.MeetingNotePath)!);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.TranscriptPath)!);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.SummaryPath)!);
|
||||
|
||||
var generatedScreenshot = Path.Combine(attachments, "20260527-093135-123-000010.png");
|
||||
var generatedCrop = Path.Combine(attachments, "20260527-093135-123-000010-cropped.png");
|
||||
var manualAttachment = Path.Combine(attachments, "manual-reference.png");
|
||||
var nonImageLinkedFile = Path.Combine(attachments, "20260527-093135-123-000011.png");
|
||||
foreach (var file in new[]
|
||||
{
|
||||
artifacts.MeetingNotePath,
|
||||
artifacts.TranscriptPath,
|
||||
artifacts.SummaryPath,
|
||||
generatedScreenshot,
|
||||
generatedCrop,
|
||||
manualAttachment,
|
||||
nonImageLinkedFile
|
||||
})
|
||||
{
|
||||
await File.WriteAllTextAsync(file, "content");
|
||||
}
|
||||
|
||||
await File.WriteAllTextAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
"""
|
||||
|
||||
|
||||
|
||||
[Reference](Attachments/20260527-093135-123-000011.png)
|
||||
""");
|
||||
var cleaner = new MeetingRunArtifactCleaner();
|
||||
|
||||
await cleaner.DeleteRunArtifactsAsync(
|
||||
artifacts,
|
||||
new MeetingAssistantOptions(),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.False(File.Exists(artifacts.MeetingNotePath));
|
||||
Assert.False(File.Exists(artifacts.TranscriptPath));
|
||||
Assert.False(File.Exists(artifacts.AssistantContextPath));
|
||||
Assert.False(File.Exists(artifacts.SummaryPath));
|
||||
Assert.False(File.Exists(generatedScreenshot));
|
||||
Assert.False(File.Exists(generatedCrop));
|
||||
Assert.True(File.Exists(manualAttachment));
|
||||
Assert.True(File.Exists(nonImageLinkedFile));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
|
||||
#pragma warning disable CA1416
|
||||
|
||||
namespace MeetingAssistant.Tests;
|
||||
|
||||
public sealed class MeetingScreenshotServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task CaptureSavesScreenshotAndAppendsTimestampedLink()
|
||||
{
|
||||
var fixture = await ScreenshotFixture.CreateAsync();
|
||||
var service = fixture.CreateService(
|
||||
new FixedScreenshotCapture([1, 2, 3]),
|
||||
new CapturingScreenshotOcrClient());
|
||||
|
||||
var result = await service.CaptureAsync(
|
||||
fixture.Artifacts,
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
DateTimeOffset.Parse("2026-05-26T10:03:05+02:00"),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.True(File.Exists(result.ScreenshotPath));
|
||||
Assert.Equal([1, 2, 3], await File.ReadAllBytesAsync(result.ScreenshotPath));
|
||||
Assert.StartsWith(
|
||||
Path.Combine(Path.GetDirectoryName(fixture.Artifacts.AssistantContextPath)!, "Attachments"),
|
||||
result.ScreenshotPath,
|
||||
StringComparison.OrdinalIgnoreCase);
|
||||
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
Assert.Contains("## Screenshot [00:03:05]", context);
|
||||
Assert.Contains("![Screenshot 00:03:05](Attachments/", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureSkipsOcrWhenOcrIsDisabled()
|
||||
{
|
||||
var fixture = await ScreenshotFixture.CreateAsync();
|
||||
var ocr = new CapturingScreenshotOcrClient();
|
||||
var service = fixture.CreateService(
|
||||
new FixedScreenshotCapture([1, 2, 3]),
|
||||
ocr);
|
||||
|
||||
await service.CaptureAsync(
|
||||
fixture.Artifacts,
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:05+02:00"),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, ocr.CallCount);
|
||||
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
Assert.DoesNotContain("OCR", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureAppendsOcrResultAfterScreenshotWhenConfigured()
|
||||
{
|
||||
var fixture = await ScreenshotFixture.CreateAsync(options =>
|
||||
{
|
||||
options.Screenshots.Ocr.Enabled = true;
|
||||
options.Screenshots.Ocr.Prompt = "Extract this meeting screenshot.";
|
||||
});
|
||||
var ocr = new CapturingScreenshotOcrClient("Slide text\n\n```mermaid\ngraph TD\nA-->B\n```");
|
||||
var service = fixture.CreateService(
|
||||
new FixedScreenshotCapture([1, 2, 3]),
|
||||
ocr);
|
||||
|
||||
await service.CaptureAsync(
|
||||
fixture.Artifacts,
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, ocr.CallCount);
|
||||
Assert.Equal("Extract this meeting screenshot.", ocr.Prompt);
|
||||
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
var imageIndex = context.IndexOf("![Screenshot 00:00:10]", StringComparison.Ordinal);
|
||||
var ocrIndex = context.IndexOf("### OCR", StringComparison.Ordinal);
|
||||
Assert.True(imageIndex >= 0);
|
||||
Assert.True(ocrIndex > imageIndex);
|
||||
Assert.Contains("Slide text", context);
|
||||
Assert.Contains("```mermaid", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureSavesCroppedImageAndLinksItBeforeOcrWhenOcrReturnsCropCoordinates()
|
||||
{
|
||||
var fixture = await ScreenshotFixture.CreateAsync(options =>
|
||||
{
|
||||
options.Screenshots.Ocr.Enabled = true;
|
||||
});
|
||||
var ocr = new CapturingScreenshotOcrClient(
|
||||
"Shared screen text",
|
||||
new ScreenshotCropCoordinates(1, 1, 2, 2));
|
||||
var service = fixture.CreateService(
|
||||
new FixedScreenshotCapture(CreatePngBytes(4, 4)),
|
||||
ocr);
|
||||
|
||||
await service.CaptureAsync(
|
||||
fixture.Artifacts,
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
await service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
|
||||
|
||||
var croppedPath = Assert.Single(Directory.GetFiles(
|
||||
Path.Combine(Path.GetDirectoryName(fixture.Artifacts.AssistantContextPath)!, "Attachments"),
|
||||
"*-cropped.png"));
|
||||
using (var cropped = new Bitmap(croppedPath))
|
||||
{
|
||||
Assert.Equal(2, cropped.Width);
|
||||
Assert.Equal(2, cropped.Height);
|
||||
}
|
||||
|
||||
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
var screenshotIndex = context.IndexOf("![Screenshot 00:00:10]", StringComparison.Ordinal);
|
||||
var cropIndex = context.IndexOf("![Cropped screenshot](Attachments/", StringComparison.Ordinal);
|
||||
var ocrIndex = context.IndexOf("### OCR", StringComparison.Ordinal);
|
||||
Assert.True(screenshotIndex >= 0);
|
||||
Assert.True(cropIndex > screenshotIndex);
|
||||
Assert.True(ocrIndex > cropIndex);
|
||||
Assert.Contains("Shared screen text", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WaitForPendingOcrWaitsForRunningScreenshotOcr()
|
||||
{
|
||||
var fixture = await ScreenshotFixture.CreateAsync(options =>
|
||||
{
|
||||
options.Screenshots.Ocr.Enabled = true;
|
||||
});
|
||||
var ocr = new BlockingScreenshotOcrClient();
|
||||
var service = fixture.CreateService(
|
||||
new FixedScreenshotCapture([1, 2, 3]),
|
||||
ocr);
|
||||
|
||||
await service.CaptureAsync(
|
||||
fixture.Artifacts,
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
await ocr.WaitUntilStartedAsync();
|
||||
var wait = service.WaitForPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
|
||||
|
||||
Assert.False(wait.IsCompleted);
|
||||
ocr.Release("OCR complete.");
|
||||
await wait;
|
||||
|
||||
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
Assert.Contains("OCR complete.", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CancelPendingOcrCancelsRunningOcrWithoutWritingFailureText()
|
||||
{
|
||||
var fixture = await ScreenshotFixture.CreateAsync(options =>
|
||||
{
|
||||
options.Screenshots.Ocr.Enabled = true;
|
||||
});
|
||||
var ocr = new BlockingScreenshotOcrClient();
|
||||
var service = fixture.CreateService(
|
||||
new FixedScreenshotCapture([1, 2, 3]),
|
||||
ocr);
|
||||
|
||||
await service.CaptureAsync(
|
||||
fixture.Artifacts,
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:10+02:00"),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
await ocr.WaitUntilStartedAsync();
|
||||
|
||||
await service.CancelPendingOcrAsync(fixture.Artifacts, TimeSpan.FromSeconds(5), CancellationToken.None);
|
||||
|
||||
var context = await File.ReadAllTextAsync(fixture.Artifacts.AssistantContextPath);
|
||||
Assert.Contains("_OCR pending..._", context);
|
||||
Assert.DoesNotContain("OCR timed out", context);
|
||||
Assert.DoesNotContain("OCR failed", context);
|
||||
}
|
||||
|
||||
private sealed class ScreenshotFixture
|
||||
{
|
||||
private ScreenshotFixture(
|
||||
MeetingAssistantOptions options,
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MarkdownMeetingArtifactStore artifactStore)
|
||||
{
|
||||
Options = options;
|
||||
Artifacts = artifacts;
|
||||
ArtifactStore = artifactStore;
|
||||
}
|
||||
|
||||
public MeetingAssistantOptions Options { get; }
|
||||
|
||||
public MeetingSessionArtifacts Artifacts { get; }
|
||||
|
||||
public MarkdownMeetingArtifactStore ArtifactStore { get; }
|
||||
|
||||
public static async Task<ScreenshotFixture> CreateAsync(
|
||||
Action<MeetingAssistantOptions>? configure = null)
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var options = new MeetingAssistantOptions
|
||||
{
|
||||
Vault = new VaultOptions
|
||||
{
|
||||
BaseFolder = root,
|
||||
AssistantContextFolder = "Context"
|
||||
}
|
||||
};
|
||||
configure?.Invoke(options);
|
||||
var artifacts = new MeetingSessionArtifacts(
|
||||
Path.Combine(root, "Notes", "meeting.md"),
|
||||
Path.Combine(root, "Transcripts", "transcript.md"),
|
||||
Path.Combine(root, "Context", "context.md"),
|
||||
Path.Combine(root, "Summaries", "summary.md"));
|
||||
var artifactStore = new MarkdownMeetingArtifactStore(
|
||||
NullLogger<MarkdownMeetingArtifactStore>.Instance);
|
||||
var meeting = MeetingNoteTemplate.Create(
|
||||
"Planning",
|
||||
DateTimeOffset.Parse("2026-05-26T10:00:00+02:00"),
|
||||
transcriptPath: artifacts.TranscriptPath,
|
||||
assistantContextPath: artifacts.AssistantContextPath,
|
||||
summaryPath: artifacts.SummaryPath) with
|
||||
{
|
||||
Path = artifacts.MeetingNotePath
|
||||
};
|
||||
await artifactStore.CreateAssistantContextAsync(
|
||||
artifacts,
|
||||
meeting,
|
||||
"",
|
||||
null,
|
||||
CancellationToken.None);
|
||||
return new ScreenshotFixture(options, artifacts, artifactStore);
|
||||
}
|
||||
|
||||
public MeetingScreenshotService CreateService(
|
||||
IActiveWindowScreenshotCapture capture,
|
||||
IScreenshotOcrClient ocrClient)
|
||||
{
|
||||
return new MeetingScreenshotService(
|
||||
capture,
|
||||
ArtifactStore,
|
||||
ocrClient,
|
||||
NullLogger<MeetingScreenshotService>.Instance);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FixedScreenshotCapture : IActiveWindowScreenshotCapture
|
||||
{
|
||||
private readonly byte[] bytes;
|
||||
|
||||
public FixedScreenshotCapture(byte[] bytes)
|
||||
{
|
||||
this.bytes = bytes;
|
||||
}
|
||||
|
||||
public Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new ActiveWindowScreenshot(bytes, "Demo Window"));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CapturingScreenshotOcrClient : IScreenshotOcrClient
|
||||
{
|
||||
private readonly ScreenshotOcrResult result;
|
||||
|
||||
public CapturingScreenshotOcrClient(
|
||||
string text = "",
|
||||
ScreenshotCropCoordinates? crop = null)
|
||||
{
|
||||
result = new ScreenshotOcrResult(text, crop);
|
||||
}
|
||||
|
||||
public int CallCount { get; private set; }
|
||||
|
||||
public string? Prompt { get; private set; }
|
||||
|
||||
public Task<ScreenshotOcrResult> ExtractAsync(
|
||||
string screenshotPath,
|
||||
string prompt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
CallCount++;
|
||||
Prompt = prompt;
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class BlockingScreenshotOcrClient : IScreenshotOcrClient
|
||||
{
|
||||
private readonly TaskCompletionSource started = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource<ScreenshotOcrResult> result = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public async Task<ScreenshotOcrResult> ExtractAsync(
|
||||
string screenshotPath,
|
||||
string prompt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
started.TrySetResult();
|
||||
await using var registration = cancellationToken.Register(() => result.TrySetCanceled(cancellationToken));
|
||||
return await result.Task;
|
||||
}
|
||||
|
||||
public Task WaitUntilStartedAsync()
|
||||
{
|
||||
return started.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
public void Release(string value)
|
||||
{
|
||||
result.TrySetResult(new ScreenshotOcrResult(value, null));
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] CreatePngBytes(int width, int height)
|
||||
{
|
||||
using var bitmap = new Bitmap(width, height);
|
||||
using (var graphics = Graphics.FromImage(bitmap))
|
||||
{
|
||||
graphics.Clear(Color.White);
|
||||
}
|
||||
|
||||
using var stream = new MemoryStream();
|
||||
bitmap.Save(stream, ImageFormat.Png);
|
||||
return stream.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
#pragma warning restore CA1416
|
||||
@@ -22,6 +22,8 @@ public sealed class MeetingSummaryInstructionBuilderTests
|
||||
var instructions = await builder.BuildAsync(artifacts, CancellationToken.None);
|
||||
|
||||
Assert.Contains("You are the Meeting Assistant summary agent.", instructions);
|
||||
Assert.Contains("include only the most relevant cropped screenshots", instructions);
|
||||
Assert.Contains("Do not include every cropped screenshot", instructions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -191,6 +191,48 @@ public sealed class MeetingSummaryToolTests
|
||||
Assert.Contains("replacement\ninserted\nline two", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ToolsWriteAssistantContextStripsAccidentalFrontmatterFromReplacementBody()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var artifacts = new MeetingSessionArtifacts(
|
||||
MeetingNotePath: Path.Combine(root, "Meetings", "Notes", "meeting.md"),
|
||||
TranscriptPath: Path.Combine(root, "Meetings", "Transcripts", "transcript.md"),
|
||||
AssistantContextPath: Path.Combine(root, "Meetings", "Assistant Context", "context.md"),
|
||||
SummaryPath: Path.Combine(root, "Meetings", "Summaries", "summary.md"));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(artifacts.AssistantContextPath)!);
|
||||
await File.WriteAllTextAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
"""
|
||||
---
|
||||
meeting: "[[../Notes/meeting|Meeting Note]]"
|
||||
transcript: "[[../Transcripts/transcript|Transcript]]"
|
||||
summary: "[[../Summaries/summary|Summary]]"
|
||||
state: summarizing
|
||||
---
|
||||
|
||||
line one
|
||||
""");
|
||||
var tools = new MeetingSummaryTools(artifacts);
|
||||
|
||||
await tools.WriteContext(
|
||||
"""
|
||||
---
|
||||
state: finished
|
||||
---
|
||||
|
||||
# Assistant Context
|
||||
|
||||
## Live Context
|
||||
""");
|
||||
|
||||
var context = await File.ReadAllTextAsync(artifacts.AssistantContextPath);
|
||||
Assert.Contains("state: summarizing", context);
|
||||
Assert.DoesNotContain("state: finished", context);
|
||||
Assert.Equal(2, context.Split("---", StringSplitOptions.None).Length - 1);
|
||||
Assert.Contains("# Assistant Context", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExternalAgentWritesAreAppendedToAssistantContextAsDiffs()
|
||||
{
|
||||
|
||||
@@ -74,6 +74,85 @@ public sealed class MeetingWorkflowEngineTests
|
||||
Assert.Equal(["Architecture"], meeting.Frontmatter.Projects);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StepValuesWithEmailAddressesAreTreatedAsLiteralText()
|
||||
{
|
||||
var fixture = await WorkflowFixture.CreateAsync(
|
||||
"""
|
||||
rules:
|
||||
- name: normalize-sew-sofi-daily
|
||||
on:
|
||||
- state_transition:
|
||||
from: collecting metadata
|
||||
to: transcribing
|
||||
if:
|
||||
- condition: "meeting.title = 'WG: Sofi - Daily - Team Velocity Vanguards'"
|
||||
steps:
|
||||
- uses: set_property
|
||||
property: title
|
||||
value: 'SEW Sofi - Daily'
|
||||
- uses: add_project
|
||||
value: 'SEW Solution Finder'
|
||||
- uses: add_attendee
|
||||
value: 'niklas.link.e@sew-eurodrive.de'
|
||||
""",
|
||||
title: "WG: Sofi - Daily - Team Velocity Vanguards");
|
||||
|
||||
await fixture.Engine.RunAsync(
|
||||
MeetingWorkflowEvent.StateTransition(
|
||||
fixture.Artifacts,
|
||||
AssistantContextState.CollectingMetadata,
|
||||
AssistantContextState.Transcribing),
|
||||
fixture.Options,
|
||||
CancellationToken.None);
|
||||
|
||||
var meeting = await fixture.NoteStore.ReadAsync(fixture.Artifacts.MeetingNotePath, CancellationToken.None);
|
||||
Assert.Equal("SEW Sofi - Daily", meeting.Frontmatter.Title);
|
||||
Assert.Equal(["SEW Solution Finder"], meeting.Frontmatter.Projects);
|
||||
Assert.Equal(["niklas.link.e@sew-eurodrive.de"], meeting.Frontmatter.Attendees);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StepValuesCanMixEmailAddressesAndRazorTemplates()
|
||||
{
|
||||
var fixture = await WorkflowFixture.CreateAsync(
|
||||
"""
|
||||
rules:
|
||||
- name: mixed-template
|
||||
on:
|
||||
- created: {}
|
||||
steps:
|
||||
- uses: add_context
|
||||
value: 'Contact Support@contoso.com about @Model.Meeting.Title.'
|
||||
""",
|
||||
title: "Architecture Sync");
|
||||
|
||||
await RunCreatedAsync(fixture);
|
||||
|
||||
var context = await fixture.ReadContextAsync();
|
||||
Assert.Contains("Contact Support@contoso.com about Architecture Sync.", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StepValuesRenderRazorTransitionsThatAreNotModelMarkers()
|
||||
{
|
||||
var fixture = await WorkflowFixture.CreateAsync(
|
||||
"""
|
||||
rules:
|
||||
- name: razor-transition
|
||||
on:
|
||||
- created: {}
|
||||
steps:
|
||||
- uses: add_context
|
||||
value: 'Generated in @System.DateTime.UtcNow.Year'
|
||||
""");
|
||||
|
||||
await RunCreatedAsync(fixture);
|
||||
|
||||
var context = await fixture.ReadContextAsync();
|
||||
Assert.Contains($"Generated in {DateTime.UtcNow.Year}", context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SpeakerIdentifiedRuleFiltersByNameAndWritesRazorContext()
|
||||
{
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using MeetingAssistant.Transcription;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Summary;
|
||||
@@ -30,6 +31,36 @@ public sealed class RecordingCoordinatorTests
|
||||
throw new TimeoutException("Condition was not met.");
|
||||
}
|
||||
|
||||
private static async Task AppendTextWithRetryAsync(string path, string text)
|
||||
{
|
||||
var deadline = DateTimeOffset.UtcNow.AddSeconds(5);
|
||||
while (true)
|
||||
{
|
||||
try
|
||||
{
|
||||
await File.AppendAllTextAsync(path, text);
|
||||
return;
|
||||
}
|
||||
catch (IOException) when (DateTimeOffset.UtcNow < deadline)
|
||||
{
|
||||
await Task.Delay(25);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool FileContainsText(string path, string text)
|
||||
{
|
||||
try
|
||||
{
|
||||
return File.Exists(path) &&
|
||||
File.ReadAllText(path).Contains(text, StringComparison.Ordinal);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ToggleStartsStreamingTranscriptionAndSecondToggleStopsIt()
|
||||
{
|
||||
@@ -183,6 +214,47 @@ public sealed class RecordingCoordinatorTests
|
||||
await coordinator.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StartSkipsOutlookMeetingAttendeesAboveConfiguredImportLimit()
|
||||
{
|
||||
var audioSource = new ControlledAudioSource();
|
||||
var noteStore = new InMemoryMeetingNoteStore("C:\\Vault\\Meetings\\Notes\\metadata-meeting.md");
|
||||
var artifactStore = new InMemoryMeetingArtifactStore();
|
||||
var coordinator = new MeetingRecordingCoordinator(
|
||||
audioSource,
|
||||
new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()),
|
||||
new InMemoryTranscriptStore(),
|
||||
noteStore,
|
||||
new CapturingMeetingNoteOpener(),
|
||||
artifactStore,
|
||||
new InMemoryRecordedAudioStore(),
|
||||
new CapturingMeetingSummaryPipeline(),
|
||||
Options.Create(new MeetingAssistantOptions
|
||||
{
|
||||
Recording =
|
||||
{
|
||||
MaxMetadataAttendeeImportCount = 2
|
||||
}
|
||||
}),
|
||||
NullLogger<MeetingRecordingCoordinator>.Instance,
|
||||
new FixedMeetingMetadataProvider(new MeetingMetadata(
|
||||
"Presentation Review",
|
||||
["Ada", "Grace", "Linus"],
|
||||
"Large audience agenda",
|
||||
DateTimeOffset.Parse("2026-05-19T14:00:00+02:00"))));
|
||||
|
||||
await coordinator.StartAsync(CancellationToken.None);
|
||||
await WaitUntilAsync(() => artifactStore.Agenda == "Large audience agenda");
|
||||
|
||||
Assert.Equal("Presentation Review", noteStore.SavedNote?.Frontmatter.Title);
|
||||
Assert.Empty(noteStore.SavedNote?.Frontmatter.Attendees ?? []);
|
||||
Assert.Equal("Presentation Review", artifactStore.ContextMeetingNote?.Frontmatter.Title);
|
||||
Assert.Equal("Large audience agenda", artifactStore.Agenda);
|
||||
Assert.Equal(DateTimeOffset.Parse("2026-05-19T14:00:00+02:00"), artifactStore.ScheduledEnd);
|
||||
|
||||
await coordinator.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StartDoesNotWaitForSlowMeetingMetadataButAppliesItLater()
|
||||
{
|
||||
@@ -364,6 +436,94 @@ public sealed class RecordingCoordinatorTests
|
||||
await store.WaitForTextAsync("final:2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AbortStopsRecordingDeletesArtifactsAndSkipsSummary()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "meeting-assistant-tests", Guid.NewGuid().ToString("N"));
|
||||
var options = new MeetingAssistantOptions
|
||||
{
|
||||
Vault = new VaultOptions
|
||||
{
|
||||
BaseFolder = root,
|
||||
TranscriptsFolder = "Transcripts",
|
||||
MeetingNotesFolder = "Notes",
|
||||
AssistantContextFolder = "Context",
|
||||
SummariesFolder = "Summaries"
|
||||
},
|
||||
Recording = new RecordingOptions
|
||||
{
|
||||
StopProcessingTimeout = TimeSpan.FromSeconds(5)
|
||||
}
|
||||
};
|
||||
var audioSource = new ControlledAudioSource();
|
||||
var summaryPipeline = new CapturingMeetingSummaryPipeline();
|
||||
var coordinator = new MeetingRecordingCoordinator(
|
||||
audioSource,
|
||||
new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()),
|
||||
new VaultTranscriptStore(Options.Create(options), NullLogger<VaultTranscriptStore>.Instance),
|
||||
new MarkdownMeetingNoteStore(Options.Create(options), NullLogger<MarkdownMeetingNoteStore>.Instance),
|
||||
new CapturingMeetingNoteOpener(),
|
||||
new MarkdownMeetingArtifactStore(NullLogger<MarkdownMeetingArtifactStore>.Instance),
|
||||
new InMemoryRecordedAudioStore(),
|
||||
summaryPipeline,
|
||||
Options.Create(options),
|
||||
NullLogger<MeetingRecordingCoordinator>.Instance);
|
||||
|
||||
var started = await coordinator.StartAsync(CancellationToken.None);
|
||||
await audioSource.WriteAsync(new AudioChunk([1, 0], 16000, 1), CancellationToken.None);
|
||||
await WaitUntilAsync(() =>
|
||||
FileContainsText(started.AssistantContextPath!, "state: transcribing"));
|
||||
var attachmentPath = Path.Combine(
|
||||
Path.GetDirectoryName(started.AssistantContextPath!)!,
|
||||
"Attachments",
|
||||
"20260527-093135-123-000010-cropped.png");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(attachmentPath)!);
|
||||
await File.WriteAllTextAsync(attachmentPath, "image");
|
||||
await AppendTextWithRetryAsync(
|
||||
started.AssistantContextPath!,
|
||||
Environment.NewLine + "" + Environment.NewLine);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(started.SummaryPath!)!);
|
||||
await File.WriteAllTextAsync(started.SummaryPath!, "draft summary");
|
||||
|
||||
var aborted = await coordinator.AbortAsync(CancellationToken.None);
|
||||
|
||||
Assert.False(aborted.IsRecording);
|
||||
Assert.Null(aborted.MeetingNotePath);
|
||||
Assert.False(File.Exists(started.MeetingNotePath));
|
||||
Assert.False(File.Exists(started.TranscriptPath));
|
||||
Assert.False(File.Exists(started.AssistantContextPath));
|
||||
Assert.False(File.Exists(started.SummaryPath));
|
||||
Assert.False(File.Exists(attachmentPath));
|
||||
Assert.Null(summaryPipeline.Artifacts);
|
||||
|
||||
await Task.Delay(250);
|
||||
Assert.False(File.Exists(started.AssistantContextPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AbortDoesNothingWhenRecordingIsIdle()
|
||||
{
|
||||
var coordinator = new MeetingRecordingCoordinator(
|
||||
new ControlledAudioSource(),
|
||||
new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()),
|
||||
new InMemoryTranscriptStore(),
|
||||
new InMemoryMeetingNoteStore(),
|
||||
new CapturingMeetingNoteOpener(),
|
||||
new InMemoryMeetingArtifactStore(),
|
||||
new InMemoryRecordedAudioStore(),
|
||||
new CapturingMeetingSummaryPipeline(),
|
||||
Options.Create(new MeetingAssistantOptions()),
|
||||
NullLogger<MeetingRecordingCoordinator>.Instance);
|
||||
|
||||
var status = await coordinator.AbortAsync(CancellationToken.None);
|
||||
|
||||
Assert.False(status.IsRecording);
|
||||
Assert.Null(status.MeetingNotePath);
|
||||
Assert.Null(status.TranscriptPath);
|
||||
Assert.Null(status.AssistantContextPath);
|
||||
Assert.Null(status.SummaryPath);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureStartsEvenWhenTranscriptionProviderIsStillWarmingUp()
|
||||
{
|
||||
@@ -950,6 +1110,98 @@ public sealed class RecordingCoordinatorTests
|
||||
artifactStore.States);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureScreenshotUsesCurrentMeetingArtifactsAndStartTime()
|
||||
{
|
||||
var audioSource = new ControlledAudioSource();
|
||||
var screenshotService = new CapturingMeetingScreenshotService();
|
||||
var coordinator = new MeetingRecordingCoordinator(
|
||||
audioSource,
|
||||
new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()),
|
||||
new InMemoryTranscriptStore(),
|
||||
new InMemoryMeetingNoteStore(),
|
||||
new CapturingMeetingNoteOpener(),
|
||||
new InMemoryMeetingArtifactStore(),
|
||||
new InMemoryRecordedAudioStore(),
|
||||
new CapturingMeetingSummaryPipeline(),
|
||||
Options.Create(CreateOptionsWithoutFinalizer()),
|
||||
NullLogger<MeetingRecordingCoordinator>.Instance,
|
||||
screenshotService: screenshotService);
|
||||
|
||||
var started = await coordinator.StartAsync(CancellationToken.None);
|
||||
|
||||
var result = await coordinator.CaptureScreenshotAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(started.AssistantContextPath, screenshotService.Artifacts?.AssistantContextPath);
|
||||
Assert.NotNull(screenshotService.MeetingStartedAt);
|
||||
Assert.Equal("memory-screenshot.png", result.ScreenshotPath);
|
||||
|
||||
await coordinator.StopAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CaptureScreenshotFailsAfterRecordingStopHasStarted()
|
||||
{
|
||||
var audioSource = new ControlledAudioSource();
|
||||
var screenshotService = new CapturingMeetingScreenshotService();
|
||||
var coordinator = new MeetingRecordingCoordinator(
|
||||
audioSource,
|
||||
new TestSpeechRecognitionPipelineFactory(new EchoStreamingTranscriptionProvider()),
|
||||
new InMemoryTranscriptStore(),
|
||||
new InMemoryMeetingNoteStore(),
|
||||
new CapturingMeetingNoteOpener(),
|
||||
new InMemoryMeetingArtifactStore(),
|
||||
new InMemoryRecordedAudioStore(),
|
||||
new CapturingMeetingSummaryPipeline(),
|
||||
Options.Create(CreateOptionsWithoutFinalizer()),
|
||||
NullLogger<MeetingRecordingCoordinator>.Instance,
|
||||
screenshotService: screenshotService);
|
||||
|
||||
await coordinator.StartAsync(CancellationToken.None);
|
||||
var stop = coordinator.StopAsync(CancellationToken.None);
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => coordinator.CaptureScreenshotAsync(CancellationToken.None));
|
||||
|
||||
audioSource.Complete();
|
||||
await stop;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StopWaitsForPendingScreenshotOcrBeforeSummarizing()
|
||||
{
|
||||
var audioSource = new CapturedChunkThenCancelAudioSource(new AudioChunk([1, 0, 2, 0], 16000, 1));
|
||||
var artifactStore = new InMemoryMeetingArtifactStore();
|
||||
var summaryPipeline = new CapturingMeetingSummaryPipeline();
|
||||
var screenshotService = new BlockingMeetingScreenshotService();
|
||||
var coordinator = new MeetingRecordingCoordinator(
|
||||
audioSource,
|
||||
new TestSpeechRecognitionPipelineFactory(new FinalSegmentOnAudioCompletionProvider()),
|
||||
new InMemoryTranscriptStore(),
|
||||
new InMemoryMeetingNoteStore(),
|
||||
new CapturingMeetingNoteOpener(),
|
||||
artifactStore,
|
||||
new InMemoryRecordedAudioStore(),
|
||||
summaryPipeline,
|
||||
Options.Create(CreateOptionsWithoutFinalizer()),
|
||||
NullLogger<MeetingRecordingCoordinator>.Instance,
|
||||
screenshotService: screenshotService);
|
||||
|
||||
await coordinator.StartAsync(CancellationToken.None);
|
||||
await audioSource.WaitUntilCapturedAsync();
|
||||
var stop = coordinator.StopAsync(CancellationToken.None);
|
||||
await screenshotService.WaitUntilOcrWaitStartedAsync();
|
||||
|
||||
Assert.DoesNotContain(AssistantContextState.Summarizing, artifactStore.States);
|
||||
Assert.Null(summaryPipeline.Artifacts);
|
||||
|
||||
screenshotService.ReleaseOcrWait();
|
||||
await stop;
|
||||
|
||||
Assert.Contains(AssistantContextState.Summarizing, artifactStore.States);
|
||||
Assert.NotNull(summaryPipeline.Artifacts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StopMarksAssistantContextErrorWhenSummaryFails()
|
||||
{
|
||||
@@ -1396,6 +1648,93 @@ public sealed class RecordingCoordinatorTests
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CapturingMeetingScreenshotService : IMeetingScreenshotService
|
||||
{
|
||||
public MeetingSessionArtifacts? Artifacts { get; private set; }
|
||||
|
||||
public DateTimeOffset? MeetingStartedAt { get; private set; }
|
||||
|
||||
public Task<MeetingScreenshotCaptureResult> CaptureAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
DateTimeOffset? meetingStartedAt,
|
||||
DateTimeOffset capturedAt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Artifacts = artifacts;
|
||||
MeetingStartedAt = meetingStartedAt;
|
||||
return Task.FromResult(new MeetingScreenshotCaptureResult(
|
||||
"memory-screenshot.png",
|
||||
TimeSpan.FromSeconds(1),
|
||||
OcrStarted: false));
|
||||
}
|
||||
|
||||
public Task WaitForPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task CancelPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class BlockingMeetingScreenshotService : IMeetingScreenshotService
|
||||
{
|
||||
private readonly TaskCompletionSource waitStarted =
|
||||
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource releaseWait =
|
||||
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public Task<MeetingScreenshotCaptureResult> CaptureAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
DateTimeOffset? meetingStartedAt,
|
||||
DateTimeOffset capturedAt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new MeetingScreenshotCaptureResult(
|
||||
"memory-screenshot.png",
|
||||
TimeSpan.Zero,
|
||||
OcrStarted: true));
|
||||
}
|
||||
|
||||
public async Task WaitForPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
waitStarted.TrySetResult();
|
||||
await releaseWait.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public Task CancelPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
releaseWait.TrySetResult();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task WaitUntilOcrWaitStartedAsync()
|
||||
{
|
||||
return waitStarted.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
public void ReleaseOcrWait()
|
||||
{
|
||||
releaseWait.TrySetResult();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InMemoryMeetingArtifactStore : IMeetingArtifactStore
|
||||
{
|
||||
public MeetingSessionArtifacts? CreatedArtifacts { get; private set; }
|
||||
@@ -2140,6 +2479,11 @@ public sealed class RecordingCoordinatorTests
|
||||
{
|
||||
return chunks.Writer.WriteAsync(chunk, cancellationToken);
|
||||
}
|
||||
|
||||
public void Complete()
|
||||
{
|
||||
chunks.Writer.TryComplete();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CapturedChunkThenCancelAudioSource : IMeetingAudioSource
|
||||
|
||||
@@ -13,7 +13,7 @@ public sealed class GlobalHotkeyService : BackgroundService
|
||||
private readonly ILaunchProfileOptionsProvider launchProfiles;
|
||||
private readonly MeetingRecordingCoordinator coordinator;
|
||||
private readonly ILogger<GlobalHotkeyService> logger;
|
||||
private readonly Dictionary<int, string> hotkeyProfiles = [];
|
||||
private readonly Dictionary<int, LaunchProfileHotkey> registeredHotkeys = [];
|
||||
private TaskCompletionSource? messageLoopCompletion;
|
||||
private uint messageThreadId;
|
||||
|
||||
@@ -66,42 +66,44 @@ public sealed class GlobalHotkeyService : BackgroundService
|
||||
for (var index = 0; index < hotkeys.Count; index++)
|
||||
{
|
||||
var profileHotkey = hotkeys[index];
|
||||
var hotkey = HotkeyDefinition.Parse(profileHotkey.Toggle);
|
||||
var hotkey = HotkeyDefinition.Parse(profileHotkey.Hotkey);
|
||||
var hotkeyId = HotkeyIdBase + index;
|
||||
if (!RegisterHotKey(IntPtr.Zero, hotkeyId, (uint)hotkey.Modifiers, hotkey.VirtualKey))
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Could not register global hotkey {Hotkey} for launch profile {LaunchProfile}",
|
||||
profileHotkey.Toggle,
|
||||
profileHotkey.ProfileName);
|
||||
"Could not register global hotkey {Hotkey} for launch profile {LaunchProfile} action {Action}",
|
||||
profileHotkey.Hotkey,
|
||||
profileHotkey.ProfileName,
|
||||
profileHotkey.Action);
|
||||
continue;
|
||||
}
|
||||
|
||||
hotkeyProfiles[hotkeyId] = profileHotkey.ProfileName;
|
||||
registeredHotkeys[hotkeyId] = profileHotkey;
|
||||
logger.LogInformation(
|
||||
"Registered global hotkey {Hotkey} for launch profile {LaunchProfile}",
|
||||
profileHotkey.Toggle,
|
||||
profileHotkey.ProfileName);
|
||||
"Registered global hotkey {Hotkey} for launch profile {LaunchProfile} action {Action}",
|
||||
profileHotkey.Hotkey,
|
||||
profileHotkey.ProfileName,
|
||||
profileHotkey.Action);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested && GetMessage(out var message, IntPtr.Zero, 0, 0) > 0)
|
||||
{
|
||||
if (message.Message == WmHotkey && hotkeyProfiles.TryGetValue((int)message.WParam, out var profileName))
|
||||
if (message.Message == WmHotkey && registeredHotkeys.TryGetValue((int)message.WParam, out var hotkey))
|
||||
{
|
||||
_ = Task.Run(() => ToggleRecordingAsync(profileName), CancellationToken.None);
|
||||
_ = Task.Run(() => HandleHotkeyAsync(hotkey), CancellationToken.None);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var hotkeyId in hotkeyProfiles.Keys)
|
||||
foreach (var hotkeyId in registeredHotkeys.Keys)
|
||||
{
|
||||
UnregisterHotKey(IntPtr.Zero, hotkeyId);
|
||||
}
|
||||
|
||||
hotkeyProfiles.Clear();
|
||||
registeredHotkeys.Clear();
|
||||
messageThreadId = 0;
|
||||
}
|
||||
}
|
||||
@@ -112,6 +114,22 @@ public sealed class GlobalHotkeyService : BackgroundService
|
||||
return base.StopAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private async Task HandleHotkeyAsync(LaunchProfileHotkey hotkey)
|
||||
{
|
||||
switch (hotkey.Action)
|
||||
{
|
||||
case LaunchProfileHotkeyAction.ToggleRecording:
|
||||
await ToggleRecordingAsync(hotkey.ProfileName);
|
||||
break;
|
||||
case LaunchProfileHotkeyAction.AbortRecording:
|
||||
await AbortRecordingAsync(hotkey.ProfileName);
|
||||
break;
|
||||
case LaunchProfileHotkeyAction.CaptureScreenshot:
|
||||
await CaptureScreenshotAsync(hotkey.ProfileName);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ToggleRecordingAsync(string profileName)
|
||||
{
|
||||
try
|
||||
@@ -128,6 +146,38 @@ public sealed class GlobalHotkeyService : BackgroundService
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AbortRecordingAsync(string profileName)
|
||||
{
|
||||
try
|
||||
{
|
||||
var status = await coordinator.AbortAsync(CancellationToken.None);
|
||||
logger.LogInformation(
|
||||
"Hotkey aborted recording for launch profile {LaunchProfile}. IsRecording={IsRecording}",
|
||||
profileName,
|
||||
status.IsRecording);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Hotkey abort failed");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CaptureScreenshotAsync(string profileName)
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = await coordinator.CaptureScreenshotAsync(profileName, CancellationToken.None);
|
||||
logger.LogInformation(
|
||||
"Hotkey captured screenshot for launch profile {LaunchProfile}: {ScreenshotPath}",
|
||||
profileName,
|
||||
result.ScreenshotPath);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Hotkey screenshot capture failed");
|
||||
}
|
||||
}
|
||||
|
||||
private void PostQuitToMessageThread()
|
||||
{
|
||||
var threadId = messageThreadId;
|
||||
|
||||
@@ -11,11 +11,27 @@ public interface ILaunchProfileOptionsProvider
|
||||
|
||||
public sealed record LaunchProfile(string Name, MeetingAssistantOptions Options);
|
||||
|
||||
public sealed record LaunchProfileHotkey(string ProfileName, string Toggle);
|
||||
public enum LaunchProfileHotkeyAction
|
||||
{
|
||||
ToggleRecording,
|
||||
AbortRecording,
|
||||
CaptureScreenshot
|
||||
}
|
||||
|
||||
public sealed record LaunchProfileHotkey(
|
||||
string ProfileName,
|
||||
string Hotkey,
|
||||
LaunchProfileHotkeyAction Action);
|
||||
|
||||
public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOptionsProvider
|
||||
{
|
||||
public const string DefaultProfileName = "default";
|
||||
private static readonly HotkeyDescriptor[] HotkeyDescriptors =
|
||||
[
|
||||
new("Hotkey:Toggle", options => options.Hotkey.Toggle, LaunchProfileHotkeyAction.ToggleRecording),
|
||||
new("Hotkey:Abort", options => options.Hotkey.Abort, LaunchProfileHotkeyAction.AbortRecording),
|
||||
new("Screenshots:Hotkey", options => options.Screenshots.Hotkey, LaunchProfileHotkeyAction.CaptureScreenshot)
|
||||
];
|
||||
|
||||
private readonly IConfiguration configuration;
|
||||
|
||||
@@ -47,25 +63,47 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
||||
public IReadOnlyList<LaunchProfileHotkey> GetHotkeys()
|
||||
{
|
||||
var hotkeys = GetProfileNames()
|
||||
.Select(profileName =>
|
||||
{
|
||||
var profile = GetRequiredProfile(profileName);
|
||||
return new LaunchProfileHotkey(profile.Name, profile.Options.Hotkey.Toggle);
|
||||
})
|
||||
.Where(hotkey => !string.IsNullOrWhiteSpace(hotkey.Toggle))
|
||||
.SelectMany(CreateHotkeys)
|
||||
.ToList();
|
||||
var duplicate = hotkeys
|
||||
.GroupBy(hotkey => hotkey.Toggle.Trim(), StringComparer.OrdinalIgnoreCase)
|
||||
.GroupBy(hotkey => hotkey.Hotkey.Trim(), StringComparer.OrdinalIgnoreCase)
|
||||
.FirstOrDefault(group => group.Count() > 1);
|
||||
if (duplicate is not null)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Launch profile hotkey '{duplicate.Key}' is configured more than once for profiles {string.Join(", ", duplicate.Select(hotkey => hotkey.ProfileName))}.");
|
||||
$"Launch profile hotkey '{duplicate.Key}' is configured more than once for profiles {string.Join(", ", duplicate.Select(hotkey => $"{hotkey.ProfileName}:{hotkey.Action}"))}.");
|
||||
}
|
||||
|
||||
return hotkeys;
|
||||
}
|
||||
|
||||
private IEnumerable<LaunchProfileHotkey> CreateHotkeys(string profileName)
|
||||
{
|
||||
var profile = GetRequiredProfile(profileName);
|
||||
var isDefaultProfile = profileName.Equals(DefaultProfileName, StringComparison.OrdinalIgnoreCase);
|
||||
var profileSection = isDefaultProfile
|
||||
? null
|
||||
: GetProfilesSection().GetSection(profileName);
|
||||
|
||||
foreach (var descriptor in HotkeyDescriptors)
|
||||
{
|
||||
var hotkey = descriptor.GetValue(profile.Options);
|
||||
if (ShouldRegisterHotkey(isDefaultProfile, profileSection?.GetSection(descriptor.SectionPath)) &&
|
||||
!string.IsNullOrWhiteSpace(hotkey))
|
||||
{
|
||||
yield return new LaunchProfileHotkey(
|
||||
profile.Name,
|
||||
hotkey,
|
||||
descriptor.Action);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool ShouldRegisterHotkey(bool isDefaultProfile, IConfigurationSection? profileHotkeySection)
|
||||
{
|
||||
return isDefaultProfile || profileHotkeySection?.Exists() == true;
|
||||
}
|
||||
|
||||
private IReadOnlyList<string> GetProfileNames()
|
||||
{
|
||||
return new[] { DefaultProfileName }
|
||||
@@ -150,4 +188,9 @@ public sealed class ConfigurationLaunchProfileOptionsProvider : ILaunchProfileOp
|
||||
.Select(value => value!.Value)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private sealed record HotkeyDescriptor(
|
||||
string SectionPath,
|
||||
Func<MeetingAssistantOptions, string> GetValue,
|
||||
LaunchProfileHotkeyAction Action);
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
<TargetFrameworks>net10.0;net10.0-windows</TargetFrameworks>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<PreserveCompilationContext>true</PreserveCompilationContext>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
@@ -18,6 +19,7 @@
|
||||
<PackageReference Include="NAudio" Version="2.3.0" />
|
||||
<PackageReference Include="NCalcSync" Version="5.12.0" />
|
||||
<PackageReference Include="RazorLight" Version="2.3.1" />
|
||||
<PackageReference Include="System.Drawing.Common" Version="10.0.0" />
|
||||
<PackageReference Include="Whisper.net" Version="1.9.0" />
|
||||
<PackageReference Include="Whisper.net.Runtime" Version="1.9.0" />
|
||||
<PackageReference Include="YamlDotNet" Version="17.1.0" />
|
||||
@@ -27,6 +29,7 @@
|
||||
<Compile Remove="Hotkeys\GlobalHotkeyService.cs" />
|
||||
<Compile Remove="Recording\NaudioCaptureSource.cs" />
|
||||
<Compile Remove="MeetingNotes\OutlookClassicMeetingMetadataProvider.Windows.cs" />
|
||||
<Compile Remove="Screenshots\ActiveWindowScreenshotCapture.Windows.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -18,6 +18,8 @@ public sealed class MeetingAssistantOptions
|
||||
|
||||
public AutomationOptions Automation { get; set; } = new();
|
||||
|
||||
public ScreenshotOptions Screenshots { get; set; } = new();
|
||||
|
||||
public AgentOptions Agent { get; set; } = new();
|
||||
|
||||
public ApiOptions Api { get; set; } = new();
|
||||
@@ -28,9 +30,60 @@ public sealed class AutomationOptions
|
||||
public string? RulesPath { get; set; } = "meeting-rules.local.yaml";
|
||||
}
|
||||
|
||||
public sealed class ScreenshotOptions
|
||||
{
|
||||
public string Hotkey { get; set; } = "Ctrl+Alt+S";
|
||||
|
||||
public string AttachmentsFolder { get; set; } = "Attachments";
|
||||
|
||||
public ScreenshotOcrOptions Ocr { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class ScreenshotOcrOptions
|
||||
{
|
||||
public const string DefaultPrompt = """
|
||||
This screenshot was captured during a meeting. Extract information that is useful for meeting notes.
|
||||
|
||||
Identify who appears to be talking, who appears to be presenting, and what is being presented when visible.
|
||||
If the screenshot contains slides, capture the visible text in clean markdown.
|
||||
If the screenshot contains a diagram, convert it to Mermaid when possible and preserve labels accurately.
|
||||
If it contains another kind of shared screen or scene, describe the scene and the relevant visible information.
|
||||
If you can isolate the actual presentation, shared screen, or similarly relevant meeting content, return crop coordinates for only that content.
|
||||
Crop coordinates must be pixel coordinates relative to the original screenshot: x, y, width, height.
|
||||
Do not return crop coordinates for a person, webcam tile, whole app chrome, or content you cannot isolate confidently.
|
||||
End your response with exactly one fenced json block containing meeting-assistant metadata, for example:
|
||||
```json
|
||||
{ "crop": { "x": 0, "y": 0, "width": 1920, "height": 1080 } }
|
||||
```
|
||||
Use `{ "crop": null }` when no confident presentation/shared-screen crop exists.
|
||||
Do not invent information that is not visible in the screenshot.
|
||||
""";
|
||||
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
public string? Endpoint { get; set; }
|
||||
|
||||
public string? Key { get; set; }
|
||||
|
||||
public string KeyEnv { get; set; } = "LITELLM_API_KEY";
|
||||
|
||||
public string? Model { get; set; }
|
||||
|
||||
public string Prompt { get; set; } = DefaultPrompt;
|
||||
|
||||
public TimeSpan Timeout { get; set; } = TimeSpan.FromMinutes(2);
|
||||
|
||||
public TimeSpan GetEffectiveTimeout()
|
||||
{
|
||||
return Timeout > TimeSpan.Zero ? Timeout : TimeSpan.FromMinutes(2);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class HotkeyOptions
|
||||
{
|
||||
public string Toggle { get; set; } = "Ctrl+Alt+M";
|
||||
|
||||
public string Abort { get; set; } = "Ctrl+Alt+D";
|
||||
}
|
||||
|
||||
public sealed class VaultOptions
|
||||
@@ -64,6 +117,8 @@ public sealed class RecordingOptions
|
||||
|
||||
public TimeSpan BackgroundMetadataLookupTimeout { get; set; } = TimeSpan.FromMinutes(1);
|
||||
|
||||
public int MaxMetadataAttendeeImportCount { get; set; } = 30;
|
||||
|
||||
public TimeSpan OpenMeetingNoteDelay { get; set; } = TimeSpan.FromSeconds(1);
|
||||
|
||||
public string TemporaryRecordingsFolder { get; set; } = @"%LOCALAPPDATA%\MeetingAssistant\Recordings";
|
||||
|
||||
@@ -3,6 +3,7 @@ using MeetingAssistant.Hotkeys;
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Recording;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Summary;
|
||||
using MeetingAssistant.Transcription;
|
||||
@@ -29,8 +30,16 @@ builder.Services.AddSingleton<ITranscriptStore, VaultTranscriptStore>();
|
||||
builder.Services.AddSingleton<IMeetingNoteStore, MarkdownMeetingNoteStore>();
|
||||
builder.Services.AddSingleton<IMeetingNoteOpener, ObsidianMeetingNoteOpener>();
|
||||
builder.Services.AddSingleton<IMeetingArtifactStore, MarkdownMeetingArtifactStore>();
|
||||
builder.Services.AddSingleton<IMeetingRunArtifactCleaner, MeetingRunArtifactCleaner>();
|
||||
builder.Services.AddSingleton<IRecordedAudioStore, TemporaryRecordedAudioStore>();
|
||||
builder.Services.AddSingleton<IDictationWordStore, MarkdownDictationWordStore>();
|
||||
#if WINDOWS
|
||||
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, ActiveWindowScreenshotCapture>();
|
||||
#else
|
||||
builder.Services.AddSingleton<IActiveWindowScreenshotCapture, UnavailableActiveWindowScreenshotCapture>();
|
||||
#endif
|
||||
builder.Services.AddSingleton<IScreenshotOcrClient, LiteLlmScreenshotOcrClient>();
|
||||
builder.Services.AddSingleton<IMeetingScreenshotService, MeetingScreenshotService>();
|
||||
builder.Services.AddDbContextFactory<SpeakerIdentityDbContext>((services, dbOptions) =>
|
||||
{
|
||||
var appOptions = services.GetRequiredService<IOptions<MeetingAssistantOptions>>().Value;
|
||||
@@ -236,6 +245,13 @@ app.MapPost("/profiles/{launchProfile}/recording/stop", async (
|
||||
MeetingRecordingCoordinator coordinator,
|
||||
CancellationToken cancellationToken) =>
|
||||
Results.Ok(await coordinator.StopAsync(cancellationToken)));
|
||||
app.MapPost("/recording/abort", async (MeetingRecordingCoordinator coordinator, CancellationToken cancellationToken) =>
|
||||
Results.Ok(await coordinator.AbortAsync(cancellationToken)));
|
||||
app.MapPost("/profiles/{launchProfile}/recording/abort", async (
|
||||
string launchProfile,
|
||||
MeetingRecordingCoordinator coordinator,
|
||||
CancellationToken cancellationToken) =>
|
||||
Results.Ok(await coordinator.AbortAsync(cancellationToken)));
|
||||
app.MapPost("/meetings/current/summary/run", async (
|
||||
MeetingRecordingCoordinator coordinator,
|
||||
IMeetingSummaryPipeline summaryPipeline,
|
||||
|
||||
@@ -3,6 +3,7 @@ using System.Collections.Concurrent;
|
||||
using MeetingAssistant.LaunchProfiles;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
using MeetingAssistant.Speakers;
|
||||
using MeetingAssistant.Screenshots;
|
||||
using MeetingAssistant.Summary;
|
||||
using MeetingAssistant.Transcription;
|
||||
using MeetingAssistant.Workflow;
|
||||
@@ -26,6 +27,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
private readonly IDictationWordStore dictationWordStore;
|
||||
private readonly ILaunchProfileOptionsProvider? launchProfiles;
|
||||
private readonly IMeetingWorkflowEngine meetingWorkflowEngine;
|
||||
private readonly IMeetingScreenshotService screenshotService;
|
||||
private readonly IMeetingRunArtifactCleaner artifactCleaner;
|
||||
private readonly MeetingAssistantOptions options;
|
||||
private readonly ILogger<MeetingRecordingCoordinator> logger;
|
||||
private readonly SemaphoreSlim gate = new(1, 1);
|
||||
@@ -50,7 +53,9 @@ public sealed class MeetingRecordingCoordinator
|
||||
IDictationWordStore? dictationWordStore = null,
|
||||
ISpeakerIdentityAttendeeCanonicalizer? attendeeCanonicalizer = null,
|
||||
ILaunchProfileOptionsProvider? launchProfiles = null,
|
||||
IMeetingWorkflowEngine? meetingWorkflowEngine = null)
|
||||
IMeetingWorkflowEngine? meetingWorkflowEngine = null,
|
||||
IMeetingScreenshotService? screenshotService = null,
|
||||
IMeetingRunArtifactCleaner? artifactCleaner = null)
|
||||
{
|
||||
this.audioSource = audioSource;
|
||||
this.speechRecognitionPipelineFactory = speechRecognitionPipelineFactory;
|
||||
@@ -66,6 +71,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
this.attendeeCanonicalizer = attendeeCanonicalizer ?? PassthroughSpeakerIdentityAttendeeCanonicalizer.Instance;
|
||||
this.launchProfiles = launchProfiles;
|
||||
this.meetingWorkflowEngine = meetingWorkflowEngine ?? NoopMeetingWorkflowEngine.Instance;
|
||||
this.screenshotService = screenshotService ?? NoopMeetingScreenshotService.Instance;
|
||||
this.artifactCleaner = artifactCleaner ?? new MeetingRunArtifactCleaner();
|
||||
this.options = options.Value;
|
||||
this.logger = logger;
|
||||
}
|
||||
@@ -230,6 +237,87 @@ public sealed class MeetingRecordingCoordinator
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
public async Task<RecordingStatus> AbortAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
RecordingRun run;
|
||||
MeetingSessionArtifacts artifacts;
|
||||
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (currentRun is null)
|
||||
{
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
run = currentRun;
|
||||
artifacts = run.Artifacts;
|
||||
run.Abort();
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await run.Task.WaitAsync(run.Options.Recording.StopProcessingTimeout, cancellationToken);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
logger.LogWarning("Timed out while aborting meeting recording; forcing transcription cancellation");
|
||||
run.CancelTranscription();
|
||||
await run.Task.WaitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
await screenshotService.CancelPendingOcrAsync(
|
||||
artifacts,
|
||||
run.Options.Screenshots.Ocr.GetEffectiveTimeout(),
|
||||
cancellationToken);
|
||||
await artifactCleaner.DeleteRunArtifactsAsync(artifacts, run.Options, cancellationToken);
|
||||
logger.LogInformation("Meeting recording aborted and artifacts deleted");
|
||||
return CurrentStatus;
|
||||
}
|
||||
|
||||
public Task<MeetingScreenshotCaptureResult> CaptureScreenshotAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return CaptureScreenshotAsync(null, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<MeetingScreenshotCaptureResult> CaptureScreenshotAsync(
|
||||
string? launchProfileName,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
MeetingSessionArtifacts artifacts;
|
||||
DateTimeOffset? meetingStartedAt;
|
||||
MeetingAssistantOptions runOptions;
|
||||
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (currentRun is not { IsCaptureStopping: false, IsAborted: false } run ||
|
||||
currentMeetingNote is null)
|
||||
{
|
||||
throw new InvalidOperationException("No active meeting is available for screenshot capture.");
|
||||
}
|
||||
|
||||
artifacts = run.Artifacts;
|
||||
meetingStartedAt = currentMeetingNote.Frontmatter.StartTime;
|
||||
runOptions = run.Options;
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
|
||||
return await screenshotService.CaptureAsync(
|
||||
artifacts,
|
||||
meetingStartedAt,
|
||||
DateTimeOffset.Now,
|
||||
runOptions,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private async Task RecordAsync(RecordingRun run)
|
||||
{
|
||||
await run.Pipeline.InitializeAsync(run.PipelineOptions, run.TranscriptionCancellation);
|
||||
@@ -457,11 +545,21 @@ public sealed class MeetingRecordingCoordinator
|
||||
return;
|
||||
}
|
||||
|
||||
if (run.IsAborted)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await gate.WaitAsync(CancellationToken.None);
|
||||
try
|
||||
{
|
||||
if (run.IsAborted)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var meetingNote = await meetingNoteStore.ReadAsync(run.MeetingNotePath, CancellationToken.None);
|
||||
await ApplyMeetingMetadataAsync(meetingNote, metadata, CancellationToken.None);
|
||||
await ApplyMeetingMetadataAsync(meetingNote, metadata, run.Options, CancellationToken.None);
|
||||
meetingNote = await meetingNoteStore.SaveAsync(meetingNote, run.Options, CancellationToken.None);
|
||||
if (currentMeetingNote?.Path == meetingNote.Path)
|
||||
{
|
||||
@@ -494,10 +592,13 @@ public sealed class MeetingRecordingCoordinator
|
||||
{
|
||||
try
|
||||
{
|
||||
await TransitionMeetingAsync(
|
||||
run,
|
||||
AssistantContextState.Transcribing,
|
||||
CancellationToken.None);
|
||||
if (!run.IsAborted)
|
||||
{
|
||||
await TransitionMeetingAsync(
|
||||
run,
|
||||
AssistantContextState.Transcribing,
|
||||
CancellationToken.None);
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
@@ -512,6 +613,7 @@ public sealed class MeetingRecordingCoordinator
|
||||
private async Task ApplyMeetingMetadataAsync(
|
||||
MeetingNote meetingNote,
|
||||
MeetingMetadata metadata,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(metadata.Title))
|
||||
@@ -521,6 +623,16 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
if (metadata.Attendees.Count > 0)
|
||||
{
|
||||
var attendeeImportLimit = Math.Max(0, options.Recording.MaxMetadataAttendeeImportCount);
|
||||
if (metadata.Attendees.Count > attendeeImportLimit)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Skipped importing {AttendeeCount} metadata attendees because it exceeds the configured limit {AttendeeImportLimit}",
|
||||
metadata.Attendees.Count,
|
||||
attendeeImportLimit);
|
||||
return;
|
||||
}
|
||||
|
||||
meetingNote.Frontmatter.Attendees = await CanonicalizeAttendeesAsync(metadata.Attendees, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -835,8 +947,17 @@ public sealed class MeetingRecordingCoordinator
|
||||
RecordingRun run,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (run.IsAborted)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await screenshotService.WaitForPendingOcrAsync(
|
||||
run.Artifacts,
|
||||
run.Options.Screenshots.Ocr.GetEffectiveTimeout(),
|
||||
cancellationToken);
|
||||
await TransitionMeetingAsync(
|
||||
run,
|
||||
AssistantContextState.Summarizing,
|
||||
@@ -866,6 +987,11 @@ public sealed class MeetingRecordingCoordinator
|
||||
AssistantContextState state,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (run.IsAborted)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!run.TryTransitionTo(state, out var fromState))
|
||||
{
|
||||
return;
|
||||
@@ -1038,6 +1164,8 @@ public sealed class MeetingRecordingCoordinator
|
||||
|
||||
public bool IsCaptureStopping => CaptureCancellationSource.IsCancellationRequested;
|
||||
|
||||
public bool IsAborted { get; private set; }
|
||||
|
||||
public AssistantContextState ContextState { get; private set; } = AssistantContextState.CollectingMetadata;
|
||||
|
||||
public void StopCapture()
|
||||
@@ -1045,6 +1173,14 @@ public sealed class MeetingRecordingCoordinator
|
||||
CaptureCancellationSource.Cancel();
|
||||
}
|
||||
|
||||
public void Abort()
|
||||
{
|
||||
IsAborted = true;
|
||||
CaptureCancellationSource.Cancel();
|
||||
TranscriptionCancellationSource.Cancel();
|
||||
LiveIdentificationCancellationSource.Cancel();
|
||||
}
|
||||
|
||||
public void CancelTranscription()
|
||||
{
|
||||
TranscriptionCancellationSource.Cancel();
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
|
||||
namespace MeetingAssistant.Recording;
|
||||
|
||||
public interface IMeetingRunArtifactCleaner
|
||||
{
|
||||
Task DeleteRunArtifactsAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class MeetingRunArtifactCleaner : IMeetingRunArtifactCleaner
|
||||
{
|
||||
private static readonly Regex MarkdownImageLinkPattern = new("!\\[[^\\]]*\\]\\((?<path>[^)]+)\\)", RegexOptions.Compiled);
|
||||
private static readonly Regex GeneratedScreenshotFileNamePattern = new(
|
||||
@"^\d{8}-\d{6}-\d{3}-\d{6}(?:-cropped)?\.png$",
|
||||
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private readonly ILogger<MeetingRunArtifactCleaner>? logger;
|
||||
|
||||
public MeetingRunArtifactCleaner(ILogger<MeetingRunArtifactCleaner>? logger = null)
|
||||
{
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task DeleteRunArtifactsAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var linkedFiles = await GetLinkedScreenshotAttachmentFilesAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
options.Screenshots.AttachmentsFolder,
|
||||
cancellationToken);
|
||||
foreach (var linkedFile in linkedFiles)
|
||||
{
|
||||
await DeleteFileIfExistsAsync(linkedFile, cancellationToken);
|
||||
}
|
||||
|
||||
await DeleteFileIfExistsAsync(artifacts.SummaryPath, cancellationToken);
|
||||
await DeleteFileIfExistsAsync(artifacts.AssistantContextPath, cancellationToken);
|
||||
await DeleteFileIfExistsAsync(artifacts.TranscriptPath, cancellationToken);
|
||||
await DeleteFileIfExistsAsync(artifacts.MeetingNotePath, cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<string>> GetLinkedScreenshotAttachmentFilesAsync(
|
||||
string assistantContextPath,
|
||||
string configuredAttachmentsFolder,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!File.Exists(assistantContextPath))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var contextDirectory = Path.GetDirectoryName(Path.GetFullPath(assistantContextPath));
|
||||
if (string.IsNullOrWhiteSpace(contextDirectory))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var content = await File.ReadAllTextAsync(assistantContextPath, cancellationToken);
|
||||
var attachmentsFolder = ResolveAttachmentsFolder(contextDirectory, configuredAttachmentsFolder);
|
||||
return MarkdownImageLinkPattern
|
||||
.Matches(content)
|
||||
.Select(match => match.Groups["path"].Value.Trim())
|
||||
.Where(path => !string.IsNullOrWhiteSpace(path))
|
||||
.Select(path => ResolveScreenshotAttachmentLink(contextDirectory, attachmentsFolder, path))
|
||||
.Where(path => path is not null)
|
||||
.Select(path => path!)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string? ResolveScreenshotAttachmentLink(
|
||||
string contextDirectory,
|
||||
string attachmentsFolder,
|
||||
string linkPath)
|
||||
{
|
||||
if (Uri.TryCreate(linkPath, UriKind.Absolute, out var uri) && !uri.IsFile)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var withoutFragment = linkPath.Split('#')[0].Split('?')[0];
|
||||
var decoded = Uri.UnescapeDataString(withoutFragment);
|
||||
var fullPath = Path.IsPathRooted(decoded)
|
||||
? Path.GetFullPath(decoded)
|
||||
: Path.GetFullPath(Path.Combine(contextDirectory, decoded));
|
||||
var attachmentsRoot = Path.GetFullPath(attachmentsFolder).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) +
|
||||
Path.DirectorySeparatorChar;
|
||||
var fileName = Path.GetFileName(fullPath);
|
||||
return fullPath.StartsWith(attachmentsRoot, StringComparison.OrdinalIgnoreCase) &&
|
||||
GeneratedScreenshotFileNamePattern.IsMatch(fileName)
|
||||
? fullPath
|
||||
: null;
|
||||
}
|
||||
|
||||
private static string ResolveAttachmentsFolder(string contextDirectory, string configuredFolder)
|
||||
{
|
||||
var expanded = Environment.ExpandEnvironmentVariables(
|
||||
string.IsNullOrWhiteSpace(configuredFolder) ? "Attachments" : configuredFolder);
|
||||
return Path.IsPathRooted(expanded)
|
||||
? Path.GetFullPath(expanded)
|
||||
: Path.GetFullPath(Path.Combine(contextDirectory, expanded));
|
||||
}
|
||||
|
||||
private async Task DeleteFileIfExistsAsync(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
var deadline = DateTimeOffset.UtcNow.AddSeconds(2);
|
||||
while (true)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
logger?.LogInformation("Deleted aborted meeting artifact {ArtifactPath}", path);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
catch (IOException) when (DateTimeOffset.UtcNow < deadline)
|
||||
{
|
||||
await Task.Delay(25, cancellationToken);
|
||||
}
|
||||
catch (UnauthorizedAccessException) when (DateTimeOffset.UtcNow < deadline)
|
||||
{
|
||||
await Task.Delay(25, cancellationToken);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger?.LogWarning(exception, "Could not delete aborted meeting artifact {ArtifactPath}", path);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
public sealed class ActiveWindowScreenshotCapture : IActiveWindowScreenshotCapture
|
||||
{
|
||||
private static readonly IntPtr DpiAwarenessContextPerMonitorAwareV2 = new(-4);
|
||||
private const int DwmWindowAttributeExtendedFrameBounds = 9;
|
||||
|
||||
public Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var previousDpiContext = TrySetThreadDpiAwarenessContext(DpiAwarenessContextPerMonitorAwareV2);
|
||||
try
|
||||
{
|
||||
var foregroundWindow = GetForegroundWindow();
|
||||
if (foregroundWindow == IntPtr.Zero ||
|
||||
!TryGetWindowBounds(foregroundWindow, out var rect))
|
||||
{
|
||||
throw new InvalidOperationException("Could not determine the active window bounds.");
|
||||
}
|
||||
|
||||
using var bitmap = new Bitmap(rect.Width, rect.Height);
|
||||
using (var graphics = Graphics.FromImage(bitmap))
|
||||
{
|
||||
graphics.CopyFromScreen(rect.Left, rect.Top, 0, 0, new Size(rect.Width, rect.Height));
|
||||
}
|
||||
|
||||
using var stream = new MemoryStream();
|
||||
bitmap.Save(stream, ImageFormat.Png);
|
||||
return Task.FromResult(new ActiveWindowScreenshot(stream.ToArray(), GetWindowTitle(foregroundWindow)));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (previousDpiContext != IntPtr.Zero)
|
||||
{
|
||||
TrySetThreadDpiAwarenessContext(previousDpiContext);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryGetWindowBounds(IntPtr window, out NativeRect rect)
|
||||
{
|
||||
if (DwmGetWindowAttribute(
|
||||
window,
|
||||
DwmWindowAttributeExtendedFrameBounds,
|
||||
out rect,
|
||||
Marshal.SizeOf<NativeRect>()) == 0 &&
|
||||
rect.Width > 0 &&
|
||||
rect.Height > 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return GetWindowRect(window, out rect) &&
|
||||
rect.Width > 0 &&
|
||||
rect.Height > 0;
|
||||
}
|
||||
|
||||
private static IntPtr TrySetThreadDpiAwarenessContext(IntPtr dpiContext)
|
||||
{
|
||||
try
|
||||
{
|
||||
return SetThreadDpiAwarenessContext(dpiContext);
|
||||
}
|
||||
catch (EntryPointNotFoundException)
|
||||
{
|
||||
return IntPtr.Zero;
|
||||
}
|
||||
}
|
||||
|
||||
private static string? GetWindowTitle(IntPtr window)
|
||||
{
|
||||
var length = GetWindowTextLength(window);
|
||||
if (length <= 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var buffer = new System.Text.StringBuilder(length + 1);
|
||||
return GetWindowText(window, buffer, buffer.Capacity) > 0
|
||||
? buffer.ToString()
|
||||
: null;
|
||||
}
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
private static extern IntPtr GetForegroundWindow();
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true)]
|
||||
private static extern bool GetWindowRect(IntPtr hWnd, out NativeRect lpRect);
|
||||
|
||||
[DllImport("dwmapi.dll", PreserveSig = true)]
|
||||
private static extern int DwmGetWindowAttribute(
|
||||
IntPtr hwnd,
|
||||
int dwAttribute,
|
||||
out NativeRect pvAttribute,
|
||||
int cbAttribute);
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
private static extern IntPtr SetThreadDpiAwarenessContext(IntPtr dpiContext);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
||||
private static extern int GetWindowText(IntPtr hWnd, System.Text.StringBuilder lpString, int nMaxCount);
|
||||
|
||||
[DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
||||
private static extern int GetWindowTextLength(IntPtr hWnd);
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
private readonly struct NativeRect
|
||||
{
|
||||
public readonly int Left;
|
||||
public readonly int Top;
|
||||
public readonly int Right;
|
||||
public readonly int Bottom;
|
||||
|
||||
public int Width => Right - Left;
|
||||
|
||||
public int Height => Bottom - Top;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
public sealed partial class LiteLlmScreenshotOcrClient : IScreenshotOcrClient
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
private readonly ILogger<LiteLlmScreenshotOcrClient> logger;
|
||||
private readonly Func<HttpMessageHandler>? httpMessageHandlerFactory;
|
||||
|
||||
public LiteLlmScreenshotOcrClient(ILogger<LiteLlmScreenshotOcrClient> logger)
|
||||
{
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
internal LiteLlmScreenshotOcrClient(
|
||||
Func<HttpMessageHandler> httpMessageHandlerFactory,
|
||||
ILogger<LiteLlmScreenshotOcrClient> logger)
|
||||
{
|
||||
this.httpMessageHandlerFactory = httpMessageHandlerFactory;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<ScreenshotOcrResult> ExtractAsync(
|
||||
string screenshotPath,
|
||||
string prompt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var endpoint = !string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Endpoint)
|
||||
? options.Screenshots.Ocr.Endpoint
|
||||
: options.Agent.Endpoint;
|
||||
var model = !string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Model)
|
||||
? options.Screenshots.Ocr.Model
|
||||
: options.Agent.Model;
|
||||
var key = ResolveApiKey(options);
|
||||
var imageBytes = await File.ReadAllBytesAsync(screenshotPath, cancellationToken);
|
||||
using var httpClient = CreateHttpClient();
|
||||
httpClient.BaseAddress = NormalizeEndpoint(new Uri(endpoint));
|
||||
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", key);
|
||||
var payload = CreatePayload(model, CreatePrompt(prompt, imageBytes), imageBytes);
|
||||
using var content = new StringContent(payload.ToJsonString(JsonOptions), Encoding.UTF8, "application/json");
|
||||
using var response = await httpClient.PostAsync("responses", content, cancellationToken);
|
||||
var responseJson = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Screenshot OCR request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {responseJson}");
|
||||
}
|
||||
|
||||
var text = ParseOutputText(responseJson);
|
||||
logger.LogInformation("Screenshot OCR completed for {ScreenshotPath}", screenshotPath);
|
||||
return ParseOcrResult(text);
|
||||
}
|
||||
|
||||
private HttpClient CreateHttpClient()
|
||||
{
|
||||
return httpMessageHandlerFactory is null
|
||||
? new HttpClient()
|
||||
: new HttpClient(httpMessageHandlerFactory());
|
||||
}
|
||||
|
||||
private static JsonObject CreatePayload(string model, string prompt, byte[] imageBytes)
|
||||
{
|
||||
return new JsonObject
|
||||
{
|
||||
["model"] = model,
|
||||
["store"] = false,
|
||||
["input"] = new JsonArray
|
||||
{
|
||||
new JsonObject
|
||||
{
|
||||
["role"] = "user",
|
||||
["content"] = new JsonArray
|
||||
{
|
||||
new JsonObject
|
||||
{
|
||||
["type"] = "input_text",
|
||||
["text"] = prompt
|
||||
},
|
||||
new JsonObject
|
||||
{
|
||||
["type"] = "input_image",
|
||||
["image_url"] = "data:image/png;base64," + Convert.ToBase64String(imageBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static string CreatePrompt(string prompt, byte[] imageBytes)
|
||||
{
|
||||
return TryReadPngDimensions(imageBytes, out var width, out var height)
|
||||
? prompt + Environment.NewLine + Environment.NewLine +
|
||||
$"Original screenshot dimensions: {width}x{height} pixels. Crop coordinates must fit within these bounds."
|
||||
: prompt;
|
||||
}
|
||||
|
||||
private static ScreenshotOcrResult ParseOcrResult(string text)
|
||||
{
|
||||
ScreenshotCropCoordinates? crop = null;
|
||||
var cleaned = JsonCodeBlockRegex().Replace(text, match =>
|
||||
{
|
||||
if (TryParseCrop(match.Groups["json"].Value, out var parsedCrop))
|
||||
{
|
||||
crop = parsedCrop;
|
||||
return "";
|
||||
}
|
||||
|
||||
return match.Value;
|
||||
}).Trim();
|
||||
return new ScreenshotOcrResult(cleaned, crop);
|
||||
}
|
||||
|
||||
private static bool TryParseCrop(string json, out ScreenshotCropCoordinates? crop)
|
||||
{
|
||||
crop = null;
|
||||
try
|
||||
{
|
||||
using var document = JsonDocument.Parse(json);
|
||||
if (!document.RootElement.TryGetProperty("crop", out var cropElement) ||
|
||||
cropElement.ValueKind == JsonValueKind.Null)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (cropElement.ValueKind != JsonValueKind.Object ||
|
||||
!TryGetInt(cropElement, "x", out var x) ||
|
||||
!TryGetInt(cropElement, "y", out var y) ||
|
||||
!TryGetInt(cropElement, "width", out var width) ||
|
||||
!TryGetInt(cropElement, "height", out var height))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
crop = new ScreenshotCropCoordinates(x, y, width, height);
|
||||
return true;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryGetInt(JsonElement element, string propertyName, out int value)
|
||||
{
|
||||
value = 0;
|
||||
return element.TryGetProperty(propertyName, out var property) &&
|
||||
property.ValueKind == JsonValueKind.Number &&
|
||||
property.TryGetInt32(out value);
|
||||
}
|
||||
|
||||
private static bool TryReadPngDimensions(byte[] bytes, out int width, out int height)
|
||||
{
|
||||
width = 0;
|
||||
height = 0;
|
||||
if (bytes.Length < 24 ||
|
||||
bytes[0] != 0x89 ||
|
||||
bytes[1] != 0x50 ||
|
||||
bytes[2] != 0x4E ||
|
||||
bytes[3] != 0x47)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
width = ReadBigEndianInt32(bytes, 16);
|
||||
height = ReadBigEndianInt32(bytes, 20);
|
||||
return width > 0 && height > 0;
|
||||
}
|
||||
|
||||
private static int ReadBigEndianInt32(byte[] bytes, int offset)
|
||||
{
|
||||
return (bytes[offset] << 24) |
|
||||
(bytes[offset + 1] << 16) |
|
||||
(bytes[offset + 2] << 8) |
|
||||
bytes[offset + 3];
|
||||
}
|
||||
|
||||
private static string ParseOutputText(string responseJson)
|
||||
{
|
||||
using var document = JsonDocument.Parse(responseJson);
|
||||
var root = document.RootElement;
|
||||
var parts = new List<string>();
|
||||
if (root.TryGetProperty("output_text", out var outputText) &&
|
||||
outputText.ValueKind == JsonValueKind.String &&
|
||||
!string.IsNullOrWhiteSpace(outputText.GetString()))
|
||||
{
|
||||
parts.Add(outputText.GetString()!);
|
||||
}
|
||||
|
||||
if (root.TryGetProperty("output", out var output) && output.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var item in output.EnumerateArray())
|
||||
{
|
||||
if (!item.TryGetProperty("content", out var content) || content.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var block in content.EnumerateArray())
|
||||
{
|
||||
if (block.TryGetProperty("type", out var type) &&
|
||||
type.GetString() == "output_text" &&
|
||||
block.TryGetProperty("text", out var text) &&
|
||||
text.ValueKind == JsonValueKind.String &&
|
||||
!string.IsNullOrWhiteSpace(text.GetString()))
|
||||
{
|
||||
parts.Add(text.GetString()!);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return parts.Count == 0
|
||||
? ""
|
||||
: string.Join(Environment.NewLine + Environment.NewLine, parts);
|
||||
}
|
||||
|
||||
private static string ResolveApiKey(MeetingAssistantOptions options)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Key))
|
||||
{
|
||||
return options.Screenshots.Ocr.Key;
|
||||
}
|
||||
|
||||
var ocrKey = Environment.GetEnvironmentVariable(options.Screenshots.Ocr.KeyEnv);
|
||||
if (!string.IsNullOrWhiteSpace(ocrKey))
|
||||
{
|
||||
return ocrKey;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.Agent.Key))
|
||||
{
|
||||
return options.Agent.Key;
|
||||
}
|
||||
|
||||
var agentKey = Environment.GetEnvironmentVariable(options.Agent.KeyEnv);
|
||||
if (!string.IsNullOrWhiteSpace(agentKey))
|
||||
{
|
||||
return agentKey;
|
||||
}
|
||||
|
||||
throw new InvalidOperationException(
|
||||
$"No screenshot OCR API key configured. Set MeetingAssistant:Screenshots:Ocr:Key or environment variable '{options.Screenshots.Ocr.KeyEnv}'.");
|
||||
}
|
||||
|
||||
private static Uri NormalizeEndpoint(Uri endpoint)
|
||||
{
|
||||
var value = endpoint.ToString().TrimEnd('/');
|
||||
if (!value.EndsWith("/v1", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
value += "/v1";
|
||||
}
|
||||
|
||||
return new Uri(value + "/");
|
||||
}
|
||||
|
||||
[GeneratedRegex("```json\\s*(?<json>.*?)\\s*```", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
||||
private static partial Regex JsonCodeBlockRegex();
|
||||
}
|
||||
@@ -0,0 +1,482 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using System.Runtime.InteropServices;
|
||||
using MeetingAssistant.MeetingNotes;
|
||||
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
public interface IActiveWindowScreenshotCapture
|
||||
{
|
||||
Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record ActiveWindowScreenshot(byte[] PngBytes, string? WindowTitle);
|
||||
|
||||
public interface IScreenshotOcrClient
|
||||
{
|
||||
Task<ScreenshotOcrResult> ExtractAsync(
|
||||
string screenshotPath,
|
||||
string prompt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record ScreenshotOcrResult(string Text, ScreenshotCropCoordinates? Crop);
|
||||
|
||||
public sealed record ScreenshotCropCoordinates(int X, int Y, int Width, int Height);
|
||||
|
||||
public interface IMeetingScreenshotService
|
||||
{
|
||||
Task<MeetingScreenshotCaptureResult> CaptureAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
DateTimeOffset? meetingStartedAt,
|
||||
DateTimeOffset capturedAt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task WaitForPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task CancelPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record MeetingScreenshotCaptureResult(
|
||||
string ScreenshotPath,
|
||||
TimeSpan MeetingTimestamp,
|
||||
bool OcrStarted);
|
||||
|
||||
public sealed class NoopMeetingScreenshotService : IMeetingScreenshotService
|
||||
{
|
||||
public static NoopMeetingScreenshotService Instance { get; } = new();
|
||||
|
||||
public Task<MeetingScreenshotCaptureResult> CaptureAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
DateTimeOffset? meetingStartedAt,
|
||||
DateTimeOffset capturedAt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(new MeetingScreenshotCaptureResult("", TimeSpan.Zero, OcrStarted: false));
|
||||
}
|
||||
|
||||
public Task WaitForPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task CancelPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class MeetingScreenshotService : IMeetingScreenshotService
|
||||
{
|
||||
private readonly IActiveWindowScreenshotCapture screenshotCapture;
|
||||
private readonly IMeetingArtifactStore artifactStore;
|
||||
private readonly IScreenshotOcrClient ocrClient;
|
||||
private readonly ILogger<MeetingScreenshotService> logger;
|
||||
private readonly ConcurrentDictionary<string, List<PendingOcrTask>> pendingOcrByContext = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> contextFileLocks = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public MeetingScreenshotService(
|
||||
IActiveWindowScreenshotCapture screenshotCapture,
|
||||
IMeetingArtifactStore artifactStore,
|
||||
IScreenshotOcrClient ocrClient,
|
||||
ILogger<MeetingScreenshotService> logger)
|
||||
{
|
||||
this.screenshotCapture = screenshotCapture;
|
||||
this.artifactStore = artifactStore;
|
||||
this.ocrClient = ocrClient;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public async Task<MeetingScreenshotCaptureResult> CaptureAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
DateTimeOffset? meetingStartedAt,
|
||||
DateTimeOffset capturedAt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var screenshot = await screenshotCapture.CaptureAsync(cancellationToken);
|
||||
var timestamp = CalculateMeetingTimestamp(meetingStartedAt, capturedAt);
|
||||
var screenshotPath = await SaveScreenshotAsync(
|
||||
artifacts,
|
||||
screenshot.PngBytes,
|
||||
timestamp,
|
||||
capturedAt,
|
||||
options,
|
||||
cancellationToken);
|
||||
var relativePath = ToMarkdownPath(Path.GetRelativePath(
|
||||
Path.GetDirectoryName(artifacts.AssistantContextPath)!,
|
||||
screenshotPath));
|
||||
var screenshotId = Guid.NewGuid().ToString("N");
|
||||
var ocrEnabled = options.Screenshots.Ocr.Enabled;
|
||||
await artifactStore.AppendAssistantContextAsync(
|
||||
artifacts,
|
||||
CreateScreenshotMarkdown(screenshotId, timestamp, relativePath, screenshot.WindowTitle, ocrEnabled),
|
||||
cancellationToken);
|
||||
|
||||
if (ocrEnabled)
|
||||
{
|
||||
var ocrCancellation = new CancellationTokenSource();
|
||||
TrackOcr(
|
||||
artifacts,
|
||||
ocrCancellation,
|
||||
RunOcrAsync(artifacts, screenshotPath, screenshotId, options, ocrCancellation.Token));
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Captured meeting screenshot {ScreenshotPath} at {MeetingTimestamp}",
|
||||
screenshotPath,
|
||||
FormatTimestamp(timestamp));
|
||||
return new MeetingScreenshotCaptureResult(screenshotPath, timestamp, ocrEnabled);
|
||||
}
|
||||
|
||||
public async Task WaitForPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var contextKey = NormalizeContextKey(artifacts.AssistantContextPath);
|
||||
if (!pendingOcrByContext.TryGetValue(contextKey, out var tasks))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
PendingOcrTask[] snapshot;
|
||||
lock (tasks)
|
||||
{
|
||||
tasks.RemoveAll(task => task.Task.IsCompleted);
|
||||
snapshot = tasks.ToArray();
|
||||
}
|
||||
|
||||
if (snapshot.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(snapshot.Select(task => task.Task)).WaitAsync(timeout, cancellationToken);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Timed out waiting for {Count} screenshot OCR task(s) for {AssistantContextPath}",
|
||||
snapshot.Length,
|
||||
artifacts.AssistantContextPath);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task CancelPendingOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var contextKey = NormalizeContextKey(artifacts.AssistantContextPath);
|
||||
if (!pendingOcrByContext.TryGetValue(contextKey, out var tasks))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
PendingOcrTask[] snapshot;
|
||||
lock (tasks)
|
||||
{
|
||||
tasks.RemoveAll(task => task.Task.IsCompleted);
|
||||
snapshot = tasks.ToArray();
|
||||
}
|
||||
|
||||
if (snapshot.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var pending in snapshot)
|
||||
{
|
||||
await pending.Cancellation.CancelAsync();
|
||||
}
|
||||
|
||||
await WaitForPendingOcrAsync(artifacts, timeout, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<string> SaveScreenshotAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
byte[] pngBytes,
|
||||
TimeSpan timestamp,
|
||||
DateTimeOffset capturedAt,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = ResolveAttachmentsFolder(artifacts.AssistantContextPath, options.Screenshots.AttachmentsFolder);
|
||||
Directory.CreateDirectory(folder);
|
||||
var path = Path.Combine(
|
||||
folder,
|
||||
$"{capturedAt:yyyyMMdd-HHmmss-fff}-{FormatTimestamp(timestamp).Replace(":", "", StringComparison.Ordinal)}.png");
|
||||
await File.WriteAllBytesAsync(path, pngBytes, cancellationToken);
|
||||
return path;
|
||||
}
|
||||
|
||||
private async Task RunOcrAsync(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
string screenshotPath,
|
||||
string screenshotId,
|
||||
MeetingAssistantOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var timeout = options.Screenshots.Ocr.GetEffectiveTimeout();
|
||||
using var timeoutSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeoutSource.CancelAfter(timeout);
|
||||
try
|
||||
{
|
||||
var result = await ocrClient.ExtractAsync(
|
||||
screenshotPath,
|
||||
string.IsNullOrWhiteSpace(options.Screenshots.Ocr.Prompt)
|
||||
? ScreenshotOcrOptions.DefaultPrompt
|
||||
: options.Screenshots.Ocr.Prompt,
|
||||
options,
|
||||
timeoutSource.Token);
|
||||
var cropMarkdown = await TryCreateCropMarkdownAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotPath,
|
||||
result.Crop,
|
||||
timeoutSource.Token);
|
||||
await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
cropMarkdown +
|
||||
"### OCR" + Environment.NewLine + Environment.NewLine + result.Text.Trim(),
|
||||
CancellationToken.None);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogInformation("Cancelled screenshot OCR for {ScreenshotPath}", screenshotPath);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
$"_OCR timed out after {timeout:g}._",
|
||||
CancellationToken.None);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Screenshot OCR failed for {ScreenshotPath}", screenshotPath);
|
||||
await ReplaceOcrPlaceholderAsync(
|
||||
artifacts.AssistantContextPath,
|
||||
screenshotId,
|
||||
$"_OCR failed: {exception.Message}_",
|
||||
CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReplaceOcrPlaceholderAsync(
|
||||
string assistantContextPath,
|
||||
string screenshotId,
|
||||
string replacement,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var fileLock = contextFileLocks.GetOrAdd(
|
||||
NormalizeContextKey(assistantContextPath),
|
||||
_ => new SemaphoreSlim(1, 1));
|
||||
await fileLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (!File.Exists(assistantContextPath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var content = await File.ReadAllTextAsync(assistantContextPath, cancellationToken);
|
||||
var startMarker = $"<!-- screenshot-ocr:{screenshotId} -->";
|
||||
var endMarker = $"<!-- /screenshot-ocr:{screenshotId} -->";
|
||||
var startIndex = content.IndexOf(startMarker, StringComparison.Ordinal);
|
||||
var endIndex = content.IndexOf(endMarker, StringComparison.Ordinal);
|
||||
if (startIndex < 0 || endIndex < startIndex)
|
||||
{
|
||||
await File.AppendAllTextAsync(
|
||||
assistantContextPath,
|
||||
Environment.NewLine + replacement.TrimEnd() + Environment.NewLine,
|
||||
cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
endIndex += endMarker.Length;
|
||||
var updated = content[..startIndex] +
|
||||
replacement.TrimEnd() +
|
||||
content[endIndex..];
|
||||
await File.WriteAllTextAsync(assistantContextPath, updated, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
fileLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> TryCreateCropMarkdownAsync(
|
||||
string assistantContextPath,
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates? crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (crop is null)
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Ignoring screenshot crop coordinates for {ScreenshotPath} because image cropping is only supported on Windows",
|
||||
screenshotPath);
|
||||
return "";
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var croppedPath = await SaveCroppedScreenshotAsync(screenshotPath, crop, cancellationToken);
|
||||
var relativePath = ToMarkdownPath(Path.GetRelativePath(
|
||||
Path.GetDirectoryName(assistantContextPath)!,
|
||||
croppedPath));
|
||||
return $"" +
|
||||
Environment.NewLine +
|
||||
Environment.NewLine;
|
||||
}
|
||||
catch (Exception exception) when (exception is InvalidDataException or ArgumentException or ExternalException)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Ignoring invalid screenshot crop coordinates for {ScreenshotPath}",
|
||||
screenshotPath);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
#pragma warning disable CA1416
|
||||
private static Task<string> SaveCroppedScreenshotAsync(
|
||||
string screenshotPath,
|
||||
ScreenshotCropCoordinates crop,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
using var original = new Bitmap(screenshotPath);
|
||||
ValidateCrop(crop, original.Width, original.Height);
|
||||
var cropRectangle = new Rectangle(crop.X, crop.Y, crop.Width, crop.Height);
|
||||
using var cropped = original.Clone(cropRectangle, original.PixelFormat);
|
||||
var croppedPath = Path.Combine(
|
||||
Path.GetDirectoryName(screenshotPath)!,
|
||||
$"{Path.GetFileNameWithoutExtension(screenshotPath)}-cropped.png");
|
||||
cropped.Save(croppedPath, ImageFormat.Png);
|
||||
return Task.FromResult(croppedPath);
|
||||
}
|
||||
#pragma warning restore CA1416
|
||||
|
||||
private static void ValidateCrop(ScreenshotCropCoordinates crop, int imageWidth, int imageHeight)
|
||||
{
|
||||
if (crop.X < 0 ||
|
||||
crop.Y < 0 ||
|
||||
crop.Width <= 0 ||
|
||||
crop.Height <= 0 ||
|
||||
crop.X + crop.Width > imageWidth ||
|
||||
crop.Y + crop.Height > imageHeight)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Screenshot crop {crop.X},{crop.Y},{crop.Width},{crop.Height} is outside image bounds {imageWidth}x{imageHeight}.");
|
||||
}
|
||||
}
|
||||
|
||||
private void TrackOcr(
|
||||
MeetingSessionArtifacts artifacts,
|
||||
CancellationTokenSource cancellation,
|
||||
Task task)
|
||||
{
|
||||
var contextKey = NormalizeContextKey(artifacts.AssistantContextPath);
|
||||
var tasks = pendingOcrByContext.GetOrAdd(contextKey, _ => []);
|
||||
lock (tasks)
|
||||
{
|
||||
tasks.RemoveAll(existing => existing.Task.IsCompleted);
|
||||
tasks.Add(new PendingOcrTask(task, cancellation));
|
||||
}
|
||||
}
|
||||
|
||||
private static string CreateScreenshotMarkdown(
|
||||
string screenshotId,
|
||||
TimeSpan timestamp,
|
||||
string relativePath,
|
||||
string? windowTitle,
|
||||
bool ocrEnabled)
|
||||
{
|
||||
var title = string.IsNullOrWhiteSpace(windowTitle)
|
||||
? ""
|
||||
: Environment.NewLine + $"Window: {windowTitle.Trim()}" + Environment.NewLine;
|
||||
var timestampText = FormatTimestamp(timestamp);
|
||||
var content = $"## Screenshot [{timestampText}]" +
|
||||
title +
|
||||
Environment.NewLine +
|
||||
$"";
|
||||
if (!ocrEnabled)
|
||||
{
|
||||
return content;
|
||||
}
|
||||
|
||||
return content +
|
||||
Environment.NewLine +
|
||||
Environment.NewLine +
|
||||
$"<!-- screenshot-ocr:{screenshotId} -->" +
|
||||
Environment.NewLine +
|
||||
"_OCR pending..._" +
|
||||
Environment.NewLine +
|
||||
$"<!-- /screenshot-ocr:{screenshotId} -->";
|
||||
}
|
||||
|
||||
private static string ResolveAttachmentsFolder(string assistantContextPath, string configuredFolder)
|
||||
{
|
||||
var expanded = Environment.ExpandEnvironmentVariables(
|
||||
string.IsNullOrWhiteSpace(configuredFolder) ? "Attachments" : configuredFolder);
|
||||
return Path.IsPathRooted(expanded)
|
||||
? Path.GetFullPath(expanded)
|
||||
: Path.GetFullPath(Path.Combine(Path.GetDirectoryName(assistantContextPath)!, expanded));
|
||||
}
|
||||
|
||||
private static TimeSpan CalculateMeetingTimestamp(DateTimeOffset? meetingStartedAt, DateTimeOffset capturedAt)
|
||||
{
|
||||
if (meetingStartedAt is null)
|
||||
{
|
||||
return TimeSpan.Zero;
|
||||
}
|
||||
|
||||
var elapsed = capturedAt - meetingStartedAt.Value;
|
||||
return elapsed < TimeSpan.Zero ? TimeSpan.Zero : elapsed;
|
||||
}
|
||||
|
||||
private static string FormatTimestamp(TimeSpan timestamp)
|
||||
{
|
||||
return $"{(int)timestamp.TotalHours:00}:{timestamp.Minutes:00}:{timestamp.Seconds:00}";
|
||||
}
|
||||
|
||||
private static string ToMarkdownPath(string path)
|
||||
{
|
||||
return path.Replace(Path.DirectorySeparatorChar, '/')
|
||||
.Replace(Path.AltDirectorySeparatorChar, '/');
|
||||
}
|
||||
|
||||
private static string NormalizeContextKey(string assistantContextPath)
|
||||
{
|
||||
return Path.GetFullPath(assistantContextPath);
|
||||
}
|
||||
|
||||
private sealed record PendingOcrTask(Task Task, CancellationTokenSource Cancellation);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace MeetingAssistant.Screenshots;
|
||||
|
||||
public sealed class UnavailableActiveWindowScreenshotCapture : IActiveWindowScreenshotCapture
|
||||
{
|
||||
public Task<ActiveWindowScreenshot> CaptureAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
throw new PlatformNotSupportedException("Active-window screenshot capture is only available on Windows.");
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ public sealed class MeetingSummaryInstructionBuilder : IMeetingSummaryInstructio
|
||||
After writing the summary, update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context.
|
||||
Use list_projects first to see which projects are bound to this meeting. Use search and read_projectfile before changing existing project files.
|
||||
The summary note should contain concise sections for summary, decisions, open questions, and next steps.
|
||||
If the assistant context contains cropped screenshot markdown links, include only the most relevant cropped screenshots in the summary by markdown-linking them near the related summary text. Do not include every cropped screenshot by default, and do not link uncropped screenshots unless no cropped version exists and the image is important.
|
||||
Keep the output grounded in the source material and explicitly say when a section has no known items.
|
||||
""";
|
||||
|
||||
|
||||
@@ -130,7 +130,7 @@ public sealed class MeetingSummaryTools
|
||||
? await File.ReadAllTextAsync(artifacts.AssistantContextPath)
|
||||
: "";
|
||||
var (frontmatter, body) = MeetingArtifactFrontmatterRenderer.Split(existing);
|
||||
var updatedBody = ApplyLineEdit(body, content, editMode);
|
||||
var updatedBody = ApplyLineEdit(body, StripAccidentalFrontmatter(content), editMode);
|
||||
var updated = string.IsNullOrWhiteSpace(frontmatter)
|
||||
? updatedBody
|
||||
: "---" + Environment.NewLine + frontmatter + Environment.NewLine + "---" + Environment.NewLine + Environment.NewLine + updatedBody;
|
||||
@@ -139,6 +139,12 @@ public sealed class MeetingSummaryTools
|
||||
return artifacts.AssistantContextPath;
|
||||
}
|
||||
|
||||
private static string StripAccidentalFrontmatter(string content)
|
||||
{
|
||||
var document = MarkdownDocumentParser.SplitOptional(content);
|
||||
return document.HasFrontmatter ? document.Body : content;
|
||||
}
|
||||
|
||||
public async Task<string> ListProjects()
|
||||
{
|
||||
var projects = await GetBoundProjectsAsync();
|
||||
|
||||
@@ -30,6 +30,10 @@ public sealed class NoopMeetingWorkflowEngine : IMeetingWorkflowEngine
|
||||
|
||||
public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
|
||||
{
|
||||
private static readonly Regex EmailAddressPattern = new(
|
||||
@"(?<![A-Za-z0-9.!#$%&'*+/=?^_`{|}~-])(?<local>[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+)@(?<domain>(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,63})(?![A-Za-z0-9-])",
|
||||
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
|
||||
private static readonly string[] ParameterNames =
|
||||
[
|
||||
"meeting.attendees.count",
|
||||
@@ -252,17 +256,65 @@ public sealed class MeetingWorkflowEngine : IMeetingWorkflowEngine
|
||||
string template,
|
||||
MeetingWorkflowTemplateModel model)
|
||||
{
|
||||
if (!template.Contains('@', StringComparison.Ordinal))
|
||||
if (!ContainsRazorTemplateSyntax(template))
|
||||
{
|
||||
return template;
|
||||
}
|
||||
|
||||
return await razorEngine.CompileRenderStringAsync(
|
||||
Guid.NewGuid().ToString("N"),
|
||||
template,
|
||||
EscapeEmailAddressAtSigns(template),
|
||||
model);
|
||||
}
|
||||
|
||||
private static bool ContainsRazorTemplateSyntax(string value)
|
||||
{
|
||||
if (!value.Contains('@', StringComparison.Ordinal))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var emailAtSigns = EmailAddressPattern
|
||||
.Matches(value)
|
||||
.Select(match => match.Index + match.Value.IndexOf('@', StringComparison.Ordinal))
|
||||
.ToHashSet();
|
||||
|
||||
for (var index = 0; index < value.Length; index++)
|
||||
{
|
||||
if (value[index] != '@')
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (index + 1 < value.Length && value[index + 1] == '@')
|
||||
{
|
||||
index++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (emailAtSigns.Contains(index))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string EscapeEmailAddressAtSigns(string value)
|
||||
{
|
||||
if (!value.Contains('@', StringComparison.Ordinal))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
return EmailAddressPattern.Replace(
|
||||
value,
|
||||
match => match.Value.Replace("@", "@@", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
private static bool SetProperty(
|
||||
MeetingNote meeting,
|
||||
string? property,
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
{
|
||||
"MeetingAssistant": {
|
||||
"Hotkey": {
|
||||
"Toggle": "Ctrl+Alt+M"
|
||||
"Toggle": "Ctrl+Alt+M",
|
||||
"Abort": "Ctrl+Alt+D"
|
||||
},
|
||||
"Vault": {
|
||||
"BaseFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Exxeta",
|
||||
@@ -19,6 +20,7 @@
|
||||
"StopProcessingTimeout": "00:10:00",
|
||||
"MetadataLookupTimeout": "00:00:10",
|
||||
"BackgroundMetadataLookupTimeout": "00:01:00",
|
||||
"MaxMetadataAttendeeImportCount": 30,
|
||||
"OpenMeetingNoteDelay": "00:00:01",
|
||||
"TemporaryRecordingsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Recordings"
|
||||
},
|
||||
@@ -107,7 +109,18 @@
|
||||
"MatchTimeout": "00:03:00"
|
||||
},
|
||||
"Automation": {
|
||||
"RulesPath": "meeting-rules.local.yaml"
|
||||
"RulesPath": "C:\\Manuel\\meeting-assistant\\meeting-rules.local.yaml"
|
||||
},
|
||||
"Screenshots": {
|
||||
"Hotkey": "Ctrl+Alt+S",
|
||||
"AttachmentsFolder": "Attachments",
|
||||
"Ocr": {
|
||||
"Enabled": true,
|
||||
"Endpoint": "",
|
||||
"KeyEnv": "LITELLM_API_KEY",
|
||||
"Model": "",
|
||||
"Timeout": "00:02:00"
|
||||
}
|
||||
},
|
||||
"Agent": {
|
||||
"Endpoint": "https://litellm.schweigert.cloud",
|
||||
@@ -121,8 +134,7 @@
|
||||
"MaxOutputTokens": 8192,
|
||||
"EnableCompaction": true,
|
||||
"CompactionRemainingRatio": 0.1,
|
||||
"ResponsesCompactPath": "responses/compact",
|
||||
"InitialPrompt": "You are the Meeting Assistant summary agent.\n\nUse the provided tools to read the meeting transcript, assistant context, user notes, glossary, and project files.\nAll read tools can return the whole file or a clamped inclusive line range when from and to are supplied; use ranges for large inputs before asking for more lines.\nThen write the finished meeting summary as markdown by calling write_summary. If the meeting note has no title, provide a concise title parameter to write_summary.\nUse read_meetingnote to inspect frontmatter such as title, attendees, projects, start_time, and end_time.\nUse read_context and write_context as your own meeting notebook. Its frontmatter may include agenda from the calendar appointment. Record useful internal notes, missing context, requests for future tools, suggested improvements, and relevant context discovered from other sources. Keep user-facing summary content in the summary note.\nUse add_dictation_word when project context, user notes, or transcript evidence show that a domain term, acronym, name, or unusual word is likely to be repeatedly mistranscribed. Add only the canonical spelling, one term at a time.\nAfter writing the summary, update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context.\nUse list_projects first to see which projects are bound to this meeting. Use search and read_projectfile before changing existing project files.\nThe summary note should contain concise sections for summary, decisions, open questions, and next steps.\nKeep the output grounded in the source material and explicitly say when a section has no known items."
|
||||
"ResponsesCompactPath": "responses/compact"
|
||||
},
|
||||
"Api": {
|
||||
"PublicBaseUrl": "http://localhost:5090"
|
||||
|
||||
@@ -9,6 +9,8 @@ The application is intended to:
|
||||
- create an Obsidian markdown note before transcription starts
|
||||
- keep meeting metadata, user notes, detected context, and links to generated output in that note
|
||||
- toggle recording/transcription mode through a configurable global hotkey
|
||||
- abort and discard an active recording through a configurable global hotkey
|
||||
- capture active-window screenshots through a configurable global hotkey
|
||||
- capture microphone input and computer output into one transcription stream
|
||||
- transcribe meetings with speaker attribution
|
||||
- use a configurable speech recognition pipeline, with local Whisper as the first version 1 fallback
|
||||
@@ -67,6 +69,7 @@ GET /recording/status
|
||||
POST /recording/toggle
|
||||
POST /recording/start
|
||||
POST /recording/stop
|
||||
POST /recording/abort
|
||||
POST /asr/transcribe-file
|
||||
POST /asr/diarize-file
|
||||
POST /diagnostics/workflow/reload
|
||||
@@ -83,7 +86,8 @@ Meeting Assistant uses normal .NET configuration under the `MeetingAssistant` se
|
||||
{
|
||||
"MeetingAssistant": {
|
||||
"Hotkey": {
|
||||
"Toggle": "Ctrl+Alt+M"
|
||||
"Toggle": "Ctrl+Alt+M",
|
||||
"Abort": "Ctrl+Alt+D"
|
||||
},
|
||||
"Vault": {
|
||||
"TranscriptsFolder": "%USERPROFILE%\\OpenCloud\\Persönlich\\Vault\\Meetings\\Transcripts",
|
||||
@@ -98,6 +102,7 @@ Meeting Assistant uses normal .NET configuration under the `MeetingAssistant` se
|
||||
"SampleRate": 16000,
|
||||
"Channels": 1,
|
||||
"StopProcessingTimeout": "00:10:00",
|
||||
"MaxMetadataAttendeeImportCount": 30,
|
||||
"TemporaryRecordingsFolder": "%LOCALAPPDATA%\\MeetingAssistant\\Recordings"
|
||||
},
|
||||
"FunAsr": {
|
||||
@@ -164,6 +169,18 @@ Meeting Assistant uses normal .NET configuration under the `MeetingAssistant` se
|
||||
"Automation": {
|
||||
"RulesPath": "meeting-rules.local.yaml"
|
||||
},
|
||||
"Screenshots": {
|
||||
"Hotkey": "Ctrl+Alt+S",
|
||||
"AttachmentsFolder": "Attachments",
|
||||
"Ocr": {
|
||||
"Enabled": false,
|
||||
"Endpoint": "",
|
||||
"KeyEnv": "LITELLM_API_KEY",
|
||||
"Model": "",
|
||||
"Timeout": "00:02:00",
|
||||
"Prompt": "This screenshot was captured during a meeting..."
|
||||
}
|
||||
},
|
||||
"Agent": {
|
||||
"Endpoint": "https://litellm.schweigert.cloud",
|
||||
"KeyEnv": "LITELLM_API_KEY",
|
||||
@@ -205,8 +222,12 @@ For real meeting recordings, Meeting Assistant derives the optional finished-tra
|
||||
|
||||
Meeting notes link to separate transcript, assistant context, and summary markdown artifacts using the configured vault folders.
|
||||
|
||||
`Hotkey:Abort` configures a global discard shortcut. The default is `Ctrl+Alt+D`. Aborting an active recording stops capture/transcription, deletes the meeting note, transcript, assistant context, summary file if present, and linked screenshot attachments for that run, and skips automatic summary generation.
|
||||
|
||||
Meeting note, transcript, assistant context, and summary artifacts use frontmatter links so they backlink to each other. Meeting note frontmatter includes `start_time` when recording starts and `end_time` when transcription processing finishes. Transcript and summary frontmatter copy the meeting title, start time, and end time from the meeting note; if the meeting has no title, the summary agent can provide one when writing the summary.
|
||||
|
||||
`Recording:MaxMetadataAttendeeImportCount` limits how many attendees Outlook metadata enrichment imports into meeting-note frontmatter. The default is `30`; when an appointment has more attendees than that, Meeting Assistant still imports title, agenda, and scheduled end time, but leaves attendees empty because large invites are usually presentation-style meetings.
|
||||
|
||||
Assistant context notes keep their artifact links in frontmatter and expose a lifecycle `state`: `collecting metadata`, `transcribing`, `speaker recognition`, `summarizing`, `finished`, or `error`. Meeting Assistant updates this state as metadata lookup, recording, final speaker recognition, and summary generation progress.
|
||||
|
||||
`ProjectsFolder` contains project knowledge. Each direct subfolder is treated as one project, and a meeting binds projects by listing those subfolder names in the meeting note `projects` frontmatter.
|
||||
@@ -217,6 +238,10 @@ See `docs/meeting-workflow-engine.md` for the detailed YAML format, supported va
|
||||
|
||||
`POST /diagnostics/workflow/reload` reloads application configuration so workflow automation settings such as `Automation:RulesPath` can be changed without restarting the application. A meeting run that already captured its options keeps using those options.
|
||||
|
||||
`Screenshots:Hotkey` configures a global hotkey that captures the currently active window during an active meeting. Screenshots are written under `Screenshots:AttachmentsFolder`, which defaults to an `Attachments` folder beside the assistant context note, and each capture appends a timestamped markdown image link to the assistant context. Launch profiles can override the screenshot hotkey with `LaunchProfiles:{profile}:Screenshots:Hotkey`; only explicitly configured profile hotkeys are registered, so profile-specific hotkeys do not silently inherit and collide with the default profile.
|
||||
|
||||
`Screenshots:Ocr` optionally enables vision extraction for screenshots. When `Enabled` is true, Meeting Assistant sends the screenshot and prompt to an OpenAI-compatible Responses endpoint and replaces the screenshot OCR placeholder with the model output. Blank `Endpoint` or `Model` values fall back to the summary `Agent` endpoint and model, which keeps local OCR on the same LiteLLM backend as summarization. `Key` or `KeyEnv` can be set specifically for OCR; otherwise the summary agent key configuration is used. The default prompt asks the model to return pixel crop coordinates when it can confidently isolate only the presentation or shared-screen content; valid crop boxes are saved as `*-cropped.png` beside the original screenshot and linked before the OCR block. Automatic summarization waits for pending screenshot OCR work to complete or hit `Timeout` before the assistant context moves to `summarizing`.
|
||||
|
||||
`Agent` configures the Microsoft Agent Framework summary pipeline. `Key` may be set directly for local testing, but `KeyEnv` is preferred; by default the API key is read from `LITELLM_API_KEY`. After transcription has fully finished, Meeting Assistant automatically runs the summary pipeline for the meeting. `ReconnectionAttempts` and `ReconnectionDelay` control retries for transient model endpoint failures such as LiteLLM token refresh or 5xx responses. If summary generation still fails, Meeting Assistant overwrites the configured summary note with a markdown failure report containing the exception details and linked meeting artifacts.
|
||||
|
||||
`ContextWindowTokens`, `MaxOutputTokens`, `EnableCompaction`, and `CompactionRemainingRatio` configure summary-agent context monitoring. Meeting Assistant estimates the outgoing Responses payload size, logs the estimated token count and remaining context, and compacts the conversation when only the configured remaining ratio is left. It first attempts `POST /v1/{ResponsesCompactPath}` on the configured OpenAI-compatible endpoint. If that endpoint is unavailable or returns invalid data, it falls back to Microsoft Agent Framework compaction: collapse old tool results, summarize older message groups, preserve only the last four user turns if needed, and finally drop oldest groups until the request is back under budget.
|
||||
@@ -235,6 +260,8 @@ Failure summary notes include a clickable retry link using `Api:PublicBaseUrl`,
|
||||
|
||||
The summary agent currently has meeting-scoped tools to read the transcript, full meeting note including frontmatter, assistant context, user notes, glossary, and project files. Every read tool that returns file-like content can read the whole content or a clamped inclusive 1-based line range with `from` and `to`, so the agent can inspect large transcripts incrementally. The assistant context note is the agent's notebook and user-visible context window; the agent can write internal notes, discovered context, missing tool requests, suggested improvements, or follow-up task ideas there using line-based overwrite, replace, and insert modes. Project lookup and search tools are constrained to the projects listed in the meeting note frontmatter:
|
||||
|
||||
When assistant context contains cropped screenshot links produced by OCR, the summary instructions tell the agent to include only the most relevant cropped screenshots in the summary and markdown-link them near the related summary text.
|
||||
|
||||
- `read_meetingnote`
|
||||
- `list_projects`
|
||||
- `list_projectfiles`
|
||||
|
||||
@@ -176,7 +176,10 @@ steps:
|
||||
value: "Known speaker identified: @Model.Speaker.Name"
|
||||
```
|
||||
|
||||
The engine only invokes Razor when the value contains `@`.
|
||||
The engine invokes Razor when the value contains an unescaped `@` that is not part of a
|
||||
valid email address token. Literal email addresses such as `Support@contoso.com` are left
|
||||
unchanged; if the same value also contains a Razor expression, the email `@` is escaped
|
||||
before rendering so the address still appears normally in the rendered result.
|
||||
|
||||
## Steps
|
||||
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
# Change: Add meeting screenshots
|
||||
|
||||
## Why
|
||||
Meetings often contain visual information that is not captured by audio transcription, such as slides, diagrams, shared screens, and active presenter context. Meeting Assistant should let the user capture the active window during a meeting and store that visual context alongside the assistant context note.
|
||||
|
||||
## What Changes
|
||||
- Add a configurable screenshot hotkey.
|
||||
- Capture the currently active window and save the image under the assistant context attachments folder.
|
||||
- Append a timestamped markdown image link to the assistant context note so the screenshot can be correlated with the transcript timeline.
|
||||
- Optionally send screenshots to a configured OCR/vision model and append the extracted result next to the screenshot entry.
|
||||
- Delay summary generation until pending screenshot OCR jobs finish or time out.
|
||||
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
## MODIFIED Requirements
|
||||
### Requirement: Meeting summary artifacts include assistant context
|
||||
Meeting Assistant SHALL generate a summary, decisions, and next steps from the meeting transcript, metadata, user notes, and assistant-discovered context.
|
||||
|
||||
Meeting Assistant SHALL use a Microsoft Agent Framework pipeline for the first summary implementation. The pipeline SHALL use a configurable OpenAI-compatible endpoint, model, and API key source, including support for a direct key or an environment variable name.
|
||||
|
||||
The summary pipeline SHALL retry transient model endpoint failures according to configurable reconnection attempts and delay settings.
|
||||
|
||||
The summary pipeline SHALL track context-window usage for the configured model using response usage when available and request-size estimates otherwise. The context-window limit, maximum output reserve, compaction enablement, compaction threshold, and Responses compact endpoint path SHALL be configurable.
|
||||
|
||||
When only the configured remaining context ratio is available, the summary pipeline SHALL try to compact the conversation through the configured OpenAI-compatible `POST /v1/responses/compact` endpoint. If that endpoint is unavailable or returns invalid data, it SHALL fall back to Microsoft Agent Framework compaction.
|
||||
|
||||
When summary generation fails after retries, Meeting Assistant SHALL write a markdown failure report to the configured summary note path. The failure report SHALL include a clickable retry link, error details, and the meeting artifact paths needed to diagnose or retry the run.
|
||||
|
||||
After transcript processing finishes for a recording, Meeting Assistant SHALL automatically invoke the summary pipeline for that meeting.
|
||||
|
||||
Before automatic summary generation starts, Meeting Assistant SHALL wait for pending screenshot OCR jobs for that meeting to complete or reach their configured timeout.
|
||||
|
||||
Meeting Assistant SHALL expose an API operation to retry summary generation for a given summary note path. The retry operation SHALL resolve the linked meeting artifacts from the meeting note frontmatter and SHALL overwrite the existing summary note with either the new summary or a new failure report.
|
||||
|
||||
The summary pipeline SHALL expose tools scoped to the current meeting:
|
||||
|
||||
- `read_meetingnote`
|
||||
- `read_transcript`
|
||||
- `read_context`
|
||||
- `read_usernotes`
|
||||
- `read_glossary`
|
||||
- `add_dictation_word`
|
||||
- `write_context`
|
||||
- `write_summary`
|
||||
- `list_projects`
|
||||
- `list_projectfiles`
|
||||
- `read_projectfile`
|
||||
- `write_projectfile`
|
||||
- `search`
|
||||
|
||||
All summary-agent read tools that return file-like content SHALL support either reading the whole file or reading a clamped inclusive 1-based line range when `from` and `to` are supplied.
|
||||
|
||||
The summary pipeline SHALL write the summary note as a markdown artifact linked to the meeting note, transcript, and assistant context.
|
||||
|
||||
The summary instructions SHALL tell the summary agent to include only the most relevant cropped screenshot links from assistant context in the summary, placing them near the related summary text.
|
||||
|
||||
The summary agent SHALL be able to read the meeting note, transcript, assistant context, glossary, and bound project files through tools.
|
||||
|
||||
The summary agent SHALL be able to write the summary and assistant context files as its owned artifacts.
|
||||
|
||||
For summary-agent writes to any file other than the summary file and assistant context file, Meeting Assistant SHALL record an in-memory diff containing removed and added lines.
|
||||
|
||||
When the summary agent finishes, Meeting Assistant SHALL append the collected external write diffs to the assistant context file.
|
||||
|
||||
After writing the meeting summary, the summary agent SHALL update existing project files when the meeting produced project-relevant knowledge.
|
||||
|
||||
The assistant context note SHALL keep `title`, `start_time`, `end_time`, `meeting`, `transcript`, `summary`, `state`, and `agenda` in frontmatter. Meeting Assistant SHALL write `title` and `start_time` when the assistant context note is created, update `title` when later meeting metadata is discovered, and write `end_time` when transcript processing stops. When a Teams appointment is detected at recording start, the assistant context note SHALL also keep `scheduled_end` in frontmatter. The `state` value SHALL be one of `collecting metadata`, `transcribing`, `speaker recognition`, `summarizing`, `finished`, or `error`.
|
||||
|
||||
The summary note SHALL keep frontmatter links to meeting artifacts and SHALL copy `title`, `start_time`, and `end_time` from the meeting note when available. If the meeting note has no title, the summary agent SHALL be able to provide a title when writing the summary.
|
||||
|
||||
The summary agent SHALL be able to read and write the assistant context body as its own notebook using overwrite, replace, and insert line modes.
|
||||
|
||||
#### Scenario: Assistant context note is created
|
||||
- **WHEN** Meeting Assistant creates a meeting session
|
||||
- **THEN** it creates the assistant context note with frontmatter links to the meeting note, transcript note, and summary note
|
||||
- **AND** the assistant context state is `collecting metadata`
|
||||
|
||||
#### Scenario: Assistant context state tracks processing
|
||||
- **WHEN** transcription, final speaker recognition, and summary generation progress
|
||||
- **THEN** Meeting Assistant updates the assistant context state through `transcribing`, `speaker recognition`, `summarizing`, `finished`, or `error`
|
||||
|
||||
#### Scenario: Summary waits for pending screenshot OCR
|
||||
- **GIVEN** screenshot OCR is still running for a meeting
|
||||
- **WHEN** transcript processing is ready to run the automatic summary
|
||||
- **THEN** Meeting Assistant waits for the screenshot OCR to complete before moving the assistant context to `summarizing`
|
||||
- **AND** proceeds after the configured OCR timeout if OCR does not finish
|
||||
|
||||
#### Scenario: Summary can include relevant cropped screenshots
|
||||
- **GIVEN** assistant context contains cropped screenshot markdown links
|
||||
- **WHEN** Meeting Assistant builds summary-agent instructions
|
||||
- **THEN** the instructions tell the summary agent to include only the most relevant cropped screenshots in the summary
|
||||
- **AND** to markdown-link them near the related summary text
|
||||
|
||||
### Requirement: Meeting screenshots are captured into assistant context
|
||||
Meeting Assistant SHALL expose a configurable screenshot hotkey.
|
||||
|
||||
When a meeting is active and the screenshot hotkey is pressed, Meeting Assistant SHALL capture the currently active window.
|
||||
|
||||
The screenshot image SHALL be saved into a configurable attachments folder for the assistant context note. By default, the folder SHALL be `Attachments` beside the assistant context note.
|
||||
|
||||
After the image is saved, Meeting Assistant SHALL append a markdown image link to the assistant context note with a meeting-relative timestamp that correlates to transcript timestamps.
|
||||
|
||||
Meeting Assistant SHALL allow optional screenshot OCR configuration with endpoint URL, API key or key environment variable, model, prompt, and timeout.
|
||||
|
||||
When screenshot OCR is configured, Meeting Assistant SHALL send the screenshot and prompt to the configured OpenAI-compatible Responses endpoint and append the model result after the screenshot link in the assistant context note.
|
||||
|
||||
The screenshot OCR prompt SHALL ask the model to return pixel crop coordinates when it can confidently isolate only the presentation, shared screen, or similarly relevant meeting content.
|
||||
|
||||
When OCR returns valid crop coordinates within the original image bounds, Meeting Assistant SHALL save a cropped PNG beside the original screenshot and SHALL link the cropped image before the OCR result in the assistant context note.
|
||||
|
||||
When OCR returns no crop coordinates or invalid crop coordinates, Meeting Assistant SHALL keep the original screenshot link and OCR result without writing a cropped image.
|
||||
|
||||
When screenshot OCR is not configured, Meeting Assistant SHALL skip OCR and keep the screenshot link.
|
||||
|
||||
The default OCR prompt SHALL explain that the image is from a meeting and ask the model to identify who is talking, who is presenting, what is presented, capture slide text in markdown, convert diagrams to Mermaid when possible, return crop coordinates only for confidently isolated presentation/shared-screen content, and otherwise describe the scene.
|
||||
|
||||
#### Scenario: Screenshot is linked with meeting timestamp
|
||||
- **GIVEN** a meeting started at `10:00:00`
|
||||
- **WHEN** the user captures a screenshot at `10:03:05`
|
||||
- **THEN** Meeting Assistant saves the screenshot under the configured attachments folder
|
||||
- **AND** appends a markdown image link to assistant context with timestamp `[00:03:05]`
|
||||
|
||||
#### Scenario: OCR result is appended after screenshot
|
||||
- **GIVEN** screenshot OCR is configured
|
||||
- **WHEN** the user captures a screenshot
|
||||
- **THEN** Meeting Assistant appends the screenshot link to assistant context
|
||||
- **AND** appends the OCR result for that screenshot after the link when processing completes
|
||||
|
||||
#### Scenario: OCR crop is saved and linked before OCR text
|
||||
- **GIVEN** screenshot OCR is configured
|
||||
- **AND** OCR returns valid crop coordinates for a shared screen
|
||||
- **WHEN** OCR processing completes
|
||||
- **THEN** Meeting Assistant saves a cropped screenshot beside the original image
|
||||
- **AND** links the cropped screenshot before the OCR text in assistant context
|
||||
|
||||
#### Scenario: OCR is skipped when not configured
|
||||
- **GIVEN** screenshot OCR is not configured
|
||||
- **WHEN** the user captures a screenshot
|
||||
- **THEN** Meeting Assistant saves and links the screenshot without calling a model endpoint
|
||||
@@ -0,0 +1,14 @@
|
||||
## 1. Implementation
|
||||
- [x] 1.1 Add screenshot and OCR configuration options.
|
||||
- [x] 1.2 Capture active-window screenshots through a Windows adapter and testable abstraction.
|
||||
- [x] 1.3 Save screenshots to the configured assistant-context attachments folder.
|
||||
- [x] 1.4 Append timestamped screenshot markdown blocks to assistant context.
|
||||
- [x] 1.5 Add optional OCR/vision processing with timeout and default prompt.
|
||||
- [x] 1.6 Wait for pending screenshot OCR before summary generation.
|
||||
- [x] 1.7 Register a configurable screenshot hotkey.
|
||||
|
||||
## 2. Verification
|
||||
- [x] 2.1 Cover screenshot save/link behavior.
|
||||
- [x] 2.2 Cover disabled and configured OCR behavior.
|
||||
- [x] 2.3 Cover summary waiting for pending OCR.
|
||||
- [x] 2.4 Run relevant tests and strict OpenSpec validation.
|
||||
@@ -0,0 +1,14 @@
|
||||
# Change: Add recording abort hotkey
|
||||
|
||||
## Why
|
||||
Users need a fast way to discard an accidental or unwanted active meeting recording without producing a transcript summary or leaving meeting artifacts in the vault.
|
||||
|
||||
## What Changes
|
||||
- Add a configurable abort/discard hotkey, defaulting to `Ctrl+Alt+D`.
|
||||
- Add an abort recording operation that stops active capture/transcription, skips final summary generation, and removes the meeting artifacts created for that run.
|
||||
- Keep normal stop behavior unchanged.
|
||||
|
||||
## Impact
|
||||
- Recording coordinator abort path
|
||||
- Global hotkey registration and launch-profile hotkey handling
|
||||
- Recording control endpoints and local configuration
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
## MODIFIED Requirements
|
||||
### Requirement: Speech recognition pipelines are streamable and configurable
|
||||
Meeting Assistant SHALL route audio through a provider-independent speech recognition pipeline contract.
|
||||
|
||||
The configured pipeline SHALL be selected through DI from the ASR backend configuration and SHALL expose operations to initialize, wait for readiness, write captured audio chunks, read live transcript segments, and read the finished transcript after capture completes.
|
||||
|
||||
Meeting Assistant SHALL expose a configurable abort/discard hotkey for an active recording.
|
||||
|
||||
When aborting an active recording, Meeting Assistant SHALL stop active audio capture and transcription, SHALL remove the meeting note, transcript, assistant context, summary artifact, and linked screenshot attachment files created for that run, and SHALL NOT invoke automatic summary generation.
|
||||
|
||||
When no recording is active, aborting SHALL leave the current recording status unchanged.
|
||||
|
||||
#### Scenario: Active recording is aborted and discarded
|
||||
- **GIVEN** a meeting recording is active and has created meeting artifacts
|
||||
- **WHEN** the user aborts the recording
|
||||
- **THEN** Meeting Assistant stops recording
|
||||
- **AND** deletes the meeting note, transcript, assistant context, summary artifact if present, and linked screenshot attachment files for that run
|
||||
- **AND** does not invoke the summary pipeline
|
||||
|
||||
#### Scenario: Abort does nothing when idle
|
||||
- **GIVEN** no meeting recording is active
|
||||
- **WHEN** the user aborts recording
|
||||
- **THEN** Meeting Assistant remains idle
|
||||
@@ -0,0 +1,10 @@
|
||||
## 1. Implementation
|
||||
- [x] 1.1 Add configurable abort hotkey with launch-profile registration.
|
||||
- [x] 1.2 Add recording abort operation/API that stops active capture and transcription.
|
||||
- [x] 1.3 Delete meeting artifacts and linked screenshot attachments for aborted runs.
|
||||
- [x] 1.4 Ensure abort does not transition to summarizing or invoke the summary pipeline.
|
||||
|
||||
## 2. Verification
|
||||
- [x] 2.1 Cover abort deleting artifacts and skipping summary.
|
||||
- [x] 2.2 Cover abort hotkey registration.
|
||||
- [x] 2.3 Run relevant tests and strict OpenSpec validation.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Change: Limit Outlook metadata attendee import
|
||||
|
||||
## Why
|
||||
Large Outlook meetings often represent presentations or broadcasts. Importing every invited attendee into the meeting note makes speaker matching noisy because most invitees are unlikely to speak.
|
||||
|
||||
## What Changes
|
||||
- Add a configurable maximum attendee count for meeting metadata imports.
|
||||
- Keep applying meeting title, agenda, and scheduled end metadata when the attendee list is too large.
|
||||
- Skip writing metadata attendees when the raw metadata attendee count is above the configured maximum.
|
||||
|
||||
## Impact
|
||||
- Affects meeting-session metadata enrichment behavior.
|
||||
- Reduces false speaker candidates for large presentation-style meetings.
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Windows Outlook enrichment is optional
|
||||
Meeting Assistant SHALL gate Outlook Classic COM enrichment behind the Windows compilation target.
|
||||
|
||||
When the Windows build starts a meeting and Outlook Classic has exactly one current Teams appointment, Meeting Assistant SHALL copy the appointment title to the meeting note and copy the appointment agenda and scheduled end time to the assistant-context frontmatter.
|
||||
|
||||
Meeting Assistant SHALL copy the appointment attendees to the meeting note only when the raw appointment attendee count is less than or equal to the configured `Recording:MaxMetadataAttendeeImportCount`. The default maximum SHALL be 30 attendees.
|
||||
|
||||
The agenda SHALL be extracted from the appointment body content before the Teams join separator or Teams join text.
|
||||
|
||||
#### Scenario: Current Teams appointment enriches meeting artifacts
|
||||
- **WHEN** a Windows build starts a meeting while Outlook Classic exposes exactly one current Teams appointment
|
||||
- **THEN** Meeting Assistant uses the appointment subject as the meeting title
|
||||
- **AND** writes the appointment attendees into meeting note frontmatter when the raw attendee count is within the configured import limit
|
||||
- **AND** writes the appointment agenda into assistant-context frontmatter
|
||||
- **AND** writes the appointment end time as `scheduled_end` into assistant-context frontmatter
|
||||
|
||||
#### Scenario: Oversized attendee list is not imported
|
||||
- **GIVEN** the configured metadata attendee import limit is 30
|
||||
- **WHEN** a Windows build starts a meeting while Outlook Classic exposes exactly one current Teams appointment with 31 attendees
|
||||
- **THEN** Meeting Assistant uses the appointment subject as the meeting title
|
||||
- **AND** does not write the appointment attendees into meeting note frontmatter
|
||||
- **AND** writes the appointment agenda into assistant-context frontmatter
|
||||
- **AND** writes the appointment end time as `scheduled_end` into assistant-context frontmatter
|
||||
|
||||
#### Scenario: Outlook is unavailable or ambiguous
|
||||
- **WHEN** Outlook Classic is unavailable or more than one current Teams appointment is found
|
||||
- **THEN** Meeting Assistant starts the recording with the default generated meeting title and empty agenda
|
||||
- **AND** omits `scheduled_end` from assistant-context frontmatter
|
||||
@@ -0,0 +1,9 @@
|
||||
## 1. Behavior
|
||||
- [x] 1.1 Add meeting-session requirement for capped metadata attendee import.
|
||||
- [x] 1.2 Add behavior test proving oversized metadata attendee lists are skipped.
|
||||
- [x] 1.3 Add configurable default maximum attendee count.
|
||||
- [x] 1.4 Preserve title, agenda, and scheduled end enrichment when attendees are skipped.
|
||||
|
||||
## 2. Verification
|
||||
- [x] 2.1 Run focused recording coordinator tests.
|
||||
- [x] 2.2 Run `openspec validate limit-outlook-metadata-attendees --strict`.
|
||||
@@ -6,6 +6,12 @@ TBD - created by archiving change define-meeting-assistant-v1. Update Purpose af
|
||||
### Requirement: Recording mode is controlled by a configurable hotkey
|
||||
Meeting Assistant SHALL use normal .NET configuration to define the global hotkey that toggles recording/transcription mode.
|
||||
|
||||
Meeting Assistant SHALL expose a configurable abort/discard hotkey for an active recording.
|
||||
|
||||
When aborting an active recording, Meeting Assistant SHALL stop active audio capture and transcription, SHALL remove the meeting note, transcript, assistant context, summary artifact, and linked screenshot attachment files created for that run, and SHALL NOT invoke automatic summary generation.
|
||||
|
||||
When no recording is active, aborting SHALL leave the current recording status unchanged.
|
||||
|
||||
#### Scenario: Hotkey starts recording
|
||||
- **WHEN** the configured hotkey is pressed while recording mode is inactive
|
||||
- **THEN** Meeting Assistant starts recording/transcription mode
|
||||
@@ -14,6 +20,18 @@ Meeting Assistant SHALL use normal .NET configuration to define the global hotke
|
||||
- **WHEN** the configured hotkey is pressed while recording mode is active
|
||||
- **THEN** Meeting Assistant stops recording/transcription mode
|
||||
|
||||
#### Scenario: Active recording is aborted and discarded
|
||||
- **GIVEN** a meeting recording is active and has created meeting artifacts
|
||||
- **WHEN** the user aborts the recording
|
||||
- **THEN** Meeting Assistant stops recording
|
||||
- **AND** deletes the meeting note, transcript, assistant context, summary artifact if present, and linked screenshot attachment files for that run
|
||||
- **AND** does not invoke the summary pipeline
|
||||
|
||||
#### Scenario: Abort does nothing when idle
|
||||
- **GIVEN** no meeting recording is active
|
||||
- **WHEN** the user aborts recording
|
||||
- **THEN** Meeting Assistant remains idle
|
||||
|
||||
### Requirement: Recording mode captures microphone and computer output
|
||||
Meeting Assistant SHALL capture microphone input and computer output and combine them into one audio stream for transcription.
|
||||
|
||||
@@ -89,4 +107,3 @@ Meeting Assistant SHALL use the selected launch profile's summary-agent settings
|
||||
- **GIVEN** two launch profiles configure the same toggle hotkey
|
||||
- **WHEN** Meeting Assistant resolves launch profiles
|
||||
- **THEN** Meeting Assistant rejects the configuration before registering global hotkeys
|
||||
|
||||
|
||||
@@ -68,14 +68,24 @@ Meeting software integrations MAY augment the meeting experience, but they SHALL
|
||||
### Requirement: Windows Outlook enrichment is optional
|
||||
Meeting Assistant SHALL gate Outlook Classic COM enrichment behind the Windows compilation target.
|
||||
|
||||
When the Windows build starts a meeting and Outlook Classic has exactly one current Teams appointment, Meeting Assistant SHALL copy the appointment title and attendees to the meeting note and copy the appointment agenda and scheduled end time to the assistant-context frontmatter.
|
||||
When the Windows build starts a meeting and Outlook Classic has exactly one current Teams appointment, Meeting Assistant SHALL copy the appointment title to the meeting note and copy the appointment agenda and scheduled end time to the assistant-context frontmatter.
|
||||
|
||||
Meeting Assistant SHALL copy the appointment attendees to the meeting note only when the raw appointment attendee count is less than or equal to the configured `Recording:MaxMetadataAttendeeImportCount`. The default maximum SHALL be 30 attendees.
|
||||
|
||||
The agenda SHALL be extracted from the appointment body content before the Teams join separator or Teams join text.
|
||||
|
||||
#### Scenario: Current Teams appointment enriches meeting artifacts
|
||||
- **WHEN** a Windows build starts a meeting while Outlook Classic exposes exactly one current Teams appointment
|
||||
- **THEN** Meeting Assistant uses the appointment subject as the meeting title
|
||||
- **AND** writes the appointment attendees into meeting note frontmatter
|
||||
- **AND** writes the appointment attendees into meeting note frontmatter when the raw attendee count is within the configured import limit
|
||||
- **AND** writes the appointment agenda into assistant-context frontmatter
|
||||
- **AND** writes the appointment end time as `scheduled_end` into assistant-context frontmatter
|
||||
|
||||
#### Scenario: Oversized attendee list is not imported
|
||||
- **GIVEN** the configured metadata attendee import limit is 30
|
||||
- **WHEN** a Windows build starts a meeting while Outlook Classic exposes exactly one current Teams appointment with 31 attendees
|
||||
- **THEN** Meeting Assistant uses the appointment subject as the meeting title
|
||||
- **AND** does not write the appointment attendees into meeting note frontmatter
|
||||
- **AND** writes the appointment agenda into assistant-context frontmatter
|
||||
- **AND** writes the appointment end time as `scheduled_end` into assistant-context frontmatter
|
||||
|
||||
@@ -84,3 +94,91 @@ The agenda SHALL be extracted from the appointment body content before the Teams
|
||||
- **THEN** Meeting Assistant starts the recording with the default generated meeting title and empty agenda
|
||||
- **AND** omits `scheduled_end` from assistant-context frontmatter
|
||||
|
||||
### Requirement: Meeting automation rules are configurable
|
||||
Meeting Assistant SHALL allow an optional local YAML rules file path to be configured.
|
||||
|
||||
When the configured rules file is missing or blank, Meeting Assistant SHALL continue without applying automation rules.
|
||||
|
||||
The local rules file SHALL be ignored by source control.
|
||||
|
||||
Rules SHALL be evaluated against the latest meeting note data read from disk for each event.
|
||||
|
||||
Meeting Assistant SHALL expose a diagnostic endpoint that reloads application configuration so workflow automation configuration changes can be picked up without restarting the application.
|
||||
|
||||
#### Scenario: Missing rules file is ignored
|
||||
- **WHEN** Meeting Assistant handles a meeting event and no configured rules file exists
|
||||
- **THEN** it leaves the meeting note and assistant context unchanged
|
||||
|
||||
#### Scenario: Created rule adds default attendee
|
||||
- **GIVEN** a configured rule that triggers on meeting creation when `meeting.attendees.count = 0`
|
||||
- **WHEN** Meeting Assistant creates a meeting note without attendees
|
||||
- **THEN** it adds the configured attendee to the meeting note
|
||||
|
||||
#### Scenario: Workflow configuration is reloaded diagnostically
|
||||
- **GIVEN** the workflow automation configuration has changed on disk
|
||||
- **WHEN** the diagnostic workflow reload endpoint is called
|
||||
- **THEN** Meeting Assistant reloads application configuration without restarting
|
||||
- **AND** future workflow events use the reloaded workflow automation configuration
|
||||
|
||||
### Requirement: Meeting automation rules support lifecycle triggers
|
||||
Meeting Assistant SHALL support rule triggers for `created`, `state_transition`, and `speaker_identified`.
|
||||
|
||||
A `state_transition` trigger MAY filter by `from`, `to`, or both state values.
|
||||
|
||||
A `speaker_identified` trigger MAY filter by speaker name.
|
||||
|
||||
#### Scenario: State transition rule matches from and to
|
||||
- **GIVEN** a configured rule that triggers on a state transition from `collecting metadata` to `transcribing`
|
||||
- **WHEN** Meeting Assistant transitions that meeting from `collecting metadata` to `transcribing`
|
||||
- **THEN** it applies the rule steps
|
||||
|
||||
#### Scenario: Speaker identified rule filters by name
|
||||
- **GIVEN** a configured rule that triggers when speaker `Ada` is identified
|
||||
- **WHEN** Meeting Assistant identifies speaker `Grace`
|
||||
- **THEN** it does not apply the rule
|
||||
- **WHEN** Meeting Assistant identifies speaker `Ada`
|
||||
- **THEN** it applies the rule steps
|
||||
|
||||
### Requirement: Meeting automation rules support conditions and steps
|
||||
Meeting Assistant SHALL support rule conditions using an expression engine.
|
||||
|
||||
Rules SHALL support nested `and`, `or`, and `not` condition groups.
|
||||
|
||||
Step values SHALL support Razor syntax against the current meeting event model.
|
||||
|
||||
Step values SHALL treat `@` characters inside valid email address tokens as literal text rather than Razor transitions.
|
||||
|
||||
Meeting Assistant SHALL support these initial rule steps:
|
||||
|
||||
- `add_attendee`
|
||||
- `remove_attendee`
|
||||
- `set_property`
|
||||
- `add_context`
|
||||
- `add_project`
|
||||
|
||||
#### Scenario: Nested conditions choose a matching rule
|
||||
- **GIVEN** a configured rule with nested `and`, `or`, and `not` conditions over meeting title, attendees, and event data
|
||||
- **WHEN** the condition evaluates to true
|
||||
- **THEN** Meeting Assistant applies the rule
|
||||
- **WHEN** the condition evaluates to false
|
||||
- **THEN** Meeting Assistant skips the rule
|
||||
|
||||
#### Scenario: Templated context mentions identified speaker
|
||||
- **GIVEN** a configured `speaker_identified` rule with an `add_context` step using Razor syntax
|
||||
- **WHEN** Meeting Assistant identifies matching speaker `Ada`
|
||||
- **THEN** it appends rendered context text containing `Ada` to the assistant context note
|
||||
|
||||
#### Scenario: Email addresses do not trigger Razor templating
|
||||
- **GIVEN** a configured rule step value containing `Support@contoso.com`
|
||||
- **WHEN** the rule runs
|
||||
- **THEN** Meeting Assistant preserves the email address as literal text
|
||||
|
||||
#### Scenario: Email addresses can appear beside Razor templating
|
||||
- **GIVEN** a configured rule step value containing both `Support@contoso.com` and a Razor expression
|
||||
- **WHEN** the rule runs
|
||||
- **THEN** Meeting Assistant renders the Razor expression and preserves the email address as literal text
|
||||
|
||||
#### Scenario: Rule can clean a meeting title
|
||||
- **GIVEN** a configured state-transition rule that matches a title containing a configured marker
|
||||
- **WHEN** the rule runs
|
||||
- **THEN** Meeting Assistant can update the meeting title through `set_property`
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
## Purpose
|
||||
TBD - created by archiving change define-meeting-assistant-v1. Update Purpose after archive.
|
||||
|
||||
## Requirements
|
||||
### Requirement: Meeting Assistant generates meeting outputs
|
||||
Meeting Assistant SHALL generate a summary, decisions, and next steps from the meeting transcript, metadata, user notes, and assistant-discovered context.
|
||||
@@ -18,7 +19,9 @@ Fallback compaction SHALL become increasingly aggressive: collapse old tool resu
|
||||
|
||||
When summary generation fails after retries, Meeting Assistant SHALL write a markdown failure report to the configured summary note path. The failure report SHALL include a clickable retry link, error details, and the meeting artifact paths needed to diagnose or retry the run.
|
||||
|
||||
After transcript processing finishes for a recording, Meeting Assistant SHALL automatically invoke the summary pipeline for that meeting.
|
||||
After transcript processing finishes for a recording, Meeting Assistant SHALL automatically invoke the summary pipeline for the meeting.
|
||||
|
||||
Before automatic summary generation starts, Meeting Assistant SHALL wait for pending screenshot OCR jobs for that meeting to complete or reach their configured timeout.
|
||||
|
||||
Meeting Assistant SHALL expose an API operation to retry summary generation for a given summary note path. The retry operation SHALL resolve the linked meeting artifacts from the meeting note frontmatter and SHALL overwrite the existing summary note with either the new summary or a new failure report.
|
||||
|
||||
@@ -29,6 +32,7 @@ The summary pipeline SHALL expose tools scoped to the current meeting:
|
||||
- `read_context`
|
||||
- `read_usernotes`
|
||||
- `read_glossary`
|
||||
- `add_dictation_word`
|
||||
- `write_summary`
|
||||
- `write_context`
|
||||
- `list_projects`
|
||||
@@ -41,6 +45,8 @@ All summary-agent read tools that return file-like content SHALL support either
|
||||
|
||||
The summary pipeline SHALL write the summary note as a markdown artifact linked to the meeting note, transcript, and assistant context.
|
||||
|
||||
The summary instructions SHALL tell the summary agent to include only the most relevant cropped screenshot links from assistant context in the summary, placing them near the related summary text.
|
||||
|
||||
The summary agent SHALL be able to read the meeting note, transcript, assistant context, glossary, and bound project files through tools.
|
||||
|
||||
The summary agent SHALL be able to write the summary and assistant context files as its owned artifacts.
|
||||
@@ -51,11 +57,13 @@ Meeting Assistant SHALL use a diff implementation that can reduce whole-file rew
|
||||
|
||||
When the summary agent finishes, Meeting Assistant SHALL append the collected external write diffs to the assistant context file.
|
||||
|
||||
After writing the meeting summary, the summary agent SHALL update existing project files when the meeting produced project-relevant knowledge.
|
||||
After writing the meeting summary, the summary agent SHALL update existing project files when the meeting produced durable project knowledge, decisions, next steps, or context.
|
||||
|
||||
The assistant context note SHALL keep `title`, `start_time`, `end_time`, `meeting`, `transcript`, `summary`, `state`, and `agenda` in frontmatter. Meeting Assistant SHALL write `title` and `start_time` when the assistant context note is created, update `title` when later meeting metadata is discovered, and write `end_time` when transcript processing stops. When a Teams appointment is detected at recording start, the assistant context note SHALL also keep `scheduled_end` in frontmatter. The `state` value SHALL be one of `collecting metadata`, `transcribing`, `speaker recognition`, `summarizing`, `finished`, or `error`.
|
||||
|
||||
The summary note SHALL keep frontmatter links to meeting artifacts and SHALL copy `title`, `start_time`, and `end_time` from the meeting note when available. If the meeting note has no title, the summary agent SHALL be able to provide a title when writing the summary.
|
||||
The summary note SHALL keep frontmatter links to meeting artifacts and SHALL copy `title`, `start_time`, `end_time`, and `attendees` from the meeting note when available. If the meeting note has no title, the summary agent SHALL be able to provide a title when writing the summary.
|
||||
|
||||
When the summary note already has an `attendees` frontmatter property, Meeting Assistant SHALL preserve the existing summary attendees instead of overwriting them from the meeting note.
|
||||
|
||||
The summary agent SHALL be able to read and write the assistant context body as its own notebook using overwrite, replace, and insert line modes.
|
||||
|
||||
@@ -71,6 +79,27 @@ The summary agent SHALL be able to read and write the assistant context body as
|
||||
- **THEN** Meeting Assistant updates assistant context frontmatter state to `collecting metadata`, `transcribing`, `speaker recognition`, `summarizing`, `finished`, or `error` as appropriate
|
||||
- **AND** preserves existing `agenda` and `scheduled_end` frontmatter values
|
||||
|
||||
#### Scenario: Summary copies final meeting attendees
|
||||
- **WHEN** the summary agent writes a summary note after meeting attendees have changed
|
||||
- **THEN** the summary note frontmatter includes the final meeting note attendees
|
||||
|
||||
#### Scenario: Existing summary attendees are preserved
|
||||
- **GIVEN** an existing summary note has `attendees` in frontmatter
|
||||
- **WHEN** the summary agent rewrites the summary
|
||||
- **THEN** Meeting Assistant preserves the existing summary attendees
|
||||
|
||||
#### Scenario: Summary waits for pending screenshot OCR
|
||||
- **GIVEN** screenshot OCR is still running for a meeting
|
||||
- **WHEN** transcript processing is ready to run the automatic summary
|
||||
- **THEN** Meeting Assistant waits for the screenshot OCR to complete before moving the assistant context to `summarizing`
|
||||
- **AND** proceeds after the configured OCR timeout if OCR does not finish
|
||||
|
||||
#### Scenario: Summary can include relevant cropped screenshots
|
||||
- **GIVEN** assistant context contains cropped screenshot markdown links
|
||||
- **WHEN** Meeting Assistant builds summary-agent instructions
|
||||
- **THEN** the instructions tell the summary agent to include only the most relevant cropped screenshots in the summary
|
||||
- **AND** to markdown-link them near the related summary text
|
||||
|
||||
#### Scenario: External project write is audited
|
||||
- **WHEN** the summary agent writes a bound project file
|
||||
- **THEN** Meeting Assistant records the changed removed and added lines for that project file
|
||||
@@ -85,6 +114,58 @@ The summary agent SHALL be able to read and write the assistant context body as
|
||||
- **WHEN** the summary agent writes the summary or assistant context file
|
||||
- **THEN** Meeting Assistant does not include those writes in the external write diff audit
|
||||
|
||||
#### Scenario: Assistant context body writes do not duplicate frontmatter
|
||||
- **WHEN** the summary agent writes assistant context content that accidentally includes a markdown frontmatter block
|
||||
- **THEN** Meeting Assistant preserves the existing assistant context frontmatter
|
||||
- **AND** writes only the replacement body content after that frontmatter
|
||||
|
||||
### Requirement: Meeting screenshots are captured into assistant context
|
||||
Meeting Assistant SHALL expose a configurable screenshot hotkey.
|
||||
|
||||
When a meeting is active and the screenshot hotkey is pressed, Meeting Assistant SHALL capture the currently active window.
|
||||
|
||||
The screenshot image SHALL be saved into a configurable attachments folder for the assistant context note. By default, the folder SHALL be `Attachments` beside the assistant context note.
|
||||
|
||||
After the image is saved, Meeting Assistant SHALL append a markdown image link to the assistant context note with a meeting-relative timestamp that correlates to transcript timestamps.
|
||||
|
||||
Meeting Assistant SHALL allow optional screenshot OCR configuration with endpoint URL, API key or key environment variable, model, prompt, and timeout.
|
||||
|
||||
When screenshot OCR is configured, Meeting Assistant SHALL send the screenshot and prompt to the configured OpenAI-compatible Responses endpoint and append the model result after the screenshot link in the assistant context note.
|
||||
|
||||
The screenshot OCR prompt SHALL ask the model to return pixel crop coordinates when it can confidently isolate only the presentation, shared screen, or similarly relevant meeting content.
|
||||
|
||||
When OCR returns valid crop coordinates within the original image bounds, Meeting Assistant SHALL save a cropped PNG beside the original screenshot and SHALL link the cropped image before the OCR result in the assistant context note.
|
||||
|
||||
When OCR returns no crop coordinates or invalid crop coordinates, Meeting Assistant SHALL keep the original screenshot link and OCR result without writing a cropped image.
|
||||
|
||||
When screenshot OCR is not configured, Meeting Assistant SHALL skip OCR and keep the screenshot link.
|
||||
|
||||
The default OCR prompt SHALL explain that the image is from a meeting and ask the model to identify who is talking, who is presenting, what is presented, capture slide text in markdown, convert diagrams to Mermaid when possible, return crop coordinates only for confidently isolated presentation/shared-screen content, and otherwise describe the scene.
|
||||
|
||||
#### Scenario: Screenshot is linked with meeting timestamp
|
||||
- **GIVEN** a meeting started at `10:00:00`
|
||||
- **WHEN** the user captures a screenshot at `10:03:05`
|
||||
- **THEN** Meeting Assistant saves the screenshot under the configured attachments folder
|
||||
- **AND** appends a markdown image link to assistant context with timestamp `[00:03:05]`
|
||||
|
||||
#### Scenario: OCR result is appended after screenshot
|
||||
- **GIVEN** screenshot OCR is configured
|
||||
- **WHEN** the user captures a screenshot
|
||||
- **THEN** Meeting Assistant appends the screenshot link to assistant context
|
||||
- **AND** appends the OCR result for that screenshot after the link when processing completes
|
||||
|
||||
#### Scenario: OCR crop is saved and linked before OCR text
|
||||
- **GIVEN** screenshot OCR is configured
|
||||
- **AND** OCR returns valid crop coordinates for a shared screen
|
||||
- **WHEN** OCR processing completes
|
||||
- **THEN** Meeting Assistant saves a cropped screenshot beside the original image
|
||||
- **AND** links the cropped screenshot before the OCR text in assistant context
|
||||
|
||||
#### Scenario: OCR is skipped when not configured
|
||||
- **GIVEN** screenshot OCR is not configured
|
||||
- **WHEN** the user captures a screenshot
|
||||
- **THEN** Meeting Assistant saves and links the screenshot without calling a model endpoint
|
||||
|
||||
### Requirement: Summary agent instructions are configurable
|
||||
Meeting Assistant SHALL allow the summary agent initial prompt to be configured through application settings.
|
||||
|
||||
|
||||
@@ -283,6 +283,8 @@ When a match is confirmed and the identity has a canonical name, Meeting Assista
|
||||
|
||||
When a match is confirmed and the matched speaker is not already listed in meeting note attendees by display name or alias, Meeting Assistant SHALL add the speaker display name to the attendee list.
|
||||
|
||||
When a match is confirmed and the meeting note attendees contain both the speaker display name and one or more accepted aliases for that same speaker, Meeting Assistant SHALL remove the alias attendee entries and keep the display name entry.
|
||||
|
||||
When Meeting Assistant writes attendees from calendar metadata, it SHALL match attendee display names exactly against known identity canonical names and aliases, replace matches with the identity display name, and deduplicate attendees that map to the same identity.
|
||||
|
||||
#### Scenario: Finished transcript is relabeled after a confirmed match
|
||||
@@ -295,6 +297,13 @@ When Meeting Assistant writes attendees from calendar metadata, it SHALL match a
|
||||
- **WHEN** a finished transcript has diarized speaker `Guest03` and the matching backend confirms it is `Chris`
|
||||
- **THEN** Meeting Assistant stores the meeting note and transcript file addresses as a reference for `Chris`
|
||||
|
||||
#### Scenario: Confirmed match removes duplicate aliases
|
||||
- **GIVEN** the speaker identity database contains canonical speaker `Christopher` with alias `Chris`
|
||||
- **AND** the meeting note attendees contain both `Christopher` and `Chris <chris@example.com>`
|
||||
- **WHEN** live or final speaker matching confirms a diarized speaker is `Christopher`
|
||||
- **THEN** Meeting Assistant keeps `Christopher` in the meeting note attendees
|
||||
- **AND** removes `Chris <chris@example.com>` from the meeting note attendees
|
||||
|
||||
### Requirement: Speaker matching runs during active transcription
|
||||
Meeting Assistant SHALL start speaker identity matching only after the configured initial transcription duration has elapsed.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user