forked from Manuel/meeting-assistant
feat: add macOS meeting audio capture
This commit is contained in:
@@ -0,0 +1,415 @@
|
|||||||
|
using MeetingAssistant.Recording;
|
||||||
|
using MeetingAssistant.MeetingNotes;
|
||||||
|
using MeetingAssistant.Summary;
|
||||||
|
using MeetingAssistant.Transcription;
|
||||||
|
using Microsoft.AspNetCore.Hosting;
|
||||||
|
using Microsoft.AspNetCore.Mvc.Testing;
|
||||||
|
using Microsoft.AspNetCore.TestHost;
|
||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using System.Net.Http.Json;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Tests;
|
||||||
|
|
||||||
|
public sealed class MacOsMeetingAudioSourceTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task MacOsMicrophoneStreamsNativePcmUsingRunFormat()
|
||||||
|
{
|
||||||
|
var pcm = Pcm16(1_000, -2_000, 3_000, -4_000);
|
||||||
|
var process = new StubMacOsAudioCaptureProcess(pcm, exitCode: 0);
|
||||||
|
var processFactory = new StubMacOsAudioCaptureProcessFactory(process);
|
||||||
|
IMeetingAudioSource source = new MacOsMicrophoneAudioSource(
|
||||||
|
processFactory,
|
||||||
|
NullLogger<MacOsMicrophoneAudioSource>.Instance);
|
||||||
|
var options = new MeetingAssistantOptions
|
||||||
|
{
|
||||||
|
Recording =
|
||||||
|
{
|
||||||
|
SampleRate = 24_000,
|
||||||
|
Channels = 2
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
var chunks = await ReadChunksAsync(source, options);
|
||||||
|
|
||||||
|
var chunk = Assert.Single(chunks);
|
||||||
|
Assert.Equal(pcm, chunk.Pcm);
|
||||||
|
Assert.Equal(24_000, chunk.SampleRate);
|
||||||
|
Assert.Equal(2, chunk.Channels);
|
||||||
|
Assert.Equal(
|
||||||
|
new MacOsAudioCaptureRequest(MacOsAudioCaptureKind.Microphone, 24_000, 2),
|
||||||
|
processFactory.Request);
|
||||||
|
Assert.True(process.Disposed);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task MacOsSystemAudioReportsNativePermissionFailure()
|
||||||
|
{
|
||||||
|
var process = new StubMacOsAudioCaptureProcess(
|
||||||
|
[],
|
||||||
|
exitCode: 77,
|
||||||
|
standardError: "Screen Recording/System Audio permission is required.");
|
||||||
|
IMeetingAudioSource source = new MacOsSystemAudioSource(
|
||||||
|
new StubMacOsAudioCaptureProcessFactory(process),
|
||||||
|
NullLogger<MacOsSystemAudioSource>.Instance);
|
||||||
|
|
||||||
|
var exception = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
|
||||||
|
await ReadChunksAsync(source, new MeetingAssistantOptions()));
|
||||||
|
|
||||||
|
Assert.Contains("Screen Recording/System Audio permission", exception.Message, StringComparison.Ordinal);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void PortableRegistrationUsesMacOsCompositeOnlyForMacOs()
|
||||||
|
{
|
||||||
|
var macServices = new ServiceCollection().AddLogging();
|
||||||
|
var unsupportedServices = new ServiceCollection().AddLogging();
|
||||||
|
|
||||||
|
PortableMeetingAudioRegistration.Add(macServices, isMacOs: true);
|
||||||
|
PortableMeetingAudioRegistration.Add(unsupportedServices, isMacOs: false);
|
||||||
|
|
||||||
|
using var macProvider = macServices.BuildServiceProvider();
|
||||||
|
using var unsupportedProvider = unsupportedServices.BuildServiceProvider();
|
||||||
|
Assert.IsType<CompositeMeetingAudioSource>(macProvider.GetRequiredService<IMeetingAudioSource>());
|
||||||
|
Assert.IsType<UnavailableMeetingAudioSource>(unsupportedProvider.GetRequiredService<IMeetingAudioSource>());
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task RecordingEndpointsCaptureMixedMacOsAudioAndStopNativeProcesses()
|
||||||
|
{
|
||||||
|
var testRoot = Path.Combine(Path.GetTempPath(), $"meeting-assistant-macos-endpoint-{Guid.NewGuid():N}");
|
||||||
|
var nativeProcesses = new EndpointMacOsAudioCaptureProcessFactory(
|
||||||
|
microphonePcm: Pcm16(2_000),
|
||||||
|
systemPcm: Pcm16(10_000));
|
||||||
|
var speechPipelines = new CapturingSpeechRecognitionPipelineFactory();
|
||||||
|
await using var factory = new WebApplicationFactory<Program>().WithWebHostBuilder(builder =>
|
||||||
|
{
|
||||||
|
builder.ConfigureAppConfiguration((_, configuration) =>
|
||||||
|
{
|
||||||
|
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||||
|
{
|
||||||
|
["MeetingAssistant:Vault:BaseFolder"] = testRoot,
|
||||||
|
["MeetingAssistant:Vault:TranscriptsFolder"] = "Transcripts",
|
||||||
|
["MeetingAssistant:Vault:MeetingNotesFolder"] = "Notes",
|
||||||
|
["MeetingAssistant:Vault:AssistantContextFolder"] = "Context",
|
||||||
|
["MeetingAssistant:Vault:SummariesFolder"] = "Summaries",
|
||||||
|
["MeetingAssistant:Vault:ProjectsFolder"] = "Projects",
|
||||||
|
["MeetingAssistant:Recording:TemporaryRecordingsFolder"] = Path.Combine(testRoot, "Recordings"),
|
||||||
|
["MeetingAssistant:Recording:InactivitySafeguard:Enabled"] = "false",
|
||||||
|
["MeetingAssistant:SpeakerIdentification:DatabasePath"] = Path.Combine(testRoot, "speakers.db"),
|
||||||
|
["MeetingAssistant:FunAsr:Backend:Enabled"] = "false"
|
||||||
|
});
|
||||||
|
});
|
||||||
|
builder.ConfigureTestServices(services =>
|
||||||
|
{
|
||||||
|
services.RemoveAll<IMeetingAudioSource>();
|
||||||
|
services.RemoveAll<IMacOsAudioCaptureProcessFactory>();
|
||||||
|
PortableMeetingAudioRegistration.Add(services, isMacOs: true);
|
||||||
|
services.RemoveAll<IMacOsAudioCaptureProcessFactory>();
|
||||||
|
services.AddSingleton<IMacOsAudioCaptureProcessFactory>(nativeProcesses);
|
||||||
|
services.RemoveAll<ISpeechRecognitionPipelineFactory>();
|
||||||
|
services.AddSingleton<ISpeechRecognitionPipelineFactory>(speechPipelines);
|
||||||
|
services.RemoveAll<IMeetingNoteOpener>();
|
||||||
|
services.AddSingleton<IMeetingNoteOpener, NoopMeetingNoteOpener>();
|
||||||
|
services.RemoveAll<IMeetingSummaryPipeline>();
|
||||||
|
services.AddSingleton<IMeetingSummaryPipeline, NoopMeetingSummaryPipeline>();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var client = factory.CreateClient();
|
||||||
|
|
||||||
|
using var startResponse = await client.PostAsync("/recording/start", content: null);
|
||||||
|
var started = await startResponse.Content.ReadFromJsonAsync<RecordingStatus>();
|
||||||
|
var mixedChunk = await speechPipelines.FirstChunk.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||||
|
|
||||||
|
Assert.True(startResponse.IsSuccessStatusCode);
|
||||||
|
Assert.True(started?.IsRecording);
|
||||||
|
Assert.Equal(12_000, BitConverter.ToInt16(mixedChunk.Pcm));
|
||||||
|
|
||||||
|
using var stopResponse = await client.PostAsync("/recording/stop", content: null);
|
||||||
|
var stopped = await stopResponse.Content.ReadFromJsonAsync<RecordingStatus>();
|
||||||
|
|
||||||
|
Assert.True(stopResponse.IsSuccessStatusCode);
|
||||||
|
Assert.False(stopped?.IsRecording);
|
||||||
|
Assert.All(nativeProcesses.Processes, process => Assert.True(process.Terminated));
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (Directory.Exists(testRoot))
|
||||||
|
{
|
||||||
|
Directory.Delete(testRoot, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<IReadOnlyList<AudioChunk>> ReadChunksAsync(
|
||||||
|
IMeetingAudioSource source,
|
||||||
|
MeetingAssistantOptions options)
|
||||||
|
{
|
||||||
|
var chunks = new List<AudioChunk>();
|
||||||
|
await foreach (var chunk in source.CaptureAsync(options, CancellationToken.None))
|
||||||
|
{
|
||||||
|
chunks.Add(chunk);
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunks;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] Pcm16(params short[] samples)
|
||||||
|
{
|
||||||
|
var pcm = new byte[samples.Length * sizeof(short)];
|
||||||
|
Buffer.BlockCopy(samples, 0, pcm, 0, pcm.Length);
|
||||||
|
return pcm;
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class StubMacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory
|
||||||
|
{
|
||||||
|
private readonly IMacOsAudioCaptureProcess process;
|
||||||
|
|
||||||
|
public StubMacOsAudioCaptureProcessFactory(IMacOsAudioCaptureProcess process)
|
||||||
|
{
|
||||||
|
this.process = process;
|
||||||
|
}
|
||||||
|
|
||||||
|
public MacOsAudioCaptureRequest? Request { get; private set; }
|
||||||
|
|
||||||
|
public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request)
|
||||||
|
{
|
||||||
|
Request = request;
|
||||||
|
return process;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class StubMacOsAudioCaptureProcess : IMacOsAudioCaptureProcess
|
||||||
|
{
|
||||||
|
private readonly MemoryStream standardOutput;
|
||||||
|
private readonly string standardError;
|
||||||
|
|
||||||
|
public StubMacOsAudioCaptureProcess(byte[] pcm, int exitCode, string standardError = "")
|
||||||
|
{
|
||||||
|
standardOutput = new MemoryStream(pcm);
|
||||||
|
this.standardError = standardError;
|
||||||
|
ExitCode = exitCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Stream StandardOutput => standardOutput;
|
||||||
|
|
||||||
|
public int ExitCode { get; }
|
||||||
|
|
||||||
|
public bool HasExited => true;
|
||||||
|
|
||||||
|
public bool Disposed { get; private set; }
|
||||||
|
|
||||||
|
public Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return Task.FromResult(standardError);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task WaitForExitAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Terminate()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public ValueTask DisposeAsync()
|
||||||
|
{
|
||||||
|
Disposed = true;
|
||||||
|
standardOutput.Dispose();
|
||||||
|
return ValueTask.CompletedTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class EndpointMacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory
|
||||||
|
{
|
||||||
|
private readonly IReadOnlyDictionary<MacOsAudioCaptureKind, byte[]> pcmByKind;
|
||||||
|
|
||||||
|
public EndpointMacOsAudioCaptureProcessFactory(byte[] microphonePcm, byte[] systemPcm)
|
||||||
|
{
|
||||||
|
pcmByKind = new Dictionary<MacOsAudioCaptureKind, byte[]>
|
||||||
|
{
|
||||||
|
[MacOsAudioCaptureKind.Microphone] = microphonePcm,
|
||||||
|
[MacOsAudioCaptureKind.System] = systemPcm
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<BlockingMacOsAudioCaptureProcess> Processes { get; } = [];
|
||||||
|
|
||||||
|
public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request)
|
||||||
|
{
|
||||||
|
var process = new BlockingMacOsAudioCaptureProcess(pcmByKind[request.Kind]);
|
||||||
|
Processes.Add(process);
|
||||||
|
return process;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class BlockingMacOsAudioCaptureProcess : IMacOsAudioCaptureProcess
|
||||||
|
{
|
||||||
|
private readonly TaskCompletionSource terminated = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
|
||||||
|
public BlockingMacOsAudioCaptureProcess(byte[] pcm)
|
||||||
|
{
|
||||||
|
StandardOutput = new InitialPcmThenBlockingStream(pcm, terminated.Task);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Stream StandardOutput { get; }
|
||||||
|
|
||||||
|
public int ExitCode => Terminated ? 137 : throw new InvalidOperationException("Capture is still running.");
|
||||||
|
|
||||||
|
public bool HasExited => Terminated;
|
||||||
|
|
||||||
|
public bool Terminated => terminated.Task.IsCompleted;
|
||||||
|
|
||||||
|
public Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return Task.FromResult("");
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task WaitForExitAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return terminated.Task.WaitAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Terminate()
|
||||||
|
{
|
||||||
|
terminated.TrySetResult();
|
||||||
|
}
|
||||||
|
|
||||||
|
public ValueTask DisposeAsync()
|
||||||
|
{
|
||||||
|
StandardOutput.Dispose();
|
||||||
|
return ValueTask.CompletedTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class InitialPcmThenBlockingStream : Stream
|
||||||
|
{
|
||||||
|
private readonly byte[] pcm;
|
||||||
|
private readonly Task processExit;
|
||||||
|
private bool read;
|
||||||
|
|
||||||
|
public InitialPcmThenBlockingStream(byte[] pcm, Task processExit)
|
||||||
|
{
|
||||||
|
this.pcm = pcm;
|
||||||
|
this.processExit = processExit;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override bool CanRead => true;
|
||||||
|
public override bool CanSeek => false;
|
||||||
|
public override bool CanWrite => false;
|
||||||
|
public override long Length => throw new NotSupportedException();
|
||||||
|
public override long Position
|
||||||
|
{
|
||||||
|
get => throw new NotSupportedException();
|
||||||
|
set => throw new NotSupportedException();
|
||||||
|
}
|
||||||
|
|
||||||
|
public override async ValueTask<int> ReadAsync(
|
||||||
|
Memory<byte> buffer,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
if (!read)
|
||||||
|
{
|
||||||
|
read = true;
|
||||||
|
pcm.AsSpan().CopyTo(buffer.Span);
|
||||||
|
return pcm.Length;
|
||||||
|
}
|
||||||
|
|
||||||
|
await processExit.WaitAsync(cancellationToken);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||||
|
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
|
||||||
|
public override void SetLength(long value) => throw new NotSupportedException();
|
||||||
|
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||||
|
public override void Flush()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class CapturingSpeechRecognitionPipelineFactory : ISpeechRecognitionPipelineFactory
|
||||||
|
{
|
||||||
|
public TaskCompletionSource<AudioChunk> FirstChunk { get; } =
|
||||||
|
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
|
||||||
|
public ISpeechRecognitionPipeline Create()
|
||||||
|
{
|
||||||
|
return new CapturingSpeechRecognitionPipeline(FirstChunk);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class CapturingSpeechRecognitionPipeline : ISpeechRecognitionPipeline
|
||||||
|
{
|
||||||
|
private readonly TaskCompletionSource<AudioChunk> firstChunk;
|
||||||
|
private readonly TaskCompletionSource completed = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
|
||||||
|
public CapturingSpeechRecognitionPipeline(TaskCompletionSource<AudioChunk> firstChunk)
|
||||||
|
{
|
||||||
|
this.firstChunk = firstChunk;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task InitializeAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||||
|
|
||||||
|
public Task InitializeAsync(
|
||||||
|
SpeechRecognitionPipelineOptions options,
|
||||||
|
CancellationToken cancellationToken) => Task.CompletedTask;
|
||||||
|
|
||||||
|
public Task WaitUntilReadyAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||||
|
|
||||||
|
public ValueTask WriteAsync(AudioChunk chunk, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
firstChunk.TrySetResult(chunk);
|
||||||
|
return ValueTask.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task CompleteAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
completed.TrySetResult();
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async IAsyncEnumerable<TranscriptionSegment> ReadLiveTranscriptAsync(
|
||||||
|
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await completed.Task.WaitAsync(cancellationToken);
|
||||||
|
yield break;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<IReadOnlyList<TranscriptionSegment>> ReadFinishedTranscriptAsync(
|
||||||
|
string audioPath,
|
||||||
|
SpeechRecognitionPipelineOptions options,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return Task.FromResult<IReadOnlyList<TranscriptionSegment>>([]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ValueTask DisposeAsync()
|
||||||
|
{
|
||||||
|
completed.TrySetResult();
|
||||||
|
return ValueTask.CompletedTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class NoopMeetingNoteOpener : IMeetingNoteOpener
|
||||||
|
{
|
||||||
|
public Task OpenAsync(string notePath, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class NoopMeetingSummaryPipeline : IMeetingSummaryPipeline
|
||||||
|
{
|
||||||
|
public Task<MeetingSummaryRunResult> RunAsync(
|
||||||
|
MeetingSessionArtifacts artifacts,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return Task.FromResult(new MeetingSummaryRunResult(artifacts.SummaryPath, "", Succeeded: true));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,6 +15,16 @@
|
|||||||
<UseWPF>true</UseWPF>
|
<UseWPF>true</UseWPF>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<PropertyGroup Condition="$([MSBuild]::IsOSPlatform('OSX')) and $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) != 'windows'">
|
||||||
|
<MacOsAudioCaptureEnabled>true</MacOsAudioCaptureEnabled>
|
||||||
|
<MacOsAudioCaptureSource>$(MSBuildProjectDirectory)/Native/MacOsMeetingAudioCapture/main.swift</MacOsAudioCaptureSource>
|
||||||
|
<MacOsAudioCaptureOutputPath>Native/macos-meeting-audio-capture</MacOsAudioCaptureOutputPath>
|
||||||
|
<MacOsAudioCaptureRid Condition="'$(RuntimeIdentifier)' != ''">$(RuntimeIdentifier)</MacOsAudioCaptureRid>
|
||||||
|
<MacOsAudioCaptureRid Condition="'$(MacOsAudioCaptureRid)' == ''">$(NETCoreSdkRuntimeIdentifier)</MacOsAudioCaptureRid>
|
||||||
|
<MacOsAudioCaptureArchitecture Condition="'$(MacOsAudioCaptureRid)' == 'osx-x64'">x86_64</MacOsAudioCaptureArchitecture>
|
||||||
|
<MacOsAudioCaptureArchitecture Condition="'$(MacOsAudioCaptureArchitecture)' == ''">arm64</MacOsAudioCaptureArchitecture>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<InternalsVisibleTo Include="MeetingAssistant.Tests" />
|
<InternalsVisibleTo Include="MeetingAssistant.Tests" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
@@ -61,6 +71,22 @@
|
|||||||
<Copy SourceFiles="@(MeetingAssistantRootConfig)" DestinationFolder="$(PublishDir)" SkipUnchangedFiles="true" />
|
<Copy SourceFiles="@(MeetingAssistantRootConfig)" DestinationFolder="$(PublishDir)" SkipUnchangedFiles="true" />
|
||||||
</Target>
|
</Target>
|
||||||
|
|
||||||
|
<Target
|
||||||
|
Name="BuildMacOsAudioCaptureHelper"
|
||||||
|
AfterTargets="Build"
|
||||||
|
Condition="'$(MacOsAudioCaptureEnabled)' == 'true'">
|
||||||
|
<MakeDir Directories="$(TargetDir)Native" />
|
||||||
|
<Exec Command="/usr/bin/xcrun swiftc -parse-as-library -O -target $(MacOsAudioCaptureArchitecture)-apple-macos13.0 -framework AVFoundation -framework CoreMedia -framework ScreenCaptureKit "$(MacOsAudioCaptureSource)" -o "$(TargetDir)$(MacOsAudioCaptureOutputPath)"" />
|
||||||
|
</Target>
|
||||||
|
|
||||||
|
<Target
|
||||||
|
Name="CopyMacOsAudioCaptureHelperToPublish"
|
||||||
|
AfterTargets="Publish"
|
||||||
|
Condition="'$(MacOsAudioCaptureEnabled)' == 'true'">
|
||||||
|
<MakeDir Directories="$(PublishDir)Native" />
|
||||||
|
<Copy SourceFiles="$(TargetDir)$(MacOsAudioCaptureOutputPath)" DestinationFolder="$(PublishDir)Native" />
|
||||||
|
</Target>
|
||||||
|
|
||||||
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) != 'windows'">
|
<ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) != 'windows'">
|
||||||
<Compile Remove="Hotkeys\GlobalHotkeyService.cs" />
|
<Compile Remove="Hotkeys\GlobalHotkeyService.cs" />
|
||||||
<Compile Remove="Recording\NaudioCaptureSource.cs" />
|
<Compile Remove="Recording\NaudioCaptureSource.cs" />
|
||||||
|
|||||||
@@ -0,0 +1,328 @@
|
|||||||
|
import AVFoundation
|
||||||
|
import CoreMedia
|
||||||
|
import Darwin
|
||||||
|
import Foundation
|
||||||
|
import ScreenCaptureKit
|
||||||
|
|
||||||
|
private enum CaptureFailure: Error, CustomStringConvertible {
|
||||||
|
case usage
|
||||||
|
case invalidFormat
|
||||||
|
case microphonePermission
|
||||||
|
case noDisplay
|
||||||
|
case audioConverter(String)
|
||||||
|
case systemAudio(String)
|
||||||
|
|
||||||
|
var description: String {
|
||||||
|
switch self {
|
||||||
|
case .usage:
|
||||||
|
return "Usage: macos-meeting-audio-capture <microphone|system> <sample-rate> <channels>"
|
||||||
|
case .invalidFormat:
|
||||||
|
return "Sample rate must be positive and channels must be 1 or 2."
|
||||||
|
case .microphonePermission:
|
||||||
|
return "Microphone permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Microphone."
|
||||||
|
case .noDisplay:
|
||||||
|
return "ScreenCaptureKit could not find a display for system-audio capture."
|
||||||
|
case .audioConverter(let detail):
|
||||||
|
return "Native PCM conversion failed: \(detail)"
|
||||||
|
case .systemAudio(let detail):
|
||||||
|
return "Screen Recording/System Audio permission is required. Enable Meeting Assistant under System Settings > Privacy & Security > Screen & System Audio Recording. \(detail)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private final class ErrorReporter: @unchecked Sendable {
|
||||||
|
private let lock = NSLock()
|
||||||
|
|
||||||
|
func write(_ message: String) {
|
||||||
|
lock.lock()
|
||||||
|
defer { lock.unlock() }
|
||||||
|
let line = message.hasSuffix("\n") ? message : message + "\n"
|
||||||
|
if let data = line.data(using: .utf8) {
|
||||||
|
try? FileHandle.standardError.write(contentsOf: data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func exit(with failure: CaptureFailure, code: Int32) -> Never {
|
||||||
|
write(failure.description)
|
||||||
|
Darwin.exit(code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private final class PcmWriter: @unchecked Sendable {
|
||||||
|
private let targetFormat: AVAudioFormat
|
||||||
|
private let output = FileHandle.standardOutput
|
||||||
|
private let reporter: ErrorReporter
|
||||||
|
private let lock = NSLock()
|
||||||
|
private var converter: AVAudioConverter?
|
||||||
|
|
||||||
|
init(sampleRate: Int, channels: Int, reporter: ErrorReporter) throws {
|
||||||
|
guard let format = AVAudioFormat(
|
||||||
|
commonFormat: .pcmFormatInt16,
|
||||||
|
sampleRate: Double(sampleRate),
|
||||||
|
channels: AVAudioChannelCount(channels),
|
||||||
|
interleaved: true
|
||||||
|
) else {
|
||||||
|
throw CaptureFailure.invalidFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
targetFormat = format
|
||||||
|
self.reporter = reporter
|
||||||
|
}
|
||||||
|
|
||||||
|
func write(_ input: AVAudioPCMBuffer) {
|
||||||
|
lock.lock()
|
||||||
|
defer { lock.unlock() }
|
||||||
|
|
||||||
|
do {
|
||||||
|
let outputBuffer = try convert(input)
|
||||||
|
let buffers = UnsafeMutableAudioBufferListPointer(outputBuffer.mutableAudioBufferList)
|
||||||
|
guard let audioBuffer = buffers.first, let bytes = audioBuffer.mData else {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let data = Data(bytes: bytes, count: Int(audioBuffer.mDataByteSize))
|
||||||
|
try output.write(contentsOf: data)
|
||||||
|
} catch {
|
||||||
|
reporter.exit(with: .audioConverter(error.localizedDescription), code: 70)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func convert(_ input: AVAudioPCMBuffer) throws -> AVAudioPCMBuffer {
|
||||||
|
let sourceFormat = input.format
|
||||||
|
let activeConverter: AVAudioConverter
|
||||||
|
if let converter {
|
||||||
|
activeConverter = converter
|
||||||
|
} else {
|
||||||
|
guard let created = AVAudioConverter(from: sourceFormat, to: targetFormat) else {
|
||||||
|
throw CaptureFailure.audioConverter("AVAudioConverter could not convert \(sourceFormat) to \(targetFormat).")
|
||||||
|
}
|
||||||
|
converter = created
|
||||||
|
activeConverter = created
|
||||||
|
}
|
||||||
|
|
||||||
|
let ratio = targetFormat.sampleRate / sourceFormat.sampleRate
|
||||||
|
let capacity = max(1, AVAudioFrameCount(ceil(Double(input.frameLength) * ratio)) + 32)
|
||||||
|
guard let converted = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
|
||||||
|
throw CaptureFailure.audioConverter("Could not allocate the target PCM buffer.")
|
||||||
|
}
|
||||||
|
|
||||||
|
var suppliedInput = false
|
||||||
|
var conversionError: NSError?
|
||||||
|
let status = activeConverter.convert(to: converted, error: &conversionError) { _, inputStatus in
|
||||||
|
if suppliedInput {
|
||||||
|
inputStatus.pointee = .noDataNow
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
suppliedInput = true
|
||||||
|
inputStatus.pointee = .haveData
|
||||||
|
return input
|
||||||
|
}
|
||||||
|
|
||||||
|
if status == .error {
|
||||||
|
throw conversionError ?? CaptureFailure.audioConverter("AVAudioConverter returned an unknown error.")
|
||||||
|
}
|
||||||
|
|
||||||
|
return converted
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private final class MicrophoneCapture: @unchecked Sendable {
|
||||||
|
private let engine = AVAudioEngine()
|
||||||
|
private let writer: PcmWriter
|
||||||
|
|
||||||
|
init(writer: PcmWriter) {
|
||||||
|
self.writer = writer
|
||||||
|
}
|
||||||
|
|
||||||
|
func run() async throws -> Never {
|
||||||
|
let authorized = await withCheckedContinuation { continuation in
|
||||||
|
AVCaptureDevice.requestAccess(for: .audio) { granted in
|
||||||
|
continuation.resume(returning: granted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
guard authorized else {
|
||||||
|
throw CaptureFailure.microphonePermission
|
||||||
|
}
|
||||||
|
|
||||||
|
let input = engine.inputNode
|
||||||
|
let inputFormat = input.outputFormat(forBus: 0)
|
||||||
|
input.installTap(onBus: 0, bufferSize: 1_024, format: inputFormat) { [writer] buffer, _ in
|
||||||
|
writer.write(buffer)
|
||||||
|
}
|
||||||
|
engine.prepare()
|
||||||
|
try engine.start()
|
||||||
|
|
||||||
|
while true {
|
||||||
|
try await Task.sleep(for: .seconds(3_600))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private final class SystemAudioCapture: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendable {
|
||||||
|
private let writer: PcmWriter
|
||||||
|
private let reporter: ErrorReporter
|
||||||
|
private let sampleQueue = DispatchQueue(label: "meeting-assistant.system-audio")
|
||||||
|
private var stream: SCStream?
|
||||||
|
|
||||||
|
init(writer: PcmWriter, reporter: ErrorReporter) {
|
||||||
|
self.writer = writer
|
||||||
|
self.reporter = reporter
|
||||||
|
}
|
||||||
|
|
||||||
|
func run(sampleRate: Int, channels: Int) async throws -> Never {
|
||||||
|
let content: SCShareableContent
|
||||||
|
do {
|
||||||
|
content = try await SCShareableContent.excludingDesktopWindows(
|
||||||
|
false,
|
||||||
|
onScreenWindowsOnly: false
|
||||||
|
)
|
||||||
|
} catch {
|
||||||
|
throw CaptureFailure.systemAudio(error.localizedDescription)
|
||||||
|
}
|
||||||
|
|
||||||
|
guard let display = content.displays.first else {
|
||||||
|
throw CaptureFailure.noDisplay
|
||||||
|
}
|
||||||
|
|
||||||
|
let filter = SCContentFilter(
|
||||||
|
display: display,
|
||||||
|
excludingApplications: [],
|
||||||
|
exceptingWindows: []
|
||||||
|
)
|
||||||
|
let configuration = SCStreamConfiguration()
|
||||||
|
configuration.capturesAudio = true
|
||||||
|
configuration.excludesCurrentProcessAudio = false
|
||||||
|
configuration.sampleRate = sampleRate
|
||||||
|
configuration.channelCount = channels
|
||||||
|
configuration.width = 2
|
||||||
|
configuration.height = 2
|
||||||
|
configuration.showsCursor = false
|
||||||
|
configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1)
|
||||||
|
|
||||||
|
let captureStream = SCStream(filter: filter, configuration: configuration, delegate: self)
|
||||||
|
try captureStream.addStreamOutput(self, type: .audio, sampleHandlerQueue: sampleQueue)
|
||||||
|
do {
|
||||||
|
try await captureStream.startCapture()
|
||||||
|
} catch {
|
||||||
|
throw CaptureFailure.systemAudio(error.localizedDescription)
|
||||||
|
}
|
||||||
|
stream = captureStream
|
||||||
|
|
||||||
|
while true {
|
||||||
|
try await Task.sleep(for: .seconds(3_600))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func stream(
|
||||||
|
_ stream: SCStream,
|
||||||
|
didOutputSampleBuffer sampleBuffer: CMSampleBuffer,
|
||||||
|
of outputType: SCStreamOutputType
|
||||||
|
) {
|
||||||
|
guard outputType == .audio,
|
||||||
|
sampleBuffer.isValid,
|
||||||
|
CMSampleBufferDataIsReady(sampleBuffer),
|
||||||
|
let description = CMSampleBufferGetFormatDescription(sampleBuffer),
|
||||||
|
let streamDescription = CMAudioFormatDescriptionGetStreamBasicDescription(description)
|
||||||
|
else {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let frameCount = AVAudioFrameCount(CMSampleBufferGetNumSamples(sampleBuffer))
|
||||||
|
guard frameCount > 0,
|
||||||
|
let sourceFormat = AVAudioFormat(streamDescription: streamDescription)
|
||||||
|
else {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var bufferListSize = 0
|
||||||
|
var retainedBlockBuffer: CMBlockBuffer?
|
||||||
|
var status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
|
||||||
|
sampleBuffer,
|
||||||
|
bufferListSizeNeededOut: &bufferListSize,
|
||||||
|
bufferListOut: nil,
|
||||||
|
bufferListSize: 0,
|
||||||
|
blockBufferAllocator: nil,
|
||||||
|
blockBufferMemoryAllocator: nil,
|
||||||
|
flags: 0,
|
||||||
|
blockBufferOut: &retainedBlockBuffer
|
||||||
|
)
|
||||||
|
guard status == noErr else {
|
||||||
|
reporter.exit(
|
||||||
|
with: .audioConverter("CoreMedia could not describe system audio (OSStatus \(status))."),
|
||||||
|
code: 71
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
let rawBufferList = UnsafeMutableRawPointer.allocate(
|
||||||
|
byteCount: bufferListSize,
|
||||||
|
alignment: MemoryLayout<AudioBufferList>.alignment
|
||||||
|
)
|
||||||
|
defer { rawBufferList.deallocate() }
|
||||||
|
let audioBufferList = rawBufferList.bindMemory(to: AudioBufferList.self, capacity: 1)
|
||||||
|
status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
|
||||||
|
sampleBuffer,
|
||||||
|
bufferListSizeNeededOut: nil,
|
||||||
|
bufferListOut: audioBufferList,
|
||||||
|
bufferListSize: bufferListSize,
|
||||||
|
blockBufferAllocator: nil,
|
||||||
|
blockBufferMemoryAllocator: nil,
|
||||||
|
flags: UInt32(kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment),
|
||||||
|
blockBufferOut: &retainedBlockBuffer
|
||||||
|
)
|
||||||
|
guard status == noErr,
|
||||||
|
let buffer = AVAudioPCMBuffer(
|
||||||
|
pcmFormat: sourceFormat,
|
||||||
|
bufferListNoCopy: audioBufferList,
|
||||||
|
deallocator: nil
|
||||||
|
)
|
||||||
|
else {
|
||||||
|
reporter.exit(
|
||||||
|
with: .audioConverter("CoreMedia could not expose system audio (OSStatus \(status))."),
|
||||||
|
code: 71
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
buffer.frameLength = frameCount
|
||||||
|
writer.write(buffer)
|
||||||
|
}
|
||||||
|
|
||||||
|
func stream(_ stream: SCStream, didStopWithError error: Error) {
|
||||||
|
reporter.exit(with: .systemAudio(error.localizedDescription), code: 72)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@main
|
||||||
|
private struct MacOsMeetingAudioCapture {
|
||||||
|
static func main() async {
|
||||||
|
let reporter = ErrorReporter()
|
||||||
|
do {
|
||||||
|
let arguments = CommandLine.arguments
|
||||||
|
guard arguments.count == 4,
|
||||||
|
let sampleRate = Int(arguments[2]),
|
||||||
|
let channels = Int(arguments[3])
|
||||||
|
else {
|
||||||
|
throw CaptureFailure.usage
|
||||||
|
}
|
||||||
|
guard sampleRate > 0, channels == 1 || channels == 2 else {
|
||||||
|
throw CaptureFailure.invalidFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
let writer = try PcmWriter(sampleRate: sampleRate, channels: channels, reporter: reporter)
|
||||||
|
switch arguments[1] {
|
||||||
|
case "microphone":
|
||||||
|
try await MicrophoneCapture(writer: writer).run()
|
||||||
|
case "system":
|
||||||
|
try await SystemAudioCapture(writer: writer, reporter: reporter)
|
||||||
|
.run(sampleRate: sampleRate, channels: channels)
|
||||||
|
default:
|
||||||
|
throw CaptureFailure.usage
|
||||||
|
}
|
||||||
|
} catch let failure as CaptureFailure {
|
||||||
|
reporter.exit(with: failure, code: 64)
|
||||||
|
} catch {
|
||||||
|
reporter.write(error.localizedDescription)
|
||||||
|
Darwin.exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -35,7 +35,7 @@ builder.Services.AddSingleton<IMeetingAudioSource>(services => new CompositeMeet
|
|||||||
services.GetRequiredService<ILogger<CompositeMeetingAudioSource>>()));
|
services.GetRequiredService<ILogger<CompositeMeetingAudioSource>>()));
|
||||||
builder.Services.AddSingleton<IMeetingMetadataProvider, OutlookClassicMeetingMetadataProvider>();
|
builder.Services.AddSingleton<IMeetingMetadataProvider, OutlookClassicMeetingMetadataProvider>();
|
||||||
#else
|
#else
|
||||||
builder.Services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
|
PortableMeetingAudioRegistration.Add(builder.Services);
|
||||||
builder.Services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
|
builder.Services.AddSingleton<IMeetingMetadataProvider, NoopMeetingMetadataProvider>();
|
||||||
#endif
|
#endif
|
||||||
builder.Services.AddSingleton<ITranscriptStore, VaultTranscriptStore>();
|
builder.Services.AddSingleton<ITranscriptStore, VaultTranscriptStore>();
|
||||||
|
|||||||
@@ -0,0 +1,268 @@
|
|||||||
|
using System.Diagnostics;
|
||||||
|
using System.Runtime.CompilerServices;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Recording;
|
||||||
|
|
||||||
|
internal enum MacOsAudioCaptureKind
|
||||||
|
{
|
||||||
|
Microphone,
|
||||||
|
System
|
||||||
|
}
|
||||||
|
|
||||||
|
internal sealed record MacOsAudioCaptureRequest(
|
||||||
|
MacOsAudioCaptureKind Kind,
|
||||||
|
int SampleRate,
|
||||||
|
int Channels);
|
||||||
|
|
||||||
|
internal interface IMacOsAudioCaptureProcessFactory
|
||||||
|
{
|
||||||
|
IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request);
|
||||||
|
}
|
||||||
|
|
||||||
|
internal interface IMacOsAudioCaptureProcess : IAsyncDisposable
|
||||||
|
{
|
||||||
|
Stream StandardOutput { get; }
|
||||||
|
|
||||||
|
int ExitCode { get; }
|
||||||
|
|
||||||
|
bool HasExited { get; }
|
||||||
|
|
||||||
|
Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken);
|
||||||
|
|
||||||
|
Task WaitForExitAsync(CancellationToken cancellationToken);
|
||||||
|
|
||||||
|
void Terminate();
|
||||||
|
}
|
||||||
|
|
||||||
|
internal abstract class MacOsMeetingAudioSource : IMeetingAudioSource
|
||||||
|
{
|
||||||
|
private const int ReadBufferSize = 8 * 1024;
|
||||||
|
private readonly MacOsAudioCaptureKind kind;
|
||||||
|
private readonly IMacOsAudioCaptureProcessFactory processFactory;
|
||||||
|
private readonly ILogger logger;
|
||||||
|
|
||||||
|
protected MacOsMeetingAudioSource(
|
||||||
|
MacOsAudioCaptureKind kind,
|
||||||
|
IMacOsAudioCaptureProcessFactory processFactory,
|
||||||
|
ILogger logger)
|
||||||
|
{
|
||||||
|
this.kind = kind;
|
||||||
|
this.processFactory = processFactory;
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return CaptureAsync(new MeetingAssistantOptions(), cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async IAsyncEnumerable<AudioChunk> CaptureAsync(
|
||||||
|
MeetingAssistantOptions options,
|
||||||
|
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var request = new MacOsAudioCaptureRequest(
|
||||||
|
kind,
|
||||||
|
options.Recording.SampleRate,
|
||||||
|
options.Recording.Channels);
|
||||||
|
await using var process = processFactory.Start(request);
|
||||||
|
var standardErrorTask = process.ReadStandardErrorAsync(CancellationToken.None);
|
||||||
|
var buffer = new byte[ReadBufferSize];
|
||||||
|
byte? pendingByte = null;
|
||||||
|
var hasLoggedFirstChunk = false;
|
||||||
|
|
||||||
|
logger.LogInformation(
|
||||||
|
"Starting macOS {CaptureKind} audio capture at {SampleRate} Hz/{Channels} channel(s)",
|
||||||
|
kind,
|
||||||
|
request.SampleRate,
|
||||||
|
request.Channels);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
var bytesRead = await process.StandardOutput.ReadAsync(buffer, cancellationToken);
|
||||||
|
if (bytesRead == 0)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
var pcm = CopyCompleteSamples(buffer, bytesRead, ref pendingByte);
|
||||||
|
if (pcm.Length == 0)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!hasLoggedFirstChunk)
|
||||||
|
{
|
||||||
|
hasLoggedFirstChunk = true;
|
||||||
|
logger.LogInformation(
|
||||||
|
"macOS {CaptureKind} audio produced its first PCM chunk: {ByteCount} bytes",
|
||||||
|
kind,
|
||||||
|
pcm.Length);
|
||||||
|
}
|
||||||
|
|
||||||
|
yield return new AudioChunk(pcm, request.SampleRate, request.Channels);
|
||||||
|
}
|
||||||
|
|
||||||
|
await process.WaitForExitAsync(cancellationToken);
|
||||||
|
var standardError = await standardErrorTask;
|
||||||
|
if (process.ExitCode != 0)
|
||||||
|
{
|
||||||
|
throw CreateCaptureException(kind, process.ExitCode, standardError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (!process.HasExited)
|
||||||
|
{
|
||||||
|
process.Terminate();
|
||||||
|
await process.WaitForExitAsync(CancellationToken.None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] CopyCompleteSamples(byte[] buffer, int bytesRead, ref byte? pendingByte)
|
||||||
|
{
|
||||||
|
var totalBytes = bytesRead + (pendingByte.HasValue ? 1 : 0);
|
||||||
|
var completeBytes = totalBytes - totalBytes % sizeof(short);
|
||||||
|
var pcm = new byte[completeBytes];
|
||||||
|
var targetOffset = 0;
|
||||||
|
|
||||||
|
if (pendingByte is { } firstByte)
|
||||||
|
{
|
||||||
|
pcm[0] = firstByte;
|
||||||
|
targetOffset = 1;
|
||||||
|
pendingByte = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var copyCount = completeBytes - targetOffset;
|
||||||
|
if (copyCount > 0)
|
||||||
|
{
|
||||||
|
Buffer.BlockCopy(buffer, 0, pcm, targetOffset, copyCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
var consumedFromBuffer = copyCount;
|
||||||
|
if (consumedFromBuffer < bytesRead)
|
||||||
|
{
|
||||||
|
pendingByte = buffer[consumedFromBuffer];
|
||||||
|
}
|
||||||
|
|
||||||
|
return pcm;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static InvalidOperationException CreateCaptureException(
|
||||||
|
MacOsAudioCaptureKind kind,
|
||||||
|
int exitCode,
|
||||||
|
string standardError)
|
||||||
|
{
|
||||||
|
var detail = string.IsNullOrWhiteSpace(standardError)
|
||||||
|
? "The native capture helper did not provide an error message."
|
||||||
|
: standardError.Trim();
|
||||||
|
return new InvalidOperationException(
|
||||||
|
$"macOS {kind.ToString().ToLowerInvariant()} audio capture exited with code {exitCode}. {detail}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal sealed class MacOsMicrophoneAudioSource : MacOsMeetingAudioSource
|
||||||
|
{
|
||||||
|
public MacOsMicrophoneAudioSource(
|
||||||
|
IMacOsAudioCaptureProcessFactory processFactory,
|
||||||
|
ILogger<MacOsMicrophoneAudioSource> logger)
|
||||||
|
: base(MacOsAudioCaptureKind.Microphone, processFactory, logger)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal sealed class MacOsSystemAudioSource : MacOsMeetingAudioSource
|
||||||
|
{
|
||||||
|
public MacOsSystemAudioSource(
|
||||||
|
IMacOsAudioCaptureProcessFactory processFactory,
|
||||||
|
ILogger<MacOsSystemAudioSource> logger)
|
||||||
|
: base(MacOsAudioCaptureKind.System, processFactory, logger)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal sealed class MacOsAudioCaptureProcessFactory : IMacOsAudioCaptureProcessFactory
|
||||||
|
{
|
||||||
|
private readonly ILogger<MacOsAudioCaptureProcessFactory> logger;
|
||||||
|
|
||||||
|
public MacOsAudioCaptureProcessFactory(ILogger<MacOsAudioCaptureProcessFactory> logger)
|
||||||
|
{
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IMacOsAudioCaptureProcess Start(MacOsAudioCaptureRequest request)
|
||||||
|
{
|
||||||
|
if (!OperatingSystem.IsMacOS())
|
||||||
|
{
|
||||||
|
throw new PlatformNotSupportedException("The macOS audio capture helper can only run on macOS.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var helperPath = Path.Combine(AppContext.BaseDirectory, "Native", "macos-meeting-audio-capture");
|
||||||
|
if (!File.Exists(helperPath))
|
||||||
|
{
|
||||||
|
throw new FileNotFoundException(
|
||||||
|
$"The macOS audio capture helper was not found at '{helperPath}'. Build or publish the application on macOS so the Swift helper is compiled into the Native folder.",
|
||||||
|
helperPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
var startInfo = new ProcessStartInfo
|
||||||
|
{
|
||||||
|
FileName = helperPath,
|
||||||
|
RedirectStandardOutput = true,
|
||||||
|
RedirectStandardError = true,
|
||||||
|
UseShellExecute = false,
|
||||||
|
CreateNoWindow = true
|
||||||
|
};
|
||||||
|
startInfo.ArgumentList.Add(request.Kind == MacOsAudioCaptureKind.Microphone ? "microphone" : "system");
|
||||||
|
startInfo.ArgumentList.Add(request.SampleRate.ToString(System.Globalization.CultureInfo.InvariantCulture));
|
||||||
|
startInfo.ArgumentList.Add(request.Channels.ToString(System.Globalization.CultureInfo.InvariantCulture));
|
||||||
|
|
||||||
|
logger.LogDebug("Starting native macOS audio helper {HelperPath} for {CaptureKind}", helperPath, request.Kind);
|
||||||
|
var process = Process.Start(startInfo)
|
||||||
|
?? throw new InvalidOperationException($"Failed to start the macOS audio capture helper '{helperPath}'.");
|
||||||
|
return new MacOsAudioCaptureProcess(process);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
internal sealed class MacOsAudioCaptureProcess : IMacOsAudioCaptureProcess
|
||||||
|
{
|
||||||
|
private readonly Process process;
|
||||||
|
|
||||||
|
public MacOsAudioCaptureProcess(Process process)
|
||||||
|
{
|
||||||
|
this.process = process;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Stream StandardOutput => process.StandardOutput.BaseStream;
|
||||||
|
|
||||||
|
public int ExitCode => process.ExitCode;
|
||||||
|
|
||||||
|
public bool HasExited => process.HasExited;
|
||||||
|
|
||||||
|
public Task<string> ReadStandardErrorAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return process.StandardError.ReadToEndAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task WaitForExitAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return process.WaitForExitAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Terminate()
|
||||||
|
{
|
||||||
|
if (!process.HasExited)
|
||||||
|
{
|
||||||
|
process.Kill(entireProcessTree: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public ValueTask DisposeAsync()
|
||||||
|
{
|
||||||
|
process.Dispose();
|
||||||
|
return ValueTask.CompletedTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
|
||||||
|
namespace MeetingAssistant.Recording;
|
||||||
|
|
||||||
|
internal static class PortableMeetingAudioRegistration
|
||||||
|
{
|
||||||
|
public static void Add(IServiceCollection services)
|
||||||
|
{
|
||||||
|
Add(services, OperatingSystem.IsMacOS());
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static void Add(IServiceCollection services, bool isMacOs)
|
||||||
|
{
|
||||||
|
if (!isMacOs)
|
||||||
|
{
|
||||||
|
services.AddSingleton<IMeetingAudioSource, UnavailableMeetingAudioSource>();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
services.AddSingleton<IMacOsAudioCaptureProcessFactory, MacOsAudioCaptureProcessFactory>();
|
||||||
|
services.AddSingleton<MacOsMicrophoneAudioSource>();
|
||||||
|
services.AddSingleton<MacOsSystemAudioSource>();
|
||||||
|
services.AddSingleton<IAcousticEchoCancellerFactory, AdaptiveFilterAcousticEchoCancellerFactory>();
|
||||||
|
services.AddSingleton<IMeetingAudioSource>(serviceProvider => new CompositeMeetingAudioSource(
|
||||||
|
serviceProvider.GetRequiredService<MacOsMicrophoneAudioSource>(),
|
||||||
|
serviceProvider.GetRequiredService<MacOsSystemAudioSource>(),
|
||||||
|
serviceProvider.GetRequiredService<IAcousticEchoCancellerFactory>(),
|
||||||
|
serviceProvider.GetRequiredService<ILogger<CompositeMeetingAudioSource>>()));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,6 +5,6 @@ public sealed class UnavailableMeetingAudioSource : IMeetingAudioSource
|
|||||||
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
public IAsyncEnumerable<AudioChunk> CaptureAsync(CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
throw new PlatformNotSupportedException(
|
throw new PlatformNotSupportedException(
|
||||||
"Meeting audio capture is only implemented for the Windows target in this version.");
|
"Meeting audio capture is supported on Windows and macOS, but is unavailable on this host.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Meeting Assistant
|
# Meeting Assistant
|
||||||
|
|
||||||
Meeting Assistant is Manuel's local .NET meeting capture and knowledge service. It runs on the Windows workstation, records microphone plus system audio, writes Obsidian meeting artifacts, transcribes with speaker attribution, enriches meetings from local context, and drives summary/project agents without depending on Teams, Zoom, or another meeting-platform API for the primary flow.
|
Meeting Assistant is Manuel's local .NET meeting capture and knowledge service. It runs on Windows and macOS, records microphone plus system audio, writes Obsidian meeting artifacts, transcribes with speaker attribution, enriches meetings from local context, and drives summary/project agents without depending on Teams, Zoom, or another meeting-platform API for the primary flow.
|
||||||
|
|
||||||
## What This Repo Owns
|
## What This Repo Owns
|
||||||
|
|
||||||
@@ -9,7 +9,8 @@ This repository owns the application source, tests, OpenSpec requirements, confi
|
|||||||
The app is intentionally local-first:
|
The app is intentionally local-first:
|
||||||
|
|
||||||
- Windows builds provide the tray icon, global hotkeys, NAudio capture, Outlook Classic COM enrichment, active-window screenshots, and notifications.
|
- Windows builds provide the tray icon, global hotkeys, NAudio capture, Outlook Classic COM enrichment, active-window screenshots, and notifications.
|
||||||
- Non-Windows builds keep the server/testable service surface but compile out Windows-only integrations.
|
- Portable builds on macOS capture the default microphone through AVFoundation and computer output through ScreenCaptureKit, while keeping Windows-only desktop integrations compiled out.
|
||||||
|
- Portable builds on other hosts keep the server/testable service surface but do not provide meeting audio capture.
|
||||||
- The normal runtime endpoint is local HTTP on port `5090`, with `/health` and `/recording/status` as the safe first checks.
|
- The normal runtime endpoint is local HTTP on port `5090`, with `/health` and `/recording/status` as the safe first checks.
|
||||||
- Behavior changes are OpenSpec-driven; current accepted requirements live under `openspec/specs`.
|
- Behavior changes are OpenSpec-driven; current accepted requirements live under `openspec/specs`.
|
||||||
|
|
||||||
@@ -24,6 +25,15 @@ dotnet test MeetingAssistant.slnx
|
|||||||
dotnet run --project MeetingAssistant
|
dotnet run --project MeetingAssistant
|
||||||
```
|
```
|
||||||
|
|
||||||
|
On macOS, build the portable target on the Mac that will run it:
|
||||||
|
|
||||||
|
```zsh
|
||||||
|
dotnet build MeetingAssistant/MeetingAssistant.csproj -f net10.0
|
||||||
|
dotnet run --project MeetingAssistant/MeetingAssistant.csproj -f net10.0
|
||||||
|
```
|
||||||
|
|
||||||
|
The macOS build compiles the bundled Swift capture helper into `Native/macos-meeting-audio-capture`. On first recording, allow both **Microphone** and **Screen & System Audio Recording** under System Settings > Privacy & Security. macOS recording is controlled through the local HTTP endpoints because the tray icon and global hotkeys remain Windows-only.
|
||||||
|
|
||||||
For the durable local background instance, use the snippets restart helper:
|
For the durable local background instance, use the snippets restart helper:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
|
|||||||
Binary file not shown.
@@ -131,6 +131,8 @@ The default profile is always named `default`. Non-default profile hotkeys are r
|
|||||||
|
|
||||||
During recording, Meeting Assistant captures microphone and system loopback separately, buffers both streams to align samples, cleans the microphone stream through a local adaptive echo canceller using loopback as the far-end reference, then mixes the cleaned microphone and system streams into the normal 16 kHz mono PCM chunks. If one source is quiet beyond the alignment timeout, the available source is mixed with synthetic silence so microphone-only speech and system-only playback keep flowing to transcription.
|
During recording, Meeting Assistant captures microphone and system loopback separately, buffers both streams to align samples, cleans the microphone stream through a local adaptive echo canceller using loopback as the far-end reference, then mixes the cleaned microphone and system streams into the normal 16 kHz mono PCM chunks. If one source is quiet beyond the alignment timeout, the available source is mixed with synthetic silence so microphone-only speech and system-only playback keep flowing to transcription.
|
||||||
|
|
||||||
|
On macOS, the portable target captures the default microphone through AVFoundation and computer output through ScreenCaptureKit. The build compiles `Native/MacOsMeetingAudioCapture/main.swift` into the application output and publish `Native` folder; build or publish on macOS with Xcode Command Line Tools installed. The first capture requires both **Microphone** and **Screen & System Audio Recording** permissions under System Settings > Privacy & Security. Restart the application after granting a newly requested permission. `Recording:MicrophoneDeviceId` and runtime microphone selection remain Windows-only; macOS follows the system default input device.
|
||||||
|
|
||||||
On Windows, `Recording:MicrophoneDeviceId` can pin capture to a specific active microphone endpoint id. Leave it blank to follow the Windows default capture endpoint. The tray icon menu also exposes `Microphone`, listing active microphone endpoints with the effective endpoint checked. Selecting a microphone there overrides the configured/default microphone for later recording starts until another microphone is selected or the process exits.
|
On Windows, `Recording:MicrophoneDeviceId` can pin capture to a specific active microphone endpoint id. Leave it blank to follow the Windows default capture endpoint. The tray icon menu also exposes `Microphone`, listing active microphone endpoints with the effective endpoint checked. Selecting a microphone there overrides the configured/default microphone for later recording starts until another microphone is selected or the process exits.
|
||||||
|
|
||||||
`Recording:MicrophoneMixGain` and `Recording:SystemAudioMixGain` are applied during the final mix and default to `1`. `Recording:TemporaryRecordingsFolder` controls where the temporary mixed WAV is written while the run is active. Temporary WAV files are deleted after the run completes, and stale temporary recordings from interrupted runs are deleted when the application starts. If an Azure Speech meeting cannot drain transcription before `Recording:StopProcessingTimeout`, Meeting Assistant keeps the WAV and writes a durable backlog item under `TemporaryRecordingsFolder\offline-transcription-backlog`. The background backlog worker retries those queued meetings, replays each WAV through a fresh speech pipeline, rewrites the original transcript, completes meeting metadata and summary generation, then removes the backlog item and WAV.
|
`Recording:MicrophoneMixGain` and `Recording:SystemAudioMixGain` are applied during the final mix and default to `1`. `Recording:TemporaryRecordingsFolder` controls where the temporary mixed WAV is written while the run is active. Temporary WAV files are deleted after the run completes, and stale temporary recordings from interrupted runs are deleted when the application starts. If an Azure Speech meeting cannot drain transcription before `Recording:StopProcessingTimeout`, Meeting Assistant keeps the WAV and writes a durable backlog item under `TemporaryRecordingsFolder\offline-transcription-backlog`. The background backlog worker retries those queued meetings, replays each WAV through a fresh speech pipeline, rewrites the original transcript, completes meeting metadata and summary generation, then removes the backlog item and WAV.
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
## Context
|
||||||
|
|
||||||
|
The recording pipeline already depends on `IMeetingAudioSource` and composes independent microphone and system streams through `CompositeMeetingAudioSource`. Windows supplies those streams with NAudio behind the `WINDOWS` compilation boundary. The portable target currently supplies an unavailable source on every non-Windows host.
|
||||||
|
|
||||||
|
.NET does not ship ScreenCaptureKit or AVFoundation bindings. A small native process is therefore the narrowest reliable adapter between Apple's capture frameworks and the existing managed PCM stream contract.
|
||||||
|
|
||||||
|
## Goals / Non-Goals
|
||||||
|
|
||||||
|
**Goals:**
|
||||||
|
|
||||||
|
- Capture the default macOS microphone and computer output without a virtual audio device.
|
||||||
|
- Emit the configured sample rate/channel count as signed 16-bit PCM.
|
||||||
|
- Reuse the existing managed alignment, AEC, gain, mixing, WAV, and transcription path.
|
||||||
|
- Keep Windows source files, registrations, packages, and runtime behavior unchanged.
|
||||||
|
- Fail with actionable privacy-permission or helper-packaging diagnostics.
|
||||||
|
|
||||||
|
**Non-Goals:**
|
||||||
|
|
||||||
|
- Add a macOS tray icon, global hotkeys, Outlook metadata, notifications, or screenshots.
|
||||||
|
- Add runtime macOS microphone selection in this change.
|
||||||
|
- Replace or refactor the Windows NAudio implementation.
|
||||||
|
- Commit long-lived user meeting audio; the proof WAV is a short implementation artifact only.
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
### Reuse the existing composite source
|
||||||
|
|
||||||
|
macOS registers separate microphone and system `IMeetingAudioSource` adapters and feeds them to `CompositeMeetingAudioSource`. This preserves the established audio semantics and keeps platform code limited to capture and PCM conversion.
|
||||||
|
|
||||||
|
### Use one native helper with two modes
|
||||||
|
|
||||||
|
The bundled Swift helper accepts `microphone` or `system`, plus sample rate and channel count, and writes headerless signed 16-bit PCM to standard output. AVFoundation supplies microphone buffers. ScreenCaptureKit supplies system-audio buffers. AVAudioConverter normalizes both sources before managed code reads them.
|
||||||
|
|
||||||
|
The managed adapter owns process lifetime, reads PCM chunks asynchronously, terminates the helper when capture is cancelled, and surfaces native diagnostics when the helper exits unexpectedly.
|
||||||
|
|
||||||
|
### Build the helper only on macOS
|
||||||
|
|
||||||
|
The Swift source is portable content, but compilation is conditioned on the build host being macOS. macOS build and publish output receives the executable under `Native`. Windows compilation remains within its existing `WINDOWS` branch and does not compile or invoke Swift code.
|
||||||
|
|
||||||
|
### Keep unsupported-host behavior
|
||||||
|
|
||||||
|
The portable target chooses macOS capture with a runtime OS check. Linux and other hosts continue resolving `UnavailableMeetingAudioSource` with an updated host-neutral error.
|
||||||
|
|
||||||
|
## Risks / Trade-offs
|
||||||
|
|
||||||
|
- [macOS privacy controls can initially deny or pause capture] → Request microphone authorization and report explicit Screen Recording/System Audio or Microphone guidance on stderr.
|
||||||
|
- [A helper process adds a packaging boundary] → Make the build fail on macOS if Swift compilation fails and verify the helper exists in build and publish output.
|
||||||
|
- [Native audio buffer formats vary] → Convert the CoreMedia/AVFoundation buffers with `AVAudioConverter` to the exact requested PCM format before writing.
|
||||||
|
- [Windows regressions from shared startup edits] → Leave the `#if WINDOWS` branch intact and build the Windows target in addition to portable tests.
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
## Implementation Evidence
|
||||||
|
|
||||||
|
Date: 2026-07-21
|
||||||
|
Host: macOS arm64
|
||||||
|
|
||||||
|
### Live audio verification
|
||||||
|
|
||||||
|
- The bundled Swift helper compiled as an arm64 Mach-O executable and was launched directly from the portable application build output.
|
||||||
|
- AVFoundation microphone capture emitted 546,920 bytes of signed 16-bit, 16 kHz mono PCM during the final 17-second proof window. The selected default input was `Jabra Link 390`; its captured samples were all zero, consistent with a muted/silent input rather than a capture-process failure.
|
||||||
|
- ScreenCaptureKit system capture emitted 551,680 bytes of signed 16-bit, 16 kHz mono PCM while the repository's known `sample-16khz-mono.wav` fixture was played 12 times through the default system output.
|
||||||
|
- The captured streams were aligned to their shared duration, summed with 16-bit clamping, and written to `docs/evidence/macos-meeting-audio-proof.wav`.
|
||||||
|
- `afinfo` reports the proof as 17.091250 seconds, mono, 16000 Hz, signed 16-bit PCM, with 546,920 audio bytes.
|
||||||
|
- Signal inspection found 48,133 non-zero samples and a peak amplitude of 12,000 in the proof WAV.
|
||||||
|
- SHA-256: `f38285a5386067f0c2637698f2e3c720ad30b65cebaaefe8d00454c78d8c2d05`.
|
||||||
|
|
||||||
|
### Behavior and packaging verification
|
||||||
|
|
||||||
|
- Public recording endpoint behavior:
|
||||||
|
- Test: `RecordingEndpointsCaptureMixedMacOsAudioAndStopNativeProcesses` calls `/recording/start`, observes known microphone/system samples combined into one `12000` PCM sample at the speech-pipeline boundary, calls `/recording/stop`, and verifies both native capture processes were terminated.
|
||||||
|
- The test uses deterministic process-boundary audio because automated test hosts cannot depend on live devices or macOS privacy state.
|
||||||
|
- Focused macOS-source, public endpoint, and existing mixer tests:
|
||||||
|
- Command: `dotnet test MeetingAssistant.Tests/MeetingAssistant.Tests.csproj --filter 'FullyQualifiedName~MacOsMeetingAudioSourceTests|FullyQualifiedName~AudioMixingTests' --no-restore --nologo`
|
||||||
|
- Result: passed, 11/11.
|
||||||
|
- Portable application build:
|
||||||
|
- Command: `dotnet build MeetingAssistant/MeetingAssistant.csproj -f net10.0 --no-restore --nologo`
|
||||||
|
- Result: succeeded with 0 warnings and 0 errors; the Swift helper was compiled into the application output.
|
||||||
|
- macOS publish:
|
||||||
|
- Command: `dotnet publish MeetingAssistant/MeetingAssistant.csproj -f net10.0 -r osx-arm64 --self-contained false -p:EnableWindowsTargeting=true -o tmp/macos-publish-proof --nologo`
|
||||||
|
- Result: succeeded; `Native/macos-meeting-audio-capture` is an executable arm64 Mach-O file in publish output.
|
||||||
|
- OpenSpec validation:
|
||||||
|
- Command: `openspec validate add-macos-meeting-audio-capture --strict`
|
||||||
|
- Result: valid.
|
||||||
|
|
||||||
|
### Windows isolation verification
|
||||||
|
|
||||||
|
- The existing `#if WINDOWS` registration continues to use `MicrophoneAudioSource`, `SystemAudioSource`, `AdaptiveFilterAcousticEchoCancellerFactory`, and `CompositeMeetingAudioSource`. macOS selection exists only inside the portable `#else` branch.
|
||||||
|
- Windows C# compilation:
|
||||||
|
- Command: `dotnet msbuild MeetingAssistant/MeetingAssistant.csproj -t:Compile -p:TargetFramework=net10.0-windows10.0.19041.0 -p:EnableWindowsTargeting=true -nologo -v:minimal`
|
||||||
|
- Result: succeeded.
|
||||||
|
- A full Windows-target build cannot finish on macOS because the Windows App SDK attempts to execute `MakePri.exe` and returns `Exec format error`. This occurs after managed compilation and is the repository's known cross-host limitation; Windows CI or a Windows host remains the full binary verification environment.
|
||||||
|
|
||||||
|
### Full portable test result
|
||||||
|
|
||||||
|
- Command: `dotnet test MeetingAssistant.Tests/MeetingAssistant.Tests.csproj --no-restore --nologo`
|
||||||
|
- Result: 425 passed and 10 failed in the combined run. One unrelated inactivity-timer test was then rerun in isolation and passed (1/1), leaving the effective result at 426 passing tests and the same 9 unrelated baseline failures.
|
||||||
|
- The 9 failures are the same pre-existing macOS baseline recorded by the active `detect-host-operating-system` change: Windows/GDI+ image rendering, Windows-path expectations, and Windows user-environment behavior. None exercises macOS audio capture or the shared mixer.
|
||||||
|
|
||||||
|
### Operational verification boundary
|
||||||
|
|
||||||
|
The live helpers were verified while the macOS display session was awake and produced the committed proof signal. The public endpoint path was verified deterministically in-process through the real registration, macOS source adapters, shared mixer, coordinator, and start/stop endpoints. A later attempt to repeat live ScreenCaptureKit capture under the transient `vstest`/console proof host could not enumerate a display after the built-in display entered the asleep state; ScreenCaptureKit returned `ScreenCaptureKit could not find a display for system-audio capture`. The production port-5090 service was deliberately not restarted or reconfigured with fake ASR dependencies. These are the reasons live native input and the public application path were verified in two complementary passes rather than one production meeting run.
|
||||||
|
|
||||||
|
The live Meeting Assistant service was not restarted, and no active recording or meeting artifacts were modified during verification.
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
## Why
|
||||||
|
|
||||||
|
The portable Meeting Assistant build runs on macOS, but starting a recording resolves `UnavailableMeetingAudioSource` and fails before any audio reaches transcription. macOS users need the same microphone-plus-computer-output capture contract without changing the established Windows NAudio implementation.
|
||||||
|
|
||||||
|
## What Changes
|
||||||
|
|
||||||
|
- Add native macOS microphone capture through AVFoundation.
|
||||||
|
- Add native macOS computer-output capture through ScreenCaptureKit.
|
||||||
|
- Stream both sources as 16-bit PCM into the existing alignment, echo-cancellation, gain, mixing, recording, and transcription pipeline.
|
||||||
|
- Select the macOS sources only when the portable build is running on macOS; retain the existing compile-time Windows registrations unchanged.
|
||||||
|
- Package a small Swift capture helper in macOS build and publish output.
|
||||||
|
- Produce a short mixed WAV from live microphone and system PCM emitted by the implemented native capture helper as operational evidence.
|
||||||
|
- Verify the public recording endpoints, managed macOS adapters, mixer, and stop behavior together with deterministic process-boundary audio.
|
||||||
|
|
||||||
|
## Capabilities
|
||||||
|
|
||||||
|
### New Capabilities
|
||||||
|
|
||||||
|
None.
|
||||||
|
|
||||||
|
### Modified Capabilities
|
||||||
|
|
||||||
|
- `meeting-recording`: Extend microphone and computer-output recording to macOS while preserving Windows behavior.
|
||||||
|
|
||||||
|
## Impact
|
||||||
|
|
||||||
|
- Portable application dependency registration.
|
||||||
|
- A macOS-native helper executable built from Swift source.
|
||||||
|
- macOS Screen Recording/System Audio and Microphone privacy permissions.
|
||||||
|
- Recording behavior tests, build verification, and runtime documentation.
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
## MODIFIED Requirements
|
||||||
|
|
||||||
|
### Requirement: Recording mode captures microphone and computer output
|
||||||
|
Meeting Assistant SHALL capture microphone input and computer output and combine them into one audio stream for transcription on supported Windows and macOS hosts.
|
||||||
|
|
||||||
|
Meeting Assistant SHALL capture audio as 16 kHz mono PCM chunks for the existing recording and transcription pipeline.
|
||||||
|
|
||||||
|
Meeting Assistant SHALL capture microphone and system loopback as separate input streams before producing the final mono chunks.
|
||||||
|
|
||||||
|
On Windows, Meeting Assistant SHALL retain the existing NAudio microphone and WASAPI loopback implementation.
|
||||||
|
|
||||||
|
On macOS, Meeting Assistant SHALL capture the default microphone through AVFoundation and computer output through ScreenCaptureKit without requiring a virtual audio device.
|
||||||
|
|
||||||
|
The macOS capture adapter SHALL convert both native sources to signed 16-bit PCM using the active run's configured sample rate and channel count before passing chunks to the existing managed mixing pipeline.
|
||||||
|
|
||||||
|
The macOS capture adapter SHALL stop its native capture process when the recording capture token is cancelled.
|
||||||
|
|
||||||
|
When macOS privacy permission is missing or denied, Meeting Assistant SHALL fail capture with an actionable message identifying the required Microphone or Screen Recording/System Audio permission.
|
||||||
|
|
||||||
|
Meeting Assistant SHALL clean the microphone stream with a local acoustic echo cancellation stage that uses system loopback as the far-end reference.
|
||||||
|
|
||||||
|
Meeting Assistant SHALL produce final mono chunks by adding the cleaned microphone samples and system samples.
|
||||||
|
|
||||||
|
Meeting Assistant SHALL align microphone and system samples through per-source buffers before producing final chunks and SHALL apply the existing configured gains and clamping behavior.
|
||||||
|
|
||||||
|
Meeting Assistant SHALL write only the mixed stream to the temporary WAV used by transcription and finalization.
|
||||||
|
|
||||||
|
#### Scenario: macOS records both meeting-audio sources
|
||||||
|
- **GIVEN** the portable build is running on macOS
|
||||||
|
- **AND** Microphone and Screen Recording/System Audio permissions are granted
|
||||||
|
- **WHEN** a meeting recording starts
|
||||||
|
- **THEN** Meeting Assistant captures microphone audio through AVFoundation
|
||||||
|
- **AND** captures computer output through ScreenCaptureKit
|
||||||
|
- **AND** passes both signed 16-bit PCM streams through the existing mixer
|
||||||
|
|
||||||
|
#### Scenario: macOS capture uses run audio format
|
||||||
|
- **GIVEN** an active macOS recording configures a sample rate and channel count
|
||||||
|
- **WHEN** the native capture helpers start
|
||||||
|
- **THEN** both helpers emit signed 16-bit PCM with that sample rate and channel count
|
||||||
|
|
||||||
|
#### Scenario: macOS recording stops native capture
|
||||||
|
- **GIVEN** macOS microphone and system capture are active
|
||||||
|
- **WHEN** Meeting Assistant stops capture
|
||||||
|
- **THEN** it terminates both native capture processes
|
||||||
|
- **AND** PCM already delivered to the managed pipeline before stop remains available to the existing recording pipeline
|
||||||
|
|
||||||
|
#### Scenario: macOS privacy permission is unavailable
|
||||||
|
- **WHEN** macOS denies microphone or Screen Recording/System Audio capture permission
|
||||||
|
- **THEN** Meeting Assistant reports which macOS privacy permission is required
|
||||||
|
|
||||||
|
#### Scenario: Windows audio capture remains isolated
|
||||||
|
- **GIVEN** Meeting Assistant is compiled for the Windows target
|
||||||
|
- **WHEN** the application registers and starts meeting audio capture
|
||||||
|
- **THEN** it uses the existing NAudio microphone and WASAPI loopback sources
|
||||||
|
- **AND** does not invoke or require the macOS native helper
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
## 1. macOS Capture Behavior
|
||||||
|
|
||||||
|
- [x] 1.1 Specify macOS microphone and computer-output capture behavior and Windows isolation.
|
||||||
|
- [x] 1.2 Add failing behavior tests for macOS PCM streaming, platform registration, and the public recording endpoints.
|
||||||
|
- [x] 1.3 Implement the managed macOS audio-source and process-lifetime adapter.
|
||||||
|
- [x] 1.4 Implement and package the AVFoundation/ScreenCaptureKit Swift helper.
|
||||||
|
- [x] 1.5 Register the macOS composite source without changing the Windows registration path.
|
||||||
|
|
||||||
|
## 2. Verification and Documentation
|
||||||
|
|
||||||
|
- [x] 2.1 Document macOS permissions, build packaging, and recording behavior.
|
||||||
|
- [x] 2.2 Capture and inspect a short example mixed WAV through the new implementation.
|
||||||
|
- [x] 2.3 Run focused tests, portable tests/build, Windows-target regression compilation, strict OpenSpec validation, and refactoring/code review.
|
||||||
Reference in New Issue
Block a user