diff --git a/.claude/skills/winapp-ui-automation/SKILL.md b/.claude/skills/winapp-ui-automation/SKILL.md index e5370488..db2b156b 100644 --- a/.claude/skills/winapp-ui-automation/SKILL.md +++ b/.claude/skills/winapp-ui-automation/SKILL.md @@ -104,6 +104,26 @@ winapp ui screenshot -a myapp --capture-screen --output with-popups.png winapp ui screenshot -a myapp --focus --output focused.png ``` +### Record video (H.264 MP4) +Record the window — or a single element's region — to an MP4. Frames are captured via Windows +Graphics Capture (PrintWindow fallback) and encoded incrementally with Media Foundation, so long +captures never buffer in memory. +```powershell +# Record a window for 10s at 15 fps +winapp ui record -a myapp --duration-sec 10 --fps 15 --output demo.mp4 + +# Record until Ctrl+C, downscaled so the longest edge is 1280px +winapp ui record -a myapp --duration-sec 0 --max-edge 1280 --output capture.mp4 + +# Record a single element's region +winapp ui record itm-chart-9f8e -a myapp --output chart.mp4 + +# Include overlays/popups (captures from screen; may include occluding windows) +winapp ui record -a myapp --capture-screen --duration-sec 5 --output with-popups.mp4 +``` +- `--duration-sec 0` records until Ctrl+C; the MP4 is finalized on exit. +- The `--json` envelope reports `path`, `frames`, `width`, `height`, `fileSize`, `codec` (`"h264"`), and `mode` — the capture path used (`wgc`, `printwindow`, or `screen`). + ### Hover (for tooltips, flyouts, hover states) `--dwell-time ` sets how long to wait after hovering (default: 800, range: 0–10000). ```powershell @@ -374,6 +394,29 @@ Capture the target window or element as a PNG image. When multiple windows exist | `--output` | Save output to file path (e.g., screenshot) | (none) | | `--window` | Target window by HWND (stable handle from list output). Takes precedence over --app. | (none) | +### `winapp ui record` + +Record the target window (or an element's region) to an H.264 MP4 video. Captures frames via Windows Graphics Capture and encodes with Media Foundation. Use --duration-sec 0 to record until Ctrl+C. Use --capture-screen to include overlays/popups. + +#### Arguments + +| Argument | Required | Description | +|----------|----------|-------------| +| `` | No | Semantic slug (e.g., btn-minimize-d1a0) or text to search by name/automationId | + +#### Options + +| Option | Description | Default | +|--------|-------------|---------| +| `--app` | Target app (process name, window title, or PID). Lists windows if ambiguous. | (none) | +| `--capture-screen` | Capture from screen DC via BitBlt (includes popups/overlays not owned by the target). Implies --focus. | (none) | +| `--duration-sec` | Recording duration in seconds. 0 = record until Ctrl+C. | (none) | +| `--fps` | Frames per second to capture | `15` | +| `--json` | Format output as JSON | (none) | +| `--max-edge` | Downscale so the longest edge is at most this many pixels (0 = no downscale) | (none) | +| `--output` | Save output to file path (e.g., screenshot) | (none) | +| `--window` | Target window by HWND (stable handle from list output). Takes precedence over --app. | (none) | + ### `winapp ui invoke` Activate an element by slug or text search. Tries InvokePattern, TogglePattern, SelectionItemPattern, and ExpandCollapsePattern in order. diff --git a/.github/plugin/skills/winapp-cli/ui-automation/SKILL.md b/.github/plugin/skills/winapp-cli/ui-automation/SKILL.md index e5370488..db2b156b 100644 --- a/.github/plugin/skills/winapp-cli/ui-automation/SKILL.md +++ b/.github/plugin/skills/winapp-cli/ui-automation/SKILL.md @@ -104,6 +104,26 @@ winapp ui screenshot -a myapp --capture-screen --output with-popups.png winapp ui screenshot -a myapp --focus --output focused.png ``` +### Record video (H.264 MP4) +Record the window — or a single element's region — to an MP4. Frames are captured via Windows +Graphics Capture (PrintWindow fallback) and encoded incrementally with Media Foundation, so long +captures never buffer in memory. +```powershell +# Record a window for 10s at 15 fps +winapp ui record -a myapp --duration-sec 10 --fps 15 --output demo.mp4 + +# Record until Ctrl+C, downscaled so the longest edge is 1280px +winapp ui record -a myapp --duration-sec 0 --max-edge 1280 --output capture.mp4 + +# Record a single element's region +winapp ui record itm-chart-9f8e -a myapp --output chart.mp4 + +# Include overlays/popups (captures from screen; may include occluding windows) +winapp ui record -a myapp --capture-screen --duration-sec 5 --output with-popups.mp4 +``` +- `--duration-sec 0` records until Ctrl+C; the MP4 is finalized on exit. +- The `--json` envelope reports `path`, `frames`, `width`, `height`, `fileSize`, `codec` (`"h264"`), and `mode` — the capture path used (`wgc`, `printwindow`, or `screen`). + ### Hover (for tooltips, flyouts, hover states) `--dwell-time ` sets how long to wait after hovering (default: 800, range: 0–10000). ```powershell @@ -374,6 +394,29 @@ Capture the target window or element as a PNG image. When multiple windows exist | `--output` | Save output to file path (e.g., screenshot) | (none) | | `--window` | Target window by HWND (stable handle from list output). Takes precedence over --app. | (none) | +### `winapp ui record` + +Record the target window (or an element's region) to an H.264 MP4 video. Captures frames via Windows Graphics Capture and encodes with Media Foundation. Use --duration-sec 0 to record until Ctrl+C. Use --capture-screen to include overlays/popups. + +#### Arguments + +| Argument | Required | Description | +|----------|----------|-------------| +| `` | No | Semantic slug (e.g., btn-minimize-d1a0) or text to search by name/automationId | + +#### Options + +| Option | Description | Default | +|--------|-------------|---------| +| `--app` | Target app (process name, window title, or PID). Lists windows if ambiguous. | (none) | +| `--capture-screen` | Capture from screen DC via BitBlt (includes popups/overlays not owned by the target). Implies --focus. | (none) | +| `--duration-sec` | Recording duration in seconds. 0 = record until Ctrl+C. | (none) | +| `--fps` | Frames per second to capture | `15` | +| `--json` | Format output as JSON | (none) | +| `--max-edge` | Downscale so the longest edge is at most this many pixels (0 = no downscale) | (none) | +| `--output` | Save output to file path (e.g., screenshot) | (none) | +| `--window` | Target window by HWND (stable handle from list output). Takes precedence over --app. | (none) | + ### `winapp ui invoke` Activate an element by slug or text search. Tries InvokePattern, TogglePattern, SelectionItemPattern, and ExpandCollapsePattern in order. diff --git a/docs/cli-schema.json b/docs/cli-schema.json index f18bc2c1..0235e8a7 100644 --- a/docs/cli-schema.json +++ b/docs/cli-schema.json @@ -2765,6 +2765,167 @@ } } }, + "record": { + "description": "Record the target window (or an element's region) to an H.264 MP4 video. Captures frames via Windows Graphics Capture and encodes with Media Foundation. Use --duration-sec 0 to record until Ctrl+C. Use --capture-screen to include overlays/popups.", + "hidden": false, + "arguments": { + "selector": { + "description": "Semantic slug (e.g., btn-minimize-d1a0) or text to search by name/automationId", + "order": 0, + "hidden": false, + "valueType": "System.String", + "hasDefaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + } + } + }, + "options": { + "--app": { + "description": "Target app (process name, window title, or PID). Lists windows if ambiguous.", + "hidden": false, + "aliases": [ + "-a" + ], + "valueType": "System.String", + "hasDefaultValue": false, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--capture-screen": { + "description": "Capture from screen DC via BitBlt (includes popups/overlays not owned by the target). Implies --focus.", + "hidden": false, + "valueType": "System.Boolean", + "hasDefaultValue": true, + "defaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--duration-sec": { + "description": "Recording duration in seconds. 0 = record until Ctrl+C.", + "hidden": false, + "valueType": "System.Int32", + "hasDefaultValue": true, + "defaultValue": 0, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--fps": { + "description": "Frames per second to capture", + "hidden": false, + "valueType": "System.Int32", + "hasDefaultValue": true, + "defaultValue": 15, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--json": { + "description": "Format output as JSON", + "hidden": false, + "valueType": "System.Boolean", + "hasDefaultValue": true, + "defaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--max-edge": { + "description": "Downscale so the longest edge is at most this many pixels (0 = no downscale)", + "hidden": false, + "valueType": "System.Int32", + "hasDefaultValue": true, + "defaultValue": 0, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--output": { + "description": "Save output to file path (e.g., screenshot)", + "hidden": false, + "aliases": [ + "-o" + ], + "valueType": "System.String", + "hasDefaultValue": false, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--quiet": { + "description": "Suppress progress messages", + "hidden": false, + "aliases": [ + "-q" + ], + "valueType": "System.Boolean", + "hasDefaultValue": true, + "defaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--verbose": { + "description": "Enable verbose output", + "hidden": false, + "aliases": [ + "-v" + ], + "valueType": "System.Boolean", + "hasDefaultValue": true, + "defaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--window": { + "description": "Target window by HWND (stable handle from list output). Takes precedence over --app.", + "hidden": false, + "aliases": [ + "-w" + ], + "valueType": "System.Nullable", + "hasDefaultValue": false, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + } + } + }, "screenshot": { "description": "Capture the target window or element as a PNG image. When multiple windows exist (e.g., dialogs), captures each to a separate file. With --json, returns file path and dimensions. Use --capture-screen for popup overlays.", "hidden": false, diff --git a/docs/fragments/skills/winapp-cli/ui-automation.md b/docs/fragments/skills/winapp-cli/ui-automation.md index c5824aa8..614bed09 100644 --- a/docs/fragments/skills/winapp-cli/ui-automation.md +++ b/docs/fragments/skills/winapp-cli/ui-automation.md @@ -99,6 +99,26 @@ winapp ui screenshot -a myapp --capture-screen --output with-popups.png winapp ui screenshot -a myapp --focus --output focused.png ``` +### Record video (H.264 MP4) +Record the window — or a single element's region — to an MP4. Frames are captured via Windows +Graphics Capture (PrintWindow fallback) and encoded incrementally with Media Foundation, so long +captures never buffer in memory. +```powershell +# Record a window for 10s at 15 fps +winapp ui record -a myapp --duration-sec 10 --fps 15 --output demo.mp4 + +# Record until Ctrl+C, downscaled so the longest edge is 1280px +winapp ui record -a myapp --duration-sec 0 --max-edge 1280 --output capture.mp4 + +# Record a single element's region +winapp ui record itm-chart-9f8e -a myapp --output chart.mp4 + +# Include overlays/popups (captures from screen; may include occluding windows) +winapp ui record -a myapp --capture-screen --duration-sec 5 --output with-popups.mp4 +``` +- `--duration-sec 0` records until Ctrl+C; the MP4 is finalized on exit. +- The `--json` envelope reports `path`, `frames`, `width`, `height`, `fileSize`, `codec` (`"h264"`), and `mode` — the capture path used (`wgc`, `printwindow`, or `screen`). + ### Hover (for tooltips, flyouts, hover states) `--dwell-time ` sets how long to wait after hovering (default: 800, range: 0–10000). ```powershell diff --git a/docs/usage.md b/docs/usage.md index 5ce162ba..d651bccf 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1080,6 +1080,7 @@ winapp ui [command] [options] - `get-property` - Read element properties - `get-text` / `get-value` - Read value/text from element (TextPattern, ValuePattern, or Name) - `screenshot` - Capture window/element as PNG (auto-captures dialogs separately) +- `record` - Record a window/element region to an H.264 MP4 video (Windows Graphics Capture + Media Foundation) - `invoke` - Activate element (click, toggle, expand) - `click` - Click element via mouse simulation (for controls that don't support invoke) - `hover` - Move mouse to element to trigger tooltips, flyouts, and hover states (default dwell: 800ms) @@ -1096,4 +1097,32 @@ winapp ui [command] [options] - `-a, --app ` - Target app (name, title, or PID) - `-w, --window ` - Target window by HWND (stable) +#### ui record + +Record the target window — or a single element's region — to an H.264 MP4 video. Frames are +captured via Windows Graphics Capture (with a PrintWindow fallback) and encoded incrementally with +Media Foundation, so long recordings never buffer in memory. + +```bash +# Record a window for 10 seconds at 15 fps +winapp ui record -a Calculator --duration-sec 10 --fps 15 -o demo.mp4 + +# Record until Ctrl+C, downscaled so the longest edge is 1280px +winapp ui record -a "My App" --duration-sec 0 --max-edge 1280 -o capture.mp4 + +# Record just one element's region +winapp ui record -a "My App" btn-save-1234 -o button.mp4 +``` + +**Record options:** +- `--duration-sec ` - Recording length in seconds. `0` records until Ctrl+C (default `0`). +- `--fps ` - Frames per second to capture (default `15`). +- `--max-edge ` - Downscale so the longest edge is at most this many pixels (`0` = no downscale). +- `--capture-screen` - Capture from the screen so overlays/popups are included (may capture occluding windows). +- `-o, --output ` - Output `.mp4` path (defaults to `recording-.mp4`). + +With `--json`, emits a `UiRecordResult` envelope including the output `path`, `frames`, `width`, +`height`, `fileSize`, `codec` (`"h264"`), and `mode` — the capture path actually used +(`wgc`, `printwindow`, or `screen`). + For full documentation, see [docs/ui-automation.md](ui-automation.md). diff --git a/scripts/generate-llm-docs.ps1 b/scripts/generate-llm-docs.ps1 index ff2735d4..0379e490 100644 --- a/scripts/generate-llm-docs.ps1 +++ b/scripts/generate-llm-docs.ps1 @@ -104,7 +104,7 @@ $SkillCommandMap = @{ "manifest" = @("manifest generate", "manifest update-assets", "manifest add-alias") "troubleshoot" = @("get-winapp-path", "tool", "store") "frameworks" = @() # No auto-generated command sections — links to guides - "ui-automation" = @("ui status", "ui inspect", "ui search", "ui get-property", "ui get-value", "ui screenshot", "ui invoke", "ui click", "ui drag", "ui hover", "ui send-keys", "ui set-value", "ui focus", "ui scroll-into-view", "ui scroll", "ui wait-for", "ui list-windows", "ui get-focused") + "ui-automation" = @("ui status", "ui inspect", "ui search", "ui get-property", "ui get-value", "ui screenshot", "ui record", "ui invoke", "ui click", "ui drag", "ui hover", "ui send-keys", "ui set-value", "ui focus", "ui scroll-into-view", "ui scroll", "ui wait-for", "ui list-windows", "ui get-focused") } # Validate that all CLI commands are covered by at least one skill diff --git a/src/winapp-CLI/WinApp.Cli.Tests/FakeUiServices.cs b/src/winapp-CLI/WinApp.Cli.Tests/FakeUiServices.cs index 9be38fd0..154a1a08 100644 --- a/src/winapp-CLI/WinApp.Cli.Tests/FakeUiServices.cs +++ b/src/winapp-CLI/WinApp.Cli.Tests/FakeUiServices.cs @@ -88,6 +88,23 @@ public Task SearchAsync(UiSessionInfo session, SelectorExpression s public Task<(byte[] Pixels, int Width, int Height)> ScreenshotAsync(UiSessionInfo session, string? elementId, bool captureScreen, bool focus, CancellationToken ct) => Task.FromResult(ScreenshotResult); + /// Configurable result for . The fake writes a tiny placeholder file to the output path. + public RecordCaptureResult RecordResult { get; set; } = new() { Frames = 3, Width = 2, Height = 2, FileSize = 0, Mode = "wgc" }; + + public async Task RecordAsync(UiSessionInfo session, string? elementId, RecordOptions options, CancellationToken ct) + { + await File.WriteAllBytesAsync(options.OutputPath, new byte[16], ct); + var size = new FileInfo(options.OutputPath).Length; + return new RecordCaptureResult + { + Frames = RecordResult.Frames, + Width = RecordResult.Width, + Height = RecordResult.Height, + FileSize = size, + Mode = RecordResult.Mode, + }; + } + public Task InvokeAsync(UiSessionInfo session, UiElement element, CancellationToken ct) => Task.FromResult(InvokeResult); diff --git a/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Record.cs b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Record.cs new file mode 100644 index 00000000..020e7e06 --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Record.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using WinApp.Cli.Commands; +using WinApp.Cli.Services; + +namespace WinApp.Cli.Tests; + +public partial class UiCommandTests +{ + // --------------------------------------------------------------------- + // record — capture a window/element region to an H.264 MP4. The fake + // RecordAsync writes a tiny placeholder file and returns configurable + // frame/mode metadata, so these tests exercise the command's validation + // and JSON envelope without touching WGC/Media Foundation. + // --------------------------------------------------------------------- + + [TestMethod] + public async Task Record_MissingApp_ReturnsError() + { + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["--json"]); + Assert.AreEqual(1, exitCode); + } + + [TestMethod] + public async Task Record_InvalidFps_ReturnsError() + { + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["-a", "TestApp", "--fps", "0", "--json"]); + Assert.AreEqual(1, exitCode); + } + + [TestMethod] + public async Task Record_InvalidMaxEdge_ReturnsError() + { + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["-a", "TestApp", "--max-edge=-1", "--json"]); + Assert.AreEqual(1, exitCode); + } + + [TestMethod] + public async Task Record_InvalidDuration_ReturnsError() + { + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["-a", "TestApp", "--duration-sec=-1", "--json"]); + Assert.AreEqual(1, exitCode); + } + + [TestMethod] + public async Task Record_Success_EmitsRecordResultJson() + { + _fakeUia.RecordResult = new RecordCaptureResult { Frames = 42, Width = 640, Height = 480, Mode = "wgc" }; + + var outputPath = Path.Combine(_tempDirectory.FullName, "capture.mp4"); + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync( + command, ["-a", "TestApp", "--duration-sec", "1", "--fps", "10", "-o", outputPath, "--json"]); + + Assert.AreEqual(0, exitCode); + + var result = System.Text.Json.JsonSerializer.Deserialize(TestAnsiConsole.Output); + Assert.AreEqual(outputPath, result.GetProperty("path").GetString()); + Assert.AreEqual("h264", result.GetProperty("codec").GetString()); + Assert.AreEqual("wgc", result.GetProperty("mode").GetString()); + Assert.AreEqual(42, result.GetProperty("frames").GetInt32()); + Assert.AreEqual(10, result.GetProperty("fps").GetInt32()); + Assert.AreEqual(1, result.GetProperty("durationSec").GetInt32()); + // The fake writes a placeholder file at the requested path. + Assert.IsTrue(File.Exists(outputPath), "record should have produced an output file"); + } + + [TestMethod] + public async Task Record_Success_ReportsPrintWindowMode() + { + // The mode field must reflect the capture path actually used (accuracy fix): a printwindow + // capture must not be mislabeled. Here the fake reports "printwindow"; assert it round-trips. + _fakeUia.RecordResult = new RecordCaptureResult { Frames = 5, Width = 100, Height = 100, Mode = "printwindow" }; + + var outputPath = Path.Combine(_tempDirectory.FullName, "pw.mp4"); + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["-a", "TestApp", "--duration-sec", "1", "-o", outputPath, "--json"]); + + Assert.AreEqual(0, exitCode); + var result = System.Text.Json.JsonSerializer.Deserialize(TestAnsiConsole.Output); + Assert.AreEqual("printwindow", result.GetProperty("mode").GetString()); + } +} diff --git a/src/winapp-CLI/WinApp.Cli.Tests/UiSessionServiceTests.cs b/src/winapp-CLI/WinApp.Cli.Tests/UiSessionServiceTests.cs index 01402e5a..62732a5f 100644 --- a/src/winapp-CLI/WinApp.Cli.Tests/UiSessionServiceTests.cs +++ b/src/winapp-CLI/WinApp.Cli.Tests/UiSessionServiceTests.cs @@ -73,6 +73,8 @@ private sealed class StubUiAutomation : IUiAutomationService public Task FindSingleElementAsync(UiSessionInfo session, SelectorExpression selector, CancellationToken ct) => Task.FromResult(null); public Task> GetPropertiesAsync(UiSessionInfo session, UiElement element, string? propertyName, CancellationToken ct) => Task.FromResult(new Dictionary()); public Task<(byte[] Pixels, int Width, int Height)> ScreenshotAsync(UiSessionInfo session, string? elementId, bool captureScreen, bool focus, CancellationToken ct) => Task.FromResult((Array.Empty(), 0, 0)); + public Task RecordAsync(UiSessionInfo session, string? elementId, RecordOptions options, CancellationToken ct) + => Task.FromResult(new RecordCaptureResult()); public Task InvokeAsync(UiSessionInfo session, UiElement element, CancellationToken ct) => Task.FromResult(""); public Task SetValueAsync(UiSessionInfo session, UiElement element, string text, CancellationToken ct) => Task.CompletedTask; public Task FocusAsync(UiSessionInfo session, UiElement element, CancellationToken ct) => Task.CompletedTask; @@ -82,4 +84,3 @@ private sealed class StubUiAutomation : IUiAutomationService public Task GetTextAsync(UiSessionInfo session, UiElement element, CancellationToken ct) => Task.FromResult(null); } } - diff --git a/src/winapp-CLI/WinApp.Cli/Commands/SharedUiOptions.cs b/src/winapp-CLI/WinApp.Cli/Commands/SharedUiOptions.cs index a31ef292..d063afd7 100644 --- a/src/winapp-CLI/WinApp.Cli/Commands/SharedUiOptions.cs +++ b/src/winapp-CLI/WinApp.Cli/Commands/SharedUiOptions.cs @@ -24,6 +24,9 @@ internal static class SharedUiOptions public static Option InteractiveOption { get; } public static Option HideDisabledOption { get; } public static Option HideOffscreenOption { get; } + public static Option DurationSecOption { get; } + public static Option FpsOption { get; } + public static Option MaxEdgeOption { get; } static SharedUiOptions() { @@ -101,5 +104,23 @@ static SharedUiOptions() { Description = "Hide offscreen elements from output" }; + + DurationSecOption = new Option("--duration-sec") + { + Description = "Recording duration in seconds. 0 = record until Ctrl+C.", + DefaultValueFactory = _ => 0 + }; + + FpsOption = new Option("--fps") + { + Description = "Frames per second to capture", + DefaultValueFactory = _ => 15 + }; + + MaxEdgeOption = new Option("--max-edge") + { + Description = "Downscale so the longest edge is at most this many pixels (0 = no downscale)", + DefaultValueFactory = _ => 0 + }; } } diff --git a/src/winapp-CLI/WinApp.Cli/Commands/UiCommand.cs b/src/winapp-CLI/WinApp.Cli/Commands/UiCommand.cs index 57bdaae9..dd9aa6cd 100644 --- a/src/winapp-CLI/WinApp.Cli/Commands/UiCommand.cs +++ b/src/winapp-CLI/WinApp.Cli/Commands/UiCommand.cs @@ -16,6 +16,7 @@ public UiCommand( UiGetPropertyCommand getPropertyCommand, UiGetValueCommand getValueCommand, UiScreenshotCommand screenshotCommand, + UiRecordCommand recordCommand, UiInvokeCommand invokeCommand, UiClickCommand clickCommand, UiDragCommand dragCommand, @@ -37,6 +38,7 @@ public UiCommand( Subcommands.Add(getPropertyCommand); Subcommands.Add(getValueCommand); Subcommands.Add(screenshotCommand); + Subcommands.Add(recordCommand); Subcommands.Add(invokeCommand); Subcommands.Add(clickCommand); Subcommands.Add(dragCommand); diff --git a/src/winapp-CLI/WinApp.Cli/Commands/UiRecordCommand.cs b/src/winapp-CLI/WinApp.Cli/Commands/UiRecordCommand.cs new file mode 100644 index 00000000..e0d95a7f --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli/Commands/UiRecordCommand.cs @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using System.CommandLine; +using System.CommandLine.Invocation; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Spectre.Console; +using WinApp.Cli.Helpers; +using WinApp.Cli.Services; + +namespace WinApp.Cli.Commands; + +internal class UiRecordCommand : Command, IShortDescription +{ + public string ShortDescription => "Record a window or element region to an MP4 (H.264) video"; + + public UiRecordCommand() + : base("record", "Record the target window (or an element's region) to an H.264 MP4 video. " + + "Captures frames via Windows Graphics Capture and encodes with Media Foundation. " + + "Use --duration-sec 0 to record until Ctrl+C. Use --capture-screen to include overlays/popups.") + { + Arguments.Add(SharedUiOptions.SelectorArgument); + Options.Add(SharedUiOptions.AppOption); + Options.Add(SharedUiOptions.WindowOption); + Options.Add(SharedUiOptions.DurationSecOption); + Options.Add(SharedUiOptions.FpsOption); + Options.Add(SharedUiOptions.MaxEdgeOption); + Options.Add(SharedUiOptions.CaptureScreenOption); + Options.Add(SharedUiOptions.OutputOption); + Options.Add(WinAppRootCommand.JsonOption); + } + + public class Handler( + IUiSessionService sessionService, + IUiAutomationService uiAutomation, + IAnsiConsole ansiConsole, + ILogger logger) : AsynchronousCommandLineAction + { + public override async Task InvokeAsync(ParseResult parseResult, CancellationToken cancellationToken = default) + { + var json = parseResult.GetValue(WinAppRootCommand.JsonOption); + var selector = parseResult.GetValue(SharedUiOptions.SelectorArgument); + var app = parseResult.GetValue(SharedUiOptions.AppOption); + var window = parseResult.GetValue(SharedUiOptions.WindowOption); + + if (string.IsNullOrWhiteSpace(app) && window is null) + { + UiErrors.MissingApp(logger, json); + return 1; + } + + var durationSec = parseResult.GetValue(SharedUiOptions.DurationSecOption); + var fps = parseResult.GetValue(SharedUiOptions.FpsOption); + var maxEdge = parseResult.GetValue(SharedUiOptions.MaxEdgeOption); + var captureScreen = parseResult.GetValue(SharedUiOptions.CaptureScreenOption); + var output = parseResult.GetValue(SharedUiOptions.OutputOption); + + if (durationSec < 0) + { + UiJsonError.Emit(json, UiJsonError.CodeInternalError, "--duration-sec must be 0 or greater."); + logger.LogError("{Symbol} --duration-sec must be 0 or greater.", UiSymbols.Error); + return 1; + } + if (fps < 1) + { + UiJsonError.Emit(json, UiJsonError.CodeInternalError, "--fps must be at least 1."); + logger.LogError("{Symbol} --fps must be at least 1.", UiSymbols.Error); + return 1; + } + if (maxEdge < 0) + { + UiJsonError.Emit(json, UiJsonError.CodeInternalError, "--max-edge must be 0 or greater."); + logger.LogError("{Symbol} --max-edge must be 0 or greater.", UiSymbols.Error); + return 1; + } + + var filePath = output ?? $"recording-{DateTime.Now:yyyyMMdd-HHmmss}.mp4"; + filePath = Path.GetFullPath(filePath); + var dir = Path.GetDirectoryName(filePath); + if (dir is not null) + { + Directory.CreateDirectory(dir); + } + + try + { + var session = await sessionService.ResolveSessionAsync(app, window, cancellationToken); + + if (!json) + { + var until = durationSec > 0 ? $"{durationSec}s" : "Ctrl+C"; + ansiConsole.MarkupLine($"[grey]Recording \"{Markup.Escape(session.WindowTitle ?? "")}\" (PID {session.ProcessId}) to {Markup.Escape(filePath)} — until {until}, {fps} fps…[/]"); + } + + var options = new RecordOptions + { + OutputPath = filePath, + DurationSec = durationSec, + Fps = fps, + MaxEdge = maxEdge, + CaptureScreen = captureScreen, + }; + + var result = await uiAutomation.RecordAsync(session, selector, options, cancellationToken); + + if (json) + { + var payload = new UiRecordResult + { + Path = filePath, + DurationSec = durationSec, + Fps = fps, + Frames = result.Frames, + Width = result.Width, + Height = result.Height, + FileSize = result.FileSize, + Codec = "h264", + Mode = result.Mode, + }; + ansiConsole.Profile.Out.Writer.WriteLine( + JsonSerializer.Serialize(payload, UiJsonContext.Default.UiRecordResult)); + return 0; + } + + logger.LogInformation( + "Recorded {Frames} frames ({Width}x{Height}, h264) to {Path} ({Size}KB)", + result.Frames, result.Width, result.Height, filePath, result.FileSize / 1024); + return 0; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // Cancellation before any recording started (e.g. session resolution). The recorder + // itself finalizes the MP4 on Ctrl+C, so this only fires for pre-capture cancellation. + logger.LogDebug("Recording cancelled before capture started."); + return 1; + } + catch (System.Runtime.InteropServices.COMException comEx) + { + logger.LogDebug("COM error: {HResult} {StackTrace}", comEx.HResult, comEx.StackTrace); + UiErrors.GenericError(logger, comEx, json); + return 1; + } + catch (Exception ex) + { + UiErrors.GenericError(logger, ex, json); + return 1; + } + } + } +} diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/HostBuilderExtensions.cs b/src/winapp-CLI/WinApp.Cli/Helpers/HostBuilderExtensions.cs index a55b1f2e..3bb95596 100644 --- a/src/winapp-CLI/WinApp.Cli/Helpers/HostBuilderExtensions.cs +++ b/src/winapp-CLI/WinApp.Cli/Helpers/HostBuilderExtensions.cs @@ -91,6 +91,7 @@ public static IServiceCollection ConfigureCommands(this IServiceCollection servi .UseCommandHandler() .UseCommandHandler() .UseCommandHandler() + .UseCommandHandler() .UseCommandHandler() .UseCommandHandler() .UseCommandHandler() diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonContext.cs b/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonContext.cs index 9abee25c..7f80bab5 100644 --- a/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonContext.cs +++ b/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonContext.cs @@ -24,6 +24,7 @@ namespace WinApp.Cli.Helpers; [JsonSerializable(typeof(UiClickResult))] [JsonSerializable(typeof(UiScreenshotResult))] [JsonSerializable(typeof(UiScreenshotResult[]))] +[JsonSerializable(typeof(UiRecordResult))] [JsonSerializable(typeof(UiGetValueResult))] [JsonSerializable(typeof(UiWaitForResult))] [JsonSerializable(typeof(UiScrollResult))] @@ -140,6 +141,19 @@ internal sealed class UiScreenshotWindowInfo public string? Error { get; set; } } +internal sealed class UiRecordResult +{ + public string Path { get; set; } = ""; + public int DurationSec { get; set; } + public int Fps { get; set; } + public int Frames { get; set; } + public int Width { get; set; } + public int Height { get; set; } + public long FileSize { get; set; } + public string Codec { get; set; } = "h264"; + public string Mode { get; set; } = ""; +} + internal sealed class UiWaitForResult { public bool Found { get; set; } diff --git a/src/winapp-CLI/WinApp.Cli/NativeMethods.txt b/src/winapp-CLI/WinApp.Cli/NativeMethods.txt index 613e8e67..9d69e2eb 100644 --- a/src/winapp-CLI/WinApp.Cli/NativeMethods.txt +++ b/src/winapp-CLI/WinApp.Cli/NativeMethods.txt @@ -124,3 +124,25 @@ DXGI_SAMPLE_DESC IDXGIDevice DwmGetWindowAttribute DWMWINDOWATTRIBUTE +MFStartup +MFShutdown +MFCreateSinkWriterFromURL +IMFSinkWriter +MFCreateMediaType +IMFMediaType +IMFAttributes +MFCreateSample +IMFSample +MFCreateMemoryBuffer +IMFMediaBuffer +MFVideoFormat_H264 +MFVideoFormat_RGB32 +MFMediaType_Video +MF_MT_MAJOR_TYPE +MF_MT_SUBTYPE +MF_MT_AVG_BITRATE +MF_MT_INTERLACE_MODE +MF_MT_FRAME_SIZE +MF_MT_FRAME_RATE +MF_MT_PIXEL_ASPECT_RATIO +MF_MT_DEFAULT_STRIDE diff --git a/src/winapp-CLI/WinApp.Cli/Services/IUiAutomationService.cs b/src/winapp-CLI/WinApp.Cli/Services/IUiAutomationService.cs index ed2effe9..6444d88e 100644 --- a/src/winapp-CLI/WinApp.Cli/Services/IUiAutomationService.cs +++ b/src/winapp-CLI/WinApp.Cli/Services/IUiAutomationService.cs @@ -28,6 +28,12 @@ internal interface IUiAutomationService Task FindSingleElementAsync(UiSessionInfo session, SelectorExpression selector, CancellationToken ct); Task> GetPropertiesAsync(UiSessionInfo session, UiElement element, string? propertyName, CancellationToken ct); Task<(byte[] Pixels, int Width, int Height)> ScreenshotAsync(UiSessionInfo session, string? elementId, bool captureScreen, bool focus, CancellationToken ct); + + /// + /// Records the target window (or an element's region) to an H.264 MP4 at the + /// requested frame rate for the requested duration, encoding incrementally. + /// + Task RecordAsync(UiSessionInfo session, string? elementId, RecordOptions options, CancellationToken ct); Task InvokeAsync(UiSessionInfo session, UiElement element, CancellationToken ct); Task SetValueAsync(UiSessionInfo session, UiElement element, string text, CancellationToken ct); Task FocusAsync(UiSessionInfo session, UiElement element, CancellationToken ct); diff --git a/src/winapp-CLI/WinApp.Cli/Services/Mp4SinkWriterEncoder.cs b/src/winapp-CLI/WinApp.Cli/Services/Mp4SinkWriterEncoder.cs new file mode 100644 index 00000000..bb152bf6 --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli/Services/Mp4SinkWriterEncoder.cs @@ -0,0 +1,217 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using Windows.Win32; +using Windows.Win32.Media.MediaFoundation; + +namespace WinApp.Cli.Services; + +// Source-generated ([GeneratedComInterface]) COM objects are ComWrappers RCWs, not classic +// RCWs, so Marshal.ReleaseComObject throws for them. The ComObject wrapper implements IDisposable, +// whose Dispose() deterministically releases the underlying IUnknown — that is the AOT-safe way to +// drop a COM reference. This local helper centralizes that pattern. + +/// +/// Encodes a sequence of BGRA (RGB32) frames to an H.264 MP4 file using the +/// Windows Media Foundation IMFSinkWriter. Frames are written +/// incrementally so the full sequence never has to be held in memory. +/// +/// +/// Input frames are treated as top-down RGB32 (BGRA byte order, matching the +/// pixel layout produced by and GDI captures). The +/// sink writer inserts the color-conversion + H.264 encoder MFTs automatically. +/// +internal sealed unsafe class Mp4SinkWriterEncoder : IDisposable +{ + // MF_VERSION for Windows 7+ (MF_SDK_VERSION 0x0002, MF_API_VERSION 0x0070). + private const uint MF_VERSION = 0x00020070; + private const uint MFSTARTUP_FULL = 0; + private const uint MFVideoInterlace_Progressive = 2; + + private readonly IMFSinkWriter _writer; + private readonly uint _streamIndex; + private readonly uint _frameBytes; + private readonly string _path; + private bool _mfStarted; + private bool _finalized; + private bool _disposed; + + public int Width { get; } + + public int Height { get; } + + public Mp4SinkWriterEncoder(string path, int width, int height, int fps, uint bitrate) + { + Width = width; + Height = height; + _path = path; + _frameBytes = checked((uint)(width * height * 4)); + + PInvoke.MFStartup(MF_VERSION, MFSTARTUP_FULL).ThrowOnFailure(); + _mfStarted = true; + + IMFMediaType? outType = null; + IMFMediaType? inType = null; + try + { + PInvoke.MFCreateSinkWriterFromURL(path, null, null, out _writer).ThrowOnFailure(); + + // Output (encoded) media type: H.264. + PInvoke.MFCreateMediaType(out outType).ThrowOnFailure(); + outType.SetGUID(PInvoke.MF_MT_MAJOR_TYPE, PInvoke.MFMediaType_Video); + outType.SetGUID(PInvoke.MF_MT_SUBTYPE, PInvoke.MFVideoFormat_H264); + outType.SetUINT32(PInvoke.MF_MT_AVG_BITRATE, bitrate); + outType.SetUINT32(PInvoke.MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + outType.SetUINT64(PInvoke.MF_MT_FRAME_SIZE, PackU64((uint)width, (uint)height)); + outType.SetUINT64(PInvoke.MF_MT_FRAME_RATE, PackU64((uint)fps, 1)); + outType.SetUINT64(PInvoke.MF_MT_PIXEL_ASPECT_RATIO, PackU64(1, 1)); + _writer.AddStream(outType, out _streamIndex); + + // Input (uncompressed) media type: RGB32, top-down (positive stride). + PInvoke.MFCreateMediaType(out inType).ThrowOnFailure(); + inType.SetGUID(PInvoke.MF_MT_MAJOR_TYPE, PInvoke.MFMediaType_Video); + inType.SetGUID(PInvoke.MF_MT_SUBTYPE, PInvoke.MFVideoFormat_RGB32); + inType.SetUINT32(PInvoke.MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + inType.SetUINT32(PInvoke.MF_MT_DEFAULT_STRIDE, (uint)(width * 4)); + inType.SetUINT64(PInvoke.MF_MT_FRAME_SIZE, PackU64((uint)width, (uint)height)); + inType.SetUINT64(PInvoke.MF_MT_FRAME_RATE, PackU64((uint)fps, 1)); + inType.SetUINT64(PInvoke.MF_MT_PIXEL_ASPECT_RATIO, PackU64(1, 1)); + _writer.SetInputMediaType(_streamIndex, inType, null); + + _writer.BeginWriting(); + } + catch + { + // Constructor failed after MFStartup — release any partial writer and undo the + // MFStartup so we don't leak a global Media Foundation reference, then rethrow. + ReleaseCom(_writer); + try + { + PInvoke.MFShutdown(); + } + catch + { + // Best-effort shutdown. + } + _mfStarted = false; + throw; + } + finally + { + // The one-time media-type descriptors are consumed by AddStream/SetInputMediaType; + // release these ComWrappers RCWs deterministically rather than waiting on the GC. + ReleaseCom(outType); + ReleaseCom(inType); + } + } + + /// + /// Writes a single top-down BGRA frame. must contain + /// exactly Width*Height*4 bytes. + /// + public void WriteFrame(ReadOnlySpan bgra, long sampleTimeHns, long sampleDurationHns) + { + if (bgra.Length < _frameBytes) + { + throw new ArgumentException($"Frame buffer is {bgra.Length} bytes; expected {_frameBytes}.", nameof(bgra)); + } + + PInvoke.MFCreateMemoryBuffer(_frameBytes, out var buffer).ThrowOnFailure(); + try + { + buffer.Lock(out var dest, out _, out _); + try + { + bgra[..(int)_frameBytes].CopyTo(new Span(dest, (int)_frameBytes)); + } + finally + { + buffer.Unlock(); + } + buffer.SetCurrentLength(_frameBytes); + + PInvoke.MFCreateSample(out var sample).ThrowOnFailure(); + try + { + sample.AddBuffer(buffer); + sample.SetSampleTime(sampleTimeHns); + sample.SetSampleDuration(sampleDurationHns); + _writer.WriteSample(_streamIndex, sample); + } + finally + { + // Release the per-frame sample so its native MF buffers don't accumulate + // across thousands of frames. + ReleaseCom(sample); + } + } + finally + { + ReleaseCom(buffer); + } + } + + /// Finalizes the MP4 container. Safe to call once; a no-op afterwards. + public void Complete() + { + if (_finalized) + { + return; + } + _writer.Finalize(); + _finalized = true; + } + + private static ulong PackU64(uint high, uint low) => ((ulong)high << 32) | low; + + /// Deterministically releases a source-generated COM RCW (ComWrappers-based). + private static void ReleaseCom(object? comObject) + { + if (comObject is IDisposable disposable) + { + disposable.Dispose(); + } + } + + public void Dispose() + { + if (_disposed) + { + return; + } + _disposed = true; + + // If the encoder is being torn down without a successful Complete() (e.g. an exception + // during capture), the on-disk .mp4 is truncated/unfinalized. Release the writer and remove + // the partial file so a corrupt recording is never left behind as if it succeeded. + var leftPartial = !_finalized; + ReleaseCom(_writer); + if (leftPartial) + { + try + { + if (File.Exists(_path)) + { + File.Delete(_path); + } + } + catch + { + // Best-effort cleanup of the partial file. + } + } + + if (_mfStarted) + { + try + { + PInvoke.MFShutdown(); + } + catch + { + // Best-effort shutdown. + } + _mfStarted = false; + } + } +} diff --git a/src/winapp-CLI/WinApp.Cli/Services/RecordModels.cs b/src/winapp-CLI/WinApp.Cli/Services/RecordModels.cs new file mode 100644 index 00000000..7a54d785 --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli/Services/RecordModels.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +namespace WinApp.Cli.Services; + +/// Options controlling an MP4 window/element recording. +internal sealed class RecordOptions +{ + /// Absolute output path for the .mp4 file. + public required string OutputPath { get; init; } + + /// Recording duration in seconds. 0 = record until cancellation (Ctrl+C). + public int DurationSec { get; init; } + + /// Target frames per second. + public int Fps { get; init; } = 15; + + /// Downscale so the longest edge is at most this many pixels. 0 = no downscale. + public int MaxEdge { get; init; } + + /// Capture from the screen DC (BitBlt) so overlays/popups are included. + public bool CaptureScreen { get; init; } +} + +/// Result of an MP4 recording. +internal sealed class RecordCaptureResult +{ + public int Frames { get; init; } + public int Width { get; init; } + public int Height { get; init; } + public long FileSize { get; init; } + + /// Capture mode actually used: "wgc", "screen", or "printwindow". + public string Mode { get; init; } = "wgc"; +} diff --git a/src/winapp-CLI/WinApp.Cli/Services/UiAutomationService.Record.cs b/src/winapp-CLI/WinApp.Cli/Services/UiAutomationService.Record.cs new file mode 100644 index 00000000..f958299f --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli/Services/UiAutomationService.Record.cs @@ -0,0 +1,307 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using System.Diagnostics; +using System.Runtime.InteropServices; +using Microsoft.Extensions.Logging; +using SkiaSharp; +using Windows.Win32.Foundation; +using Windows.Win32.UI.Accessibility; +using WinApp.Cli.Models; + +namespace WinApp.Cli.Services; + +/// +/// Video recording: captures window/element frames at a fixed cadence and encodes +/// them incrementally to an H.264 MP4 via Media Foundation (). +/// +internal sealed partial class UiAutomationService +{ + public async Task RecordAsync(UiSessionInfo session, string? elementId, RecordOptions options, CancellationToken ct) + { + _logger.LogDebug("Recording process {Pid} (duration={Dur}s, fps={Fps}, maxEdge={MaxEdge}, captureScreen={Screen})", + session.ProcessId, options.DurationSec, options.Fps, options.MaxEdge, options.CaptureScreen); + + var root = GetRootElement(session); + if (root is null) + { + throw new InvalidOperationException($"No UIA window found for {session.ProcessName} (PID {session.ProcessId})."); + } + + var rootName = SafeGetBstr(() => root.get_CurrentName()); + if (rootName is not null) + { + session.WindowTitle = rootName; + } + + var hwnd = root.get_CurrentNativeWindowHandle(); + if (hwnd.IsNull && session.WindowHandle != 0) + { + hwnd = new HWND((nint)session.WindowHandle); + } + if (hwnd.IsNull) + { + throw new InvalidOperationException($"No native window handle for {session.ProcessName}. Is the window visible?"); + } + + if (Windows.Win32.PInvoke.IsIconic(hwnd)) + { + Windows.Win32.PInvoke.ShowWindow(hwnd, Windows.Win32.UI.WindowsAndMessaging.SHOW_WINDOW_CMD.SW_RESTORE); + await Task.Delay(300, ct).ConfigureAwait(false); + } + + // Bring to foreground for screen-DC capture. + if (options.CaptureScreen) + { + Windows.Win32.PInvoke.SetForegroundWindow(hwnd); + await Task.Delay(150, ct).ConfigureAwait(false); + } + + Windows.Win32.PInvoke.GetWindowRect(hwnd, out var rect); + if (rect.right - rect.left <= 0 || rect.bottom - rect.top <= 0) + { + throw new InvalidOperationException("Window has zero size. Is it minimized?"); + } + + var useScreen = options.CaptureScreen; + var useWgc = !useScreen && WgcCapture.IsSupported(); + + WgcCapture.FrameGrabber? grabber = null; + var mode = useScreen ? "screen" : (useWgc ? "wgc" : "printwindow"); + + try + { + int srcWidth; + int srcHeight; + int captureOriginLeft; + int captureOriginTop; + + if (useWgc) + { + try + { + grabber = WgcCapture.StartGrabber(hwnd, _logger); + if (!await grabber.WaitForFirstFrameAsync(TimeSpan.FromSeconds(3), ct).ConfigureAwait(false)) + { + throw new InvalidOperationException("Timed out waiting for the first captured frame."); + } + var first = grabber.TryGetLatest()!.Value; + srcWidth = first.Width; + srcHeight = first.Height; + var visibleRect = GetVisibleWindowRect(hwnd, rect); + captureOriginLeft = visibleRect.left; + captureOriginTop = visibleRect.top; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogDebug(ex, "WGC recorder init failed; falling back to screen-DC capture"); + grabber?.Dispose(); + grabber = null; + useScreen = true; + useWgc = false; + mode = "screen"; + srcWidth = rect.right - rect.left; + srcHeight = rect.bottom - rect.top; + captureOriginLeft = rect.left; + captureOriginTop = rect.top; + } + } + else + { + srcWidth = rect.right - rect.left; + srcHeight = rect.bottom - rect.top; + captureOriginLeft = rect.left; + captureOriginTop = rect.top; + } + + // Determine the crop rectangle (in capture-space) for an element selector. + var cropX = 0; + var cropY = 0; + var cropW = srcWidth; + var cropH = srcHeight; + if (!string.IsNullOrEmpty(elementId)) + { + var elRect = TryResolveElementRect(elementId, session, root); + if (elRect is { } er) + { + cropX = Math.Clamp(er.left - captureOriginLeft, 0, Math.Max(0, srcWidth - 1)); + cropY = Math.Clamp(er.top - captureOriginTop, 0, Math.Max(0, srcHeight - 1)); + cropW = Math.Clamp(er.right - er.left, 1, srcWidth - cropX); + cropH = Math.Clamp(er.bottom - er.top, 1, srcHeight - cropY); + } + else + { + _logger.LogWarning("Element '{Selector}' not found; recording the whole window instead.", elementId); + } + } + + var (targetW, targetH) = ComputeTargetSize(cropW, cropH, options.MaxEdge); + var bitrate = (uint)Math.Clamp((long)targetW * targetH * options.Fps / 8, 1_000_000, 24_000_000); + + using var encoder = new Mp4SinkWriterEncoder(options.OutputPath, targetW, targetH, options.Fps, bitrate); + + var frameDurationHns = 10_000_000L / options.Fps; + var totalFrames = options.DurationSec > 0 ? options.DurationSec * options.Fps : (int?)null; + var stopwatch = Stopwatch.StartNew(); + var frameIndex = 0; + + while (!ct.IsCancellationRequested) + { + if (totalFrames.HasValue && frameIndex >= totalFrames.Value) + { + break; + } + + byte[]? source; + int sw, sh; + if (useWgc) + { + var latest = grabber!.TryGetLatest(); + if (latest is null) + { + await Task.Delay(5, ct).ConfigureAwait(false); + continue; + } + (source, sw, sh) = latest.Value; + } + else if (useScreen) + { + source = CaptureFromScreen(captureOriginLeft, captureOriginTop, srcWidth, srcHeight); + sw = srcWidth; + sh = srcHeight; + } + else + { + // PrintWindow fallback (WGC unsupported, not --capture-screen): render the window + // into an offscreen DC. Unlike BitBlt-from-screen this excludes occluding windows, + // matching `ui screenshot`. captureOrigin is the window's top-left so crop math holds. + source = CaptureFromWindowWithBlankRetry(hwnd, srcWidth, srcHeight); + sw = srcWidth; + sh = srcHeight; + } + + var frame = ProcessFrame(source, sw, sh, cropX, cropY, cropW, cropH, targetW, targetH); + encoder.WriteFrame(frame, frameIndex * frameDurationHns, frameDurationHns); + frameIndex++; + + var targetMs = frameIndex * 1000.0 / options.Fps; + var delayMs = targetMs - stopwatch.Elapsed.TotalMilliseconds; + if (delayMs > 1) + { + try + { + await Task.Delay((int)delayMs, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + } + } + + encoder.Complete(); + + var fileSize = new FileInfo(options.OutputPath).Length; + return new RecordCaptureResult + { + Frames = frameIndex, + Width = targetW, + Height = targetH, + FileSize = fileSize, + Mode = mode, + }; + } + finally + { + grabber?.Dispose(); + } + } + + /// Computes an even (H.264-safe) target size, downscaled so the longest edge is ≤ maxEdge (0 = no downscale). + private static (int Width, int Height) ComputeTargetSize(int width, int height, int maxEdge) + { + var scale = 1.0; + var longest = Math.Max(width, height); + if (maxEdge > 0 && longest > maxEdge) + { + scale = (double)maxEdge / longest; + } + + var w = EvenClamp((int)Math.Round(width * scale)); + var h = EvenClamp((int)Math.Round(height * scale)); + return (w, h); + + static int EvenClamp(int v) => Math.Max(2, v & ~1); + } + + /// Crops and scales a captured BGRA frame to the target dimensions (top-down output). + private static byte[] ProcessFrame( + byte[] source, int sourceWidth, int sourceHeight, + int cropX, int cropY, int cropW, int cropH, + int targetWidth, int targetHeight) + { + // Fast path: whole frame at native size. + if (cropX == 0 && cropY == 0 && cropW == sourceWidth && cropH == sourceHeight + && targetWidth == sourceWidth && targetHeight == sourceHeight + && source.Length == targetWidth * targetHeight * 4) + { + return source; + } + + // Clamp crop against the actual frame in case dimensions drifted between frames. + cropX = Math.Clamp(cropX, 0, Math.Max(0, sourceWidth - 1)); + cropY = Math.Clamp(cropY, 0, Math.Max(0, sourceHeight - 1)); + cropW = Math.Clamp(cropW, 1, sourceWidth - cropX); + cropH = Math.Clamp(cropH, 1, sourceHeight - cropY); + + var srcInfo = new SKImageInfo(sourceWidth, sourceHeight, SKColorType.Bgra8888, SKAlphaType.Opaque); + using var srcBitmap = new SKBitmap(srcInfo); + Marshal.Copy(source, 0, srcBitmap.GetPixels(), Math.Min(source.Length, srcInfo.BytesSize)); + + var dstInfo = new SKImageInfo(targetWidth, targetHeight, SKColorType.Bgra8888, SKAlphaType.Opaque); + using var dstBitmap = new SKBitmap(dstInfo); + using (var canvas = new SKCanvas(dstBitmap)) + { + var srcRect = SKRect.Create(cropX, cropY, cropW, cropH); + var dstRect = SKRect.Create(0, 0, targetWidth, targetHeight); + using var paint = new SKPaint { FilterQuality = SKFilterQuality.Medium, IsAntialias = false }; + canvas.DrawBitmap(srcBitmap, srcRect, dstRect, paint); + } + + var output = new byte[dstInfo.BytesSize]; + Marshal.Copy(dstBitmap.GetPixels(), output, 0, output.Length); + return output; + } + + /// Resolves an element selector to its screen-space bounding rectangle, or null if not found. + private RECT? TryResolveElementRect(string selector, UiSessionInfo session, IUIAutomationElement root) + { + Windows.Win32.UI.Accessibility.IUIAutomationElement? target = null; + + var slugParsed = SlugGenerator.ParseSlug(selector); + if (slugParsed is not null) + { + var slugResult = FindElementBySlug(selector, root); + if (slugResult is not null) + { + target = ResolveComElement(session, slugResult); + } + } + else + { + var parsed = _selectorService.Parse(selector); + var condition = BuildCondition(parsed); + if (condition is not null) + { + target = root.FindFirst(Windows.Win32.UI.Accessibility.TreeScope.TreeScope_Descendants, condition); + } + } + + if (target is null) + { + return null; + } + + return target.get_CurrentBoundingRectangle(); + } +} diff --git a/src/winapp-CLI/WinApp.Cli/Services/WgcCapture.Record.cs b/src/winapp-CLI/WinApp.Cli/Services/WgcCapture.Record.cs new file mode 100644 index 00000000..53bb1120 --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli/Services/WgcCapture.Record.cs @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; +using Windows.Graphics.Capture; +using Windows.Graphics.DirectX; +using Windows.Win32; +using Windows.Win32.Foundation; +using D3D = Windows.Win32.Graphics.Direct3D11; +using D3DCommon = Windows.Win32.Graphics.Direct3D; + +namespace WinApp.Cli.Services; + +internal static partial class WgcCapture +{ + /// + /// Opens a persistent Windows Graphics Capture session for a window and keeps + /// the most recently arrived frame available on demand. Used by the recorder to + /// sample frames at a fixed cadence without re-initializing D3D per frame. + /// + public static FrameGrabber StartGrabber(HWND hwnd, ILogger logger) + { + if (!GraphicsCaptureSession.IsSupported()) + { + throw new PlatformNotSupportedException("Windows.Graphics.Capture is not supported on this system."); + } + + PInvoke.D3D11CreateDevice( + pAdapter: null, + D3DCommon.D3D_DRIVER_TYPE.D3D_DRIVER_TYPE_HARDWARE, + Software: default, + D3D.D3D11_CREATE_DEVICE_FLAG.D3D11_CREATE_DEVICE_BGRA_SUPPORT, + pFeatureLevels: default, + SDKVersion: D3D11_SDK_VERSION, + out var device, + out _, + out var context).ThrowOnFailure(); + + try + { + var winrtDevice = CreateDirect3DDevice(device); + var item = CreateItemForWindow(hwnd); + var pool = Direct3D11CaptureFramePool.CreateFreeThreaded( + winrtDevice, + DirectXPixelFormat.B8G8R8A8UIntNormalized, + numberOfBuffers: 2, + item.Size); + var session = pool.CreateCaptureSession(item); + session.IsCursorCaptureEnabled = false; + + return new FrameGrabber(device, context, pool, session, item, logger); + } + catch + { + (context as IDisposable)?.Dispose(); + (device as IDisposable)?.Dispose(); + throw; + } + } + + /// + /// Holds a live capture session and caches the latest CPU-readable frame. Thread-safe: + /// frames arrive on WGC's free-threaded pool while the recorder samples via + /// . + /// + internal sealed class FrameGrabber : IDisposable + { + private readonly D3D.ID3D11Device _device; + private readonly D3D.ID3D11DeviceContext _context; + private readonly Direct3D11CaptureFramePool _pool; + private readonly GraphicsCaptureSession _session; + private readonly GraphicsCaptureItem _item; + private readonly ILogger _logger; + private readonly Lock _lock = new(); + private byte[]? _latestPixels; + private int _latestWidth; + private int _latestHeight; + private bool _disposed; + + internal FrameGrabber( + D3D.ID3D11Device device, + D3D.ID3D11DeviceContext context, + Direct3D11CaptureFramePool pool, + GraphicsCaptureSession session, + GraphicsCaptureItem item, + ILogger logger) + { + _device = device; + _context = context; + _pool = pool; + _session = session; + _item = item; + _logger = logger; + _pool.FrameArrived += OnFrameArrived; + _session.StartCapture(); + } + + private void OnFrameArrived(Direct3D11CaptureFramePool sender, object args) + { + Direct3D11CaptureFrame? frame = null; + try + { + frame = sender.TryGetNextFrame(); + if (frame is null) + { + return; + } + + var (pixels, width, height) = CopyFrame(_device, _context, frame); + lock (_lock) + { + _latestPixels = pixels; + _latestWidth = width; + _latestHeight = height; + } + } + catch (Exception ex) + { + _logger.LogDebug(ex, "WGC frame copy failed during recording"); + } + finally + { + frame?.Dispose(); + } + } + + /// Returns the most recently captured frame, or if none has arrived yet. + public (byte[] Pixels, int Width, int Height)? TryGetLatest() + { + lock (_lock) + { + if (_latestPixels is null) + { + return null; + } + return (_latestPixels, _latestWidth, _latestHeight); + } + } + + /// Waits (up to ) for the first frame to arrive. + public async Task WaitForFirstFrameAsync(TimeSpan timeout, CancellationToken ct) + { + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + if (TryGetLatest() is not null) + { + return true; + } + await Task.Delay(30, ct).ConfigureAwait(false); + } + return TryGetLatest() is not null; + } + + public void Dispose() + { + if (_disposed) + { + return; + } + _disposed = true; + + _pool.FrameArrived -= OnFrameArrived; + _session.Dispose(); + _pool.Dispose(); + (_context as IDisposable)?.Dispose(); + (_device as IDisposable)?.Dispose(); + } + } +}