From 011826db60919dfafb28b31eb2b019160c271078 Mon Sep 17 00:00:00 2001 From: Gordon Lam Date: Mon, 6 Jul 2026 10:59:38 +0800 Subject: [PATCH] ui touch/pen: add synthetic touch and stylus commands Adds `winapp ui touch` and `winapp ui pen` so automation can inject touch gestures (tap, swipe, pinch, stretch) and stylus/inking input (tap, path, pressure, tilt, eraser) against running Windows apps. Also updates the generated CLI schema, npm command surface, and UI automation skill/docs to surface the new commands. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .claude/skills/winapp-ui-automation/SKILL.md | 79 +++- .../skills/winapp-cli/ui-automation/SKILL.md | 79 +++- docs/cli-schema.json | 356 ++++++++++++++++ .../skills/winapp-cli/ui-automation.md | 30 +- docs/ui-automation.md | 47 ++- docs/usage.md | 2 + scripts/generate-llm-docs.ps1 | 2 +- .../WinApp.Cli.Tests/FakeUiServices.cs | 61 +++ .../WinApp.Cli.Tests/UiCommandTests.Pen.cs | 135 ++++++ .../WinApp.Cli.Tests/UiCommandTests.Touch.cs | 139 +++++++ .../WinApp.Cli.Tests/UiCommandTests.cs | 5 +- .../WinApp.Cli.Tests/UiSessionServiceTests.cs | 1 + .../WinApp.Cli/Commands/UiCommand.cs | 4 + .../WinApp.Cli/Commands/UiPenCommand.cs | 291 +++++++++++++ .../WinApp.Cli/Commands/UiTouchCommand.cs | 333 +++++++++++++++ .../WinApp.Cli/Helpers/CliSchema.cs | 1 + .../Helpers/HostBuilderExtensions.cs | 3 + .../WinApp.Cli/Helpers/IPointerInput.cs | 67 +++ .../Helpers/PointerGesturePlanner.cs | 185 +++++++++ .../WinApp.Cli/Helpers/PointerInput.cs | 389 ++++++++++++++++++ .../WinApp.Cli/Helpers/RealPointerInput.cs | 17 + .../WinApp.Cli/Helpers/UiJsonContext.cs | 35 ++ .../WinApp.Cli/Helpers/UiJsonError.cs | 2 + src/winapp-CLI/WinApp.Cli/NativeMethods.txt | 12 + .../Services/IUiAutomationService.cs | 10 + .../Services/UiAutomationService.cs | 33 ++ src/winapp-npm/scripts/generate-commands.mjs | 2 +- src/winapp-npm/src/winapp-commands.ts | 93 +++++ 28 files changed, 2405 insertions(+), 8 deletions(-) create mode 100644 src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Pen.cs create mode 100644 src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Touch.cs create mode 100644 src/winapp-CLI/WinApp.Cli/Commands/UiPenCommand.cs create mode 100644 src/winapp-CLI/WinApp.Cli/Commands/UiTouchCommand.cs create mode 100644 src/winapp-CLI/WinApp.Cli/Helpers/IPointerInput.cs create mode 100644 src/winapp-CLI/WinApp.Cli/Helpers/PointerGesturePlanner.cs create mode 100644 src/winapp-CLI/WinApp.Cli/Helpers/PointerInput.cs create mode 100644 src/winapp-CLI/WinApp.Cli/Helpers/RealPointerInput.cs diff --git a/.claude/skills/winapp-ui-automation/SKILL.md b/.claude/skills/winapp-ui-automation/SKILL.md index e5370488..c4e8b710 100644 --- a/.claude/skills/winapp-ui-automation/SKILL.md +++ b/.claude/skills/winapp-ui-automation/SKILL.md @@ -12,7 +12,7 @@ version: 0.4.1 ## Prerequisites - For UIA mode (any app): No setup needed — works with any running Windows app -- For input-injecting verbs (`click`, `hover`, `drag`, `scroll --wheel`, `send-keys --via send-input`): an **unlocked, interactive desktop** with the target window foregroundable. On a locked/secure desktop they fail fast with `no_interactive_desktop`. The UIA-pattern verbs (`inspect`, `search`, `get-*`, `wait-for`, `set-value`, `invoke`, `scroll --direction/--to`, `screenshot`) are headless/locked-session friendly — prefer them in CI. +- For input-injecting verbs (`click`, `hover`, `drag`, `touch`, `pen`, `scroll --wheel`, `send-keys --via send-input`): an **unlocked, interactive desktop** with the target window foregroundable. On a locked/secure desktop they fail fast with `no_interactive_desktop`. The UIA-pattern verbs (`inspect`, `search`, `get-*`, `wait-for`, `set-value`, `invoke`, `scroll --direction/--to`, `screenshot`) are headless/locked-session friendly — prefer them in CI. ## Common patterns @@ -157,6 +157,34 @@ winapp ui drag itm-card-9f8e itm-trash-0001 -a myapp --right ``` - A selector drags from/to the element's center; `x,y` are app coordinates in the same space `ui inspect`/`search` report. Element endpoints are re-resolved just before the drag and fail with `target_moved` if still animating; on a locked/secure desktop the drag fails with `no_interactive_desktop`. +### Touch gestures (tap, swipe, pinch, stretch, long-press) +Inject synthetic touch. The contact anchor is an element selector (its center) or an explicit `--at x,y` app coordinate. Prefers the modern synthetic-pointer device and falls back to the legacy touch-injection API. +```powershell +# Tap an element center; or tap explicit app coordinates +winapp ui touch btn-ok-1a2b -a myapp +winapp ui touch -a myapp --at 320,240 + +# Long-press, swipe, and two-finger pinch/stretch (zoom) +winapp ui touch tile-photo-7b3c -a myapp --gesture long-press --hold-ms 600 +winapp ui touch -a myapp --at 100,300 --gesture swipe --to-point 400,300 +winapp ui touch img-map-9f8e -a myapp --gesture pinch --distance 200 +winapp ui touch img-map-9f8e -a myapp --gesture stretch --distance 200 +``` +- Gestures: `tap` (default), `double-tap`, `long-press`, `swipe`, `pinch`, `stretch`. `--fingers` 1–10 (pinch/stretch always 2). Refuses without a non-zero foregrounded target (`no_target`/`foreground_not_target`/`no_interactive_desktop`); every coordinate is bounds-checked against the target window and rejected with `invalid_arguments` if outside. If injected touch is unsupported on the device, the command surfaces the real Win32 error rather than a false success. + +### Pen / stylus (taps and ink strokes) +Inject synthetic pen input (Windows 10 1809+). Target an element center, an explicit `--at`, or a full `--path` ink stroke, with pressure/tilt/eraser control. +```powershell +# Pen tap at element center; firm tap at explicit coords +winapp ui pen canvas-1a2b -a myapp +winapp ui pen -a myapp --at 320,240 --pressure 0.8 + +# Draw an ink stroke, or erase along one +winapp ui pen -a myapp --path "100,100 150,120 210,140 260,120" +winapp ui pen -a myapp --path "100,100 260,100" --eraser +``` +- `--pressure` 0.0–1.0, `--tilt-x`/`--tilt-y` ±90°, `--eraser` for the eraser end. Same injection safety as `touch`: requires a non-zero foregrounded target window and bounds-checks every ink point (out-of-bounds → `invalid_arguments`, nothing injected). + ### Read element state ```powershell # Read text/value content (works for RichEditBox, TextBox, ComboBox, Slider, labels) @@ -434,6 +462,55 @@ Press the mouse button at one point, move to another, then release. 'drag | `--right` | Drag with the right mouse button instead of the left button | (none) | | `--window` | Target window by HWND (stable handle from list output). Takes precedence over --app. | (none) | +### `winapp ui touch` + +Inject synthetic touch input using the Windows touch-injection API. Supports tap, double-tap, long-press, swipe, pinch and stretch gestures at an element's center or explicit app x,y coordinates. Requires an unlocked, interactive desktop with the target window foregroundable. + +#### Arguments + +| Argument | Required | Description | +|----------|----------|-------------| +| `` | No | Semantic slug (e.g., btn-minimize-d1a0) or text to search by name/automationId | + +#### Options + +| Option | Description | Default | +|--------|-------------|---------| +| `--app` | Target app (process name, window title, or PID). Lists windows if ambiguous. | (none) | +| `--at` | Explicit start point as app coordinates x,y (as reported by 'ui inspect'). Defaults to the selector's element center. | (none) | +| `--distance` | Distance in pixels for pinch/stretch (finger spread) or a directionless swipe. | (none) | +| `--duration-ms` | Glide time in milliseconds for moving gestures (swipe/pinch/stretch). | `300` | +| `--fingers` | Number of touch contacts (default: 1). Pinch/stretch always use 2. | `1` | +| `--gesture` | Gesture to perform: tap, double-tap, long-press, swipe, pinch, stretch (default: tap). | `tap` | +| `--hold-ms` | Milliseconds to hold contacts down before lifting (long-press hold time). | (none) | +| `--json` | Format output as JSON | (none) | +| `--to-point` | End point x,y for a swipe (app coordinates). | (none) | +| `--window` | Target window by HWND (stable handle from list output). Takes precedence over --app. | (none) | + +### `winapp ui pen` + +Inject synthetic pen/stylus input using the Windows synthetic-pointer API. Taps or draws ink strokes with configurable pressure, tilt and eraser mode, at an element's center or explicit app x,y coordinates. Requires an unlocked, interactive desktop with the target window foregroundable (Windows 10 1809+). + +#### Arguments + +| Argument | Required | Description | +|----------|----------|-------------| +| `` | No | Semantic slug (e.g., btn-minimize-d1a0) or text to search by name/automationId | + +#### Options + +| Option | Description | Default | +|--------|-------------|---------| +| `--app` | Target app (process name, window title, or PID). Lists windows if ambiguous. | (none) | +| `--at` | Pen contact point as app coordinates x,y (as reported by 'ui inspect'). Defaults to the selector's element center. Ignored when --path is given. | (none) | +| `--eraser` | Use the eraser end of the pen instead of the tip. | (none) | +| `--json` | Format output as JSON | (none) | +| `--path` | Ink stroke path as a whitespace-separated list of x,y pairs, e.g. "10,10 20,30 40,50". | (none) | +| `--pressure` | Pen pressure from 0.0 to 1.0 (default: 0.5). | `0.5` | +| `--tilt-x` | Pen tilt along the x-axis in degrees (-90 to 90, default: 0). | (none) | +| `--tilt-y` | Pen tilt along the y-axis in degrees (-90 to 90, default: 0). | (none) | +| `--window` | Target window by HWND (stable handle from list output). Takes precedence over --app. | (none) | + ### `winapp ui hover` Move the mouse to an element's center to trigger hover effects (tooltips, flyouts, visual states). Uses SendInput for realistic mouse movement and waits for a configurable dwell time. diff --git a/.github/plugin/skills/winapp-cli/ui-automation/SKILL.md b/.github/plugin/skills/winapp-cli/ui-automation/SKILL.md index e5370488..c4e8b710 100644 --- a/.github/plugin/skills/winapp-cli/ui-automation/SKILL.md +++ b/.github/plugin/skills/winapp-cli/ui-automation/SKILL.md @@ -12,7 +12,7 @@ version: 0.4.1 ## Prerequisites - For UIA mode (any app): No setup needed — works with any running Windows app -- For input-injecting verbs (`click`, `hover`, `drag`, `scroll --wheel`, `send-keys --via send-input`): an **unlocked, interactive desktop** with the target window foregroundable. On a locked/secure desktop they fail fast with `no_interactive_desktop`. The UIA-pattern verbs (`inspect`, `search`, `get-*`, `wait-for`, `set-value`, `invoke`, `scroll --direction/--to`, `screenshot`) are headless/locked-session friendly — prefer them in CI. +- For input-injecting verbs (`click`, `hover`, `drag`, `touch`, `pen`, `scroll --wheel`, `send-keys --via send-input`): an **unlocked, interactive desktop** with the target window foregroundable. On a locked/secure desktop they fail fast with `no_interactive_desktop`. The UIA-pattern verbs (`inspect`, `search`, `get-*`, `wait-for`, `set-value`, `invoke`, `scroll --direction/--to`, `screenshot`) are headless/locked-session friendly — prefer them in CI. ## Common patterns @@ -157,6 +157,34 @@ winapp ui drag itm-card-9f8e itm-trash-0001 -a myapp --right ``` - A selector drags from/to the element's center; `x,y` are app coordinates in the same space `ui inspect`/`search` report. Element endpoints are re-resolved just before the drag and fail with `target_moved` if still animating; on a locked/secure desktop the drag fails with `no_interactive_desktop`. +### Touch gestures (tap, swipe, pinch, stretch, long-press) +Inject synthetic touch. The contact anchor is an element selector (its center) or an explicit `--at x,y` app coordinate. Prefers the modern synthetic-pointer device and falls back to the legacy touch-injection API. +```powershell +# Tap an element center; or tap explicit app coordinates +winapp ui touch btn-ok-1a2b -a myapp +winapp ui touch -a myapp --at 320,240 + +# Long-press, swipe, and two-finger pinch/stretch (zoom) +winapp ui touch tile-photo-7b3c -a myapp --gesture long-press --hold-ms 600 +winapp ui touch -a myapp --at 100,300 --gesture swipe --to-point 400,300 +winapp ui touch img-map-9f8e -a myapp --gesture pinch --distance 200 +winapp ui touch img-map-9f8e -a myapp --gesture stretch --distance 200 +``` +- Gestures: `tap` (default), `double-tap`, `long-press`, `swipe`, `pinch`, `stretch`. `--fingers` 1–10 (pinch/stretch always 2). Refuses without a non-zero foregrounded target (`no_target`/`foreground_not_target`/`no_interactive_desktop`); every coordinate is bounds-checked against the target window and rejected with `invalid_arguments` if outside. If injected touch is unsupported on the device, the command surfaces the real Win32 error rather than a false success. + +### Pen / stylus (taps and ink strokes) +Inject synthetic pen input (Windows 10 1809+). Target an element center, an explicit `--at`, or a full `--path` ink stroke, with pressure/tilt/eraser control. +```powershell +# Pen tap at element center; firm tap at explicit coords +winapp ui pen canvas-1a2b -a myapp +winapp ui pen -a myapp --at 320,240 --pressure 0.8 + +# Draw an ink stroke, or erase along one +winapp ui pen -a myapp --path "100,100 150,120 210,140 260,120" +winapp ui pen -a myapp --path "100,100 260,100" --eraser +``` +- `--pressure` 0.0–1.0, `--tilt-x`/`--tilt-y` ±90°, `--eraser` for the eraser end. Same injection safety as `touch`: requires a non-zero foregrounded target window and bounds-checks every ink point (out-of-bounds → `invalid_arguments`, nothing injected). + ### Read element state ```powershell # Read text/value content (works for RichEditBox, TextBox, ComboBox, Slider, labels) @@ -434,6 +462,55 @@ Press the mouse button at one point, move to another, then release. 'drag | `--right` | Drag with the right mouse button instead of the left button | (none) | | `--window` | Target window by HWND (stable handle from list output). Takes precedence over --app. | (none) | +### `winapp ui touch` + +Inject synthetic touch input using the Windows touch-injection API. Supports tap, double-tap, long-press, swipe, pinch and stretch gestures at an element's center or explicit app x,y coordinates. Requires an unlocked, interactive desktop with the target window foregroundable. + +#### Arguments + +| Argument | Required | Description | +|----------|----------|-------------| +| `` | No | Semantic slug (e.g., btn-minimize-d1a0) or text to search by name/automationId | + +#### Options + +| Option | Description | Default | +|--------|-------------|---------| +| `--app` | Target app (process name, window title, or PID). Lists windows if ambiguous. | (none) | +| `--at` | Explicit start point as app coordinates x,y (as reported by 'ui inspect'). Defaults to the selector's element center. | (none) | +| `--distance` | Distance in pixels for pinch/stretch (finger spread) or a directionless swipe. | (none) | +| `--duration-ms` | Glide time in milliseconds for moving gestures (swipe/pinch/stretch). | `300` | +| `--fingers` | Number of touch contacts (default: 1). Pinch/stretch always use 2. | `1` | +| `--gesture` | Gesture to perform: tap, double-tap, long-press, swipe, pinch, stretch (default: tap). | `tap` | +| `--hold-ms` | Milliseconds to hold contacts down before lifting (long-press hold time). | (none) | +| `--json` | Format output as JSON | (none) | +| `--to-point` | End point x,y for a swipe (app coordinates). | (none) | +| `--window` | Target window by HWND (stable handle from list output). Takes precedence over --app. | (none) | + +### `winapp ui pen` + +Inject synthetic pen/stylus input using the Windows synthetic-pointer API. Taps or draws ink strokes with configurable pressure, tilt and eraser mode, at an element's center or explicit app x,y coordinates. Requires an unlocked, interactive desktop with the target window foregroundable (Windows 10 1809+). + +#### Arguments + +| Argument | Required | Description | +|----------|----------|-------------| +| `` | No | Semantic slug (e.g., btn-minimize-d1a0) or text to search by name/automationId | + +#### Options + +| Option | Description | Default | +|--------|-------------|---------| +| `--app` | Target app (process name, window title, or PID). Lists windows if ambiguous. | (none) | +| `--at` | Pen contact point as app coordinates x,y (as reported by 'ui inspect'). Defaults to the selector's element center. Ignored when --path is given. | (none) | +| `--eraser` | Use the eraser end of the pen instead of the tip. | (none) | +| `--json` | Format output as JSON | (none) | +| `--path` | Ink stroke path as a whitespace-separated list of x,y pairs, e.g. "10,10 20,30 40,50". | (none) | +| `--pressure` | Pen pressure from 0.0 to 1.0 (default: 0.5). | `0.5` | +| `--tilt-x` | Pen tilt along the x-axis in degrees (-90 to 90, default: 0). | (none) | +| `--tilt-y` | Pen tilt along the y-axis in degrees (-90 to 90, default: 0). | (none) | +| `--window` | Target window by HWND (stable handle from list output). Takes precedence over --app. | (none) | + ### `winapp ui hover` Move the mouse to an element's center to trigger hover effects (tooltips, flyouts, visual states). Uses SendInput for realistic mouse movement and waits for a configurable dwell time. diff --git a/docs/cli-schema.json b/docs/cli-schema.json index f18bc2c1..030d8b2e 100644 --- a/docs/cli-schema.json +++ b/docs/cli-schema.json @@ -2765,6 +2765,176 @@ } } }, + "pen": { + "description": "Inject synthetic pen/stylus input using the Windows synthetic-pointer API. Taps or draws ink strokes with configurable pressure, tilt and eraser mode, at an element's center or explicit app x,y coordinates. Requires an unlocked, interactive desktop with the target window foregroundable (Windows 10 1809+).", + "hidden": false, + "arguments": { + "selector": { + "description": "Semantic slug (e.g., btn-minimize-d1a0) or text to search by name/automationId", + "order": 0, + "hidden": false, + "valueType": "System.String", + "hasDefaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + } + } + }, + "options": { + "--app": { + "description": "Target app (process name, window title, or PID). Lists windows if ambiguous.", + "hidden": false, + "aliases": [ + "-a" + ], + "valueType": "System.String", + "hasDefaultValue": false, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--at": { + "description": "Pen contact point as app coordinates x,y (as reported by 'ui inspect'). Defaults to the selector's element center. Ignored when --path is given.", + "hidden": false, + "valueType": "System.String", + "hasDefaultValue": false, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--eraser": { + "description": "Use the eraser end of the pen instead of the tip.", + "hidden": false, + "valueType": "System.Boolean", + "hasDefaultValue": true, + "defaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--json": { + "description": "Format output as JSON", + "hidden": false, + "valueType": "System.Boolean", + "hasDefaultValue": true, + "defaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--path": { + "description": "Ink stroke path as a whitespace-separated list of x,y pairs, e.g. \"10,10 20,30 40,50\".", + "hidden": false, + "valueType": "System.String", + "hasDefaultValue": false, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--pressure": { + "description": "Pen pressure from 0.0 to 1.0 (default: 0.5).", + "hidden": false, + "valueType": "System.Single", + "hasDefaultValue": true, + "defaultValue": 0.5, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--quiet": { + "description": "Suppress progress messages", + "hidden": false, + "aliases": [ + "-q" + ], + "valueType": "System.Boolean", + "hasDefaultValue": true, + "defaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--tilt-x": { + "description": "Pen tilt along the x-axis in degrees (-90 to 90, default: 0).", + "hidden": false, + "valueType": "System.Int32", + "hasDefaultValue": true, + "defaultValue": 0, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--tilt-y": { + "description": "Pen tilt along the y-axis in degrees (-90 to 90, default: 0).", + "hidden": false, + "valueType": "System.Int32", + "hasDefaultValue": true, + "defaultValue": 0, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--verbose": { + "description": "Enable verbose output", + "hidden": false, + "aliases": [ + "-v" + ], + "valueType": "System.Boolean", + "hasDefaultValue": true, + "defaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--window": { + "description": "Target window by HWND (stable handle from list output). Takes precedence over --app.", + "hidden": false, + "aliases": [ + "-w" + ], + "valueType": "System.Nullable", + "hasDefaultValue": false, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + } + } + }, "screenshot": { "description": "Capture the target window or element as a PNG image. When multiple windows exist (e.g., dialogs), captures each to a separate file. With --json, returns file path and dimensions. Use --capture-screen for popup overlays.", "hidden": false, @@ -3549,6 +3719,192 @@ } } }, + "touch": { + "description": "Inject synthetic touch input using the Windows touch-injection API. Supports tap, double-tap, long-press, swipe, pinch and stretch gestures at an element's center or explicit app x,y coordinates. Requires an unlocked, interactive desktop with the target window foregroundable.", + "hidden": false, + "arguments": { + "selector": { + "description": "Semantic slug (e.g., btn-minimize-d1a0) or text to search by name/automationId", + "order": 0, + "hidden": false, + "valueType": "System.String", + "hasDefaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + } + } + }, + "options": { + "--app": { + "description": "Target app (process name, window title, or PID). Lists windows if ambiguous.", + "hidden": false, + "aliases": [ + "-a" + ], + "valueType": "System.String", + "hasDefaultValue": false, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--at": { + "description": "Explicit start point as app coordinates x,y (as reported by 'ui inspect'). Defaults to the selector's element center.", + "hidden": false, + "valueType": "System.String", + "hasDefaultValue": false, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--distance": { + "description": "Distance in pixels for pinch/stretch (finger spread) or a directionless swipe.", + "hidden": false, + "valueType": "System.Int32", + "hasDefaultValue": true, + "defaultValue": 0, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--duration-ms": { + "description": "Glide time in milliseconds for moving gestures (swipe/pinch/stretch).", + "hidden": false, + "valueType": "System.Int32", + "hasDefaultValue": true, + "defaultValue": 300, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--fingers": { + "description": "Number of touch contacts (default: 1). Pinch/stretch always use 2.", + "hidden": false, + "valueType": "System.Int32", + "hasDefaultValue": true, + "defaultValue": 1, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--gesture": { + "description": "Gesture to perform: tap, double-tap, long-press, swipe, pinch, stretch (default: tap).", + "hidden": false, + "aliases": [ + "-g" + ], + "valueType": "System.String", + "hasDefaultValue": true, + "defaultValue": "tap", + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--hold-ms": { + "description": "Milliseconds to hold contacts down before lifting (long-press hold time).", + "hidden": false, + "valueType": "System.Int32", + "hasDefaultValue": true, + "defaultValue": 0, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--json": { + "description": "Format output as JSON", + "hidden": false, + "valueType": "System.Boolean", + "hasDefaultValue": true, + "defaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--quiet": { + "description": "Suppress progress messages", + "hidden": false, + "aliases": [ + "-q" + ], + "valueType": "System.Boolean", + "hasDefaultValue": true, + "defaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--to-point": { + "description": "End point x,y for a swipe (app coordinates).", + "hidden": false, + "valueType": "System.String", + "hasDefaultValue": false, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--verbose": { + "description": "Enable verbose output", + "hidden": false, + "aliases": [ + "-v" + ], + "valueType": "System.Boolean", + "hasDefaultValue": true, + "defaultValue": false, + "arity": { + "minimum": 0, + "maximum": 1 + }, + "required": false, + "recursive": false + }, + "--window": { + "description": "Target window by HWND (stable handle from list output). Takes precedence over --app.", + "hidden": false, + "aliases": [ + "-w" + ], + "valueType": "System.Nullable", + "hasDefaultValue": false, + "arity": { + "minimum": 1, + "maximum": 1 + }, + "required": false, + "recursive": false + } + } + }, "wait-for": { "description": "Wait for an element to appear, disappear, or have a property reach a target value. Polls at 100ms intervals until condition met or timeout.", "hidden": false, diff --git a/docs/fragments/skills/winapp-cli/ui-automation.md b/docs/fragments/skills/winapp-cli/ui-automation.md index c5824aa8..f7c8dcc2 100644 --- a/docs/fragments/skills/winapp-cli/ui-automation.md +++ b/docs/fragments/skills/winapp-cli/ui-automation.md @@ -7,7 +7,7 @@ ## Prerequisites - For UIA mode (any app): No setup needed — works with any running Windows app -- For input-injecting verbs (`click`, `hover`, `drag`, `scroll --wheel`, `send-keys --via send-input`): an **unlocked, interactive desktop** with the target window foregroundable. On a locked/secure desktop they fail fast with `no_interactive_desktop`. The UIA-pattern verbs (`inspect`, `search`, `get-*`, `wait-for`, `set-value`, `invoke`, `scroll --direction/--to`, `screenshot`) are headless/locked-session friendly — prefer them in CI. +- For input-injecting verbs (`click`, `hover`, `drag`, `touch`, `pen`, `scroll --wheel`, `send-keys --via send-input`): an **unlocked, interactive desktop** with the target window foregroundable. On a locked/secure desktop they fail fast with `no_interactive_desktop`. The UIA-pattern verbs (`inspect`, `search`, `get-*`, `wait-for`, `set-value`, `invoke`, `scroll --direction/--to`, `screenshot`) are headless/locked-session friendly — prefer them in CI. ## Common patterns @@ -152,6 +152,34 @@ winapp ui drag itm-card-9f8e itm-trash-0001 -a myapp --right ``` - A selector drags from/to the element's center; `x,y` are app coordinates in the same space `ui inspect`/`search` report. Element endpoints are re-resolved just before the drag and fail with `target_moved` if still animating; on a locked/secure desktop the drag fails with `no_interactive_desktop`. +### Touch gestures (tap, swipe, pinch, stretch, long-press) +Inject synthetic touch. The contact anchor is an element selector (its center) or an explicit `--at x,y` app coordinate. Prefers the modern synthetic-pointer device and falls back to the legacy touch-injection API. +```powershell +# Tap an element center; or tap explicit app coordinates +winapp ui touch btn-ok-1a2b -a myapp +winapp ui touch -a myapp --at 320,240 + +# Long-press, swipe, and two-finger pinch/stretch (zoom) +winapp ui touch tile-photo-7b3c -a myapp --gesture long-press --hold-ms 600 +winapp ui touch -a myapp --at 100,300 --gesture swipe --to-point 400,300 +winapp ui touch img-map-9f8e -a myapp --gesture pinch --distance 200 +winapp ui touch img-map-9f8e -a myapp --gesture stretch --distance 200 +``` +- Gestures: `tap` (default), `double-tap`, `long-press`, `swipe`, `pinch`, `stretch`. `--fingers` 1–10 (pinch/stretch always 2). Refuses without a non-zero foregrounded target (`no_target`/`foreground_not_target`/`no_interactive_desktop`); every coordinate is bounds-checked against the target window and rejected with `invalid_arguments` if outside. If injected touch is unsupported on the device, the command surfaces the real Win32 error rather than a false success. + +### Pen / stylus (taps and ink strokes) +Inject synthetic pen input (Windows 10 1809+). Target an element center, an explicit `--at`, or a full `--path` ink stroke, with pressure/tilt/eraser control. +```powershell +# Pen tap at element center; firm tap at explicit coords +winapp ui pen canvas-1a2b -a myapp +winapp ui pen -a myapp --at 320,240 --pressure 0.8 + +# Draw an ink stroke, or erase along one +winapp ui pen -a myapp --path "100,100 150,120 210,140 260,120" +winapp ui pen -a myapp --path "100,100 260,100" --eraser +``` +- `--pressure` 0.0–1.0, `--tilt-x`/`--tilt-y` ±90°, `--eraser` for the eraser end. Same injection safety as `touch`: requires a non-zero foregrounded target window and bounds-checks every ink point (out-of-bounds → `invalid_arguments`, nothing injected). + ### Read element state ```powershell # Read text/value content (works for RichEditBox, TextBox, ComboBox, Slider, labels) diff --git a/docs/ui-automation.md b/docs/ui-automation.md index 6443dd43..63851e6a 100644 --- a/docs/ui-automation.md +++ b/docs/ui-automation.md @@ -10,9 +10,9 @@ Used by AI agents and developers for UI testing, debugging, and automation. `winapp ui` provides commands for inspecting and interacting with Windows app UIs. Uses Windows UI Automation (UIA). Works with any Windows app — WPF, WinForms, Win32, Electron, and WinUI 3. -Most commands drive the app through UIA patterns (no input injection). The exceptions inject real input: `ui click`/`ui hover`/`ui drag` use mouse simulation, and `ui send-keys` synthesizes keyboard input — for controls and scenarios that UIA patterns can't drive. +Most commands drive the app through UIA patterns (no input injection). The exceptions inject real input: `ui click`/`ui hover`/`ui drag` use mouse simulation, `ui touch`/`ui pen` synthesize touch and pen/stylus input, and `ui send-keys` synthesizes keyboard input — for controls and scenarios that UIA patterns can't drive. -> **Interactive-desktop requirement (input-injecting verbs).** `click`, `hover`, `drag`, `scroll --wheel`, and `send-keys --via send-input` synthesize OS-level input, so they need an **unlocked, interactive desktop** with the target window in the foreground. On a **locked workstation or secure desktop** (LogonUI/UAC) they can't inject and fail fast with **`no_interactive_desktop`** (distinct from the elevation/`foreground_not_target` cases). Everything else — `inspect`, `search`, `get-property`, `get-value`, `wait-for`, `set-value`, `invoke`, `scroll --direction/--to`, `screenshot` — drives the app through UIA patterns and is **headless/locked-session friendly**. Prefer the UIA-pattern verbs in CI; reserve the injection verbs for scenarios that genuinely need real input. Before injecting, the gesture verbs also **re-resolve the target element** and refuse with **`target_moved`** if it's still animating/relocating, rather than landing input on empty space. +> **Interactive-desktop requirement (input-injecting verbs).** `click`, `hover`, `drag`, `touch`, `pen`, `scroll --wheel`, and `send-keys --via send-input` synthesize OS-level input, so they need an **unlocked, interactive desktop** with the target window in the foreground. On a **locked workstation or secure desktop** (LogonUI/UAC) they can't inject and fail fast with **`no_interactive_desktop`** (distinct from the elevation/`foreground_not_target` cases). `touch`/`pen` additionally refuse when no window resolves (**`no_target`**) and reject coordinates outside the target window (**`invalid_arguments`**). Everything else — `inspect`, `search`, `get-property`, `get-value`, `wait-for`, `set-value`, `invoke`, `scroll --direction/--to`, `screenshot` — drives the app through UIA patterns and is **headless/locked-session friendly**. Prefer the UIA-pattern verbs in CI; reserve the injection verbs for scenarios that genuinely need real input. Before injecting, the gesture verbs also **re-resolve the target element** and refuse with **`target_moved`** if it's still animating/relocating, rather than landing input on empty space. ## Quick Start @@ -285,6 +285,49 @@ winapp ui drag itm-card-9f8e pane-left-2c1a -a myapp --dwell-ms 350 # settl > Like `send-keys --via send-input`, `drag` injects OS-wide at screen coordinates after bringing the target to the foreground. If focus can't be brought to the target (e.g. focus-stealing prevention from a background process), the command **fails (`foreground_not_target`)** rather than dragging on the wrong window — focus or click the window first. On a locked/secure desktop it fails with **`no_interactive_desktop`**. Each element endpoint is **re-resolved immediately before the drag**; if it's still moving/resizing (an animating target), the command fails with **`target_moved`** instead of dragging to a stale point. (Bare `x,y` endpoints can't be re-verified, so they're used as-is.) +### touch +Inject synthetic **touch** gestures using the Windows pointer-injection API. The contact anchor is either an **element selector** (uses the element's center) or an explicit **app coordinate `x,y`** via `--at` (same space `winapp ui inspect` reports). Use it for tap/press interactions and multi-touch gestures that mouse simulation can't express. +```bash +winapp ui touch btn-ok-1a2b -a myapp # tap at the element center +winapp ui touch -a myapp --at 320,240 # tap at explicit app coords +winapp ui touch tile-photo-7b3c -a myapp --gesture long-press --hold-ms 600 +winapp ui touch -a myapp --at 100,300 --gesture swipe --to-point 400,300 +winapp ui touch img-map-9f8e -a myapp --gesture pinch --distance 200 # pinch-to-zoom out (2 fingers) +winapp ui touch img-map-9f8e -a myapp --gesture stretch --distance 200 # stretch-to-zoom in (2 fingers) +``` + +**Options:** +- `--gesture ` — `tap` (default), `double-tap`, `long-press`, `swipe`, `pinch`, `stretch`. +- `--at ` — Explicit start point (app coordinates). Defaults to the selector's element center. +- `--to-point ` — End point for a `swipe`. +- `--distance ` — Finger spread for `pinch`/`stretch`, or a directionless `swipe` distance. +- `--hold-ms ` — Hold contacts down before lifting (long-press hold time). +- `--duration-ms ` — Glide time for moving gestures (swipe/pinch/stretch; default 300). +- `--fingers ` — Number of contacts (1–10; default 1). `pinch`/`stretch` always use 2. + +> **Injection safety.** `touch` refuses to inject unless a **non-zero target window handle** resolves and that window holds the foreground — it fails with **`no_target`** when no window can be resolved, **`foreground_not_target`** if focus couldn't be transferred, or **`no_interactive_desktop`** on a locked/secure desktop. Every coordinate (element center, explicit `--at`/`--to-point`, and generated waypoints) is **bounds-checked against the target window rectangle**; a point outside the window is rejected with **`invalid_arguments`** and nothing is injected. `--fingers` above 10 is rejected up front. +> +> **Hardware note.** Touch prefers the modern synthetic-pointer device (`CreateSyntheticPointerDevice(PT_TOUCH)`) and falls back to the legacy `InitializeTouchInjection`/`InjectTouchInput` API. If injection is unsupported on the current device/session, the command surfaces the **actual Win32 error code** (e.g. "unsupported") rather than reporting a false success — treat a non-zero exit as "touch not delivered". + +### pen +Inject synthetic **pen/stylus** input — taps and ink strokes — using the Windows synthetic-pointer API (`CreateSyntheticPointerDevice(PT_PEN)`; Windows 10 1809+). Target an element center, an explicit `--at` point, or a full `--path` ink stroke. +```bash +winapp ui pen canvas-1a2b -a myapp # pen tap at the element center +winapp ui pen -a myapp --at 320,240 --pressure 0.8 # firm pen tap at explicit coords +winapp ui pen -a myapp --path "100,100 150,120 210,140 260,120" # draw an ink stroke +winapp ui pen -a myapp --path "100,100 260,100" --eraser # erase along a stroke +winapp ui pen -a myapp --at 200,200 --tilt-x 30 --tilt-y -15 # tilted pen contact +``` + +**Options:** +- `--at ` — Pen contact point (app coordinates). Defaults to the selector's element center. Ignored when `--path` is given. +- `--path ""` — Ink stroke path as whitespace-separated `x,y` pairs (a one-point path is a tap). +- `--pressure <0.0–1.0>` — Pen pressure (default 0.5). +- `--tilt-x ` / `--tilt-y ` — Pen tilt angles, −90 to 90 (default 0). +- `--eraser` — Use the eraser end of the pen instead of the tip. + +> **Injection safety.** Like `touch`, `pen` refuses to inject without a **non-zero, foregrounded target window** (`no_target` / `foreground_not_target` / `no_interactive_desktop`) and **bounds-checks every ink point** against the target window rectangle, rejecting out-of-bounds coordinates with **`invalid_arguments`** before any input is injected. Invalid `--pressure` (outside 0.0–1.0) or tilt (outside ±90°) are rejected up front. + ### hover Move the mouse to an element's center to trigger hover effects (tooltips, flyouts, visual states). Uses `SendInput` for realistic mouse movement with a small wiggle, then waits for a configurable dwell time. ```bash diff --git a/docs/usage.md b/docs/usage.md index 5ce162ba..8932ee3b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1084,6 +1084,8 @@ winapp ui [command] [options] - `click` - Click element via mouse simulation (for controls that don't support invoke) - `hover` - Move mouse to element to trigger tooltips, flyouts, and hover states (default dwell: 800ms) - `drag` - Drag the mouse from one point to another, by element selector or app `x,y` coordinates (reorder, resize, sliders, drag-and-drop) +- `touch` - Inject synthetic touch gestures (tap, double-tap, long-press, swipe, pinch, stretch) at an element center or app `x,y` coordinates +- `pen` - Inject synthetic pen/stylus input — taps and ink strokes with configurable pressure, tilt, and eraser mode - `send-keys` - Send synthetic keyboard input (named keys, combos, raw vk=0xNN, or literal text) to a window - `set-value` - Set value on editable element (text, number) - `focus` - Move keyboard focus diff --git a/scripts/generate-llm-docs.ps1 b/scripts/generate-llm-docs.ps1 index ff2735d4..d98bdd46 100644 --- a/scripts/generate-llm-docs.ps1 +++ b/scripts/generate-llm-docs.ps1 @@ -104,7 +104,7 @@ $SkillCommandMap = @{ "manifest" = @("manifest generate", "manifest update-assets", "manifest add-alias") "troubleshoot" = @("get-winapp-path", "tool", "store") "frameworks" = @() # No auto-generated command sections — links to guides - "ui-automation" = @("ui status", "ui inspect", "ui search", "ui get-property", "ui get-value", "ui screenshot", "ui invoke", "ui click", "ui drag", "ui hover", "ui send-keys", "ui set-value", "ui focus", "ui scroll-into-view", "ui scroll", "ui wait-for", "ui list-windows", "ui get-focused") + "ui-automation" = @("ui status", "ui inspect", "ui search", "ui get-property", "ui get-value", "ui screenshot", "ui invoke", "ui click", "ui drag", "ui touch", "ui pen", "ui hover", "ui send-keys", "ui set-value", "ui focus", "ui scroll-into-view", "ui scroll", "ui wait-for", "ui list-windows", "ui get-focused") } # Validate that all CLI commands are covered by at least one skill diff --git a/src/winapp-CLI/WinApp.Cli.Tests/FakeUiServices.cs b/src/winapp-CLI/WinApp.Cli.Tests/FakeUiServices.cs index 9be38fd0..422e22c8 100644 --- a/src/winapp-CLI/WinApp.Cli.Tests/FakeUiServices.cs +++ b/src/winapp-CLI/WinApp.Cli.Tests/FakeUiServices.cs @@ -43,6 +43,29 @@ internal class FakeUiAutomationService : IUiAutomationService public List<(nint Hwnd, int Pid, string Title)> FindWindowsByTitle(string titleQuery) => WindowsByTitleResult; public List<(nint Hwnd, int Pid, string Title)> FindWindowsByPid(int pid) => WindowsByPidResult; + /// The rectangle returned for any nonzero handle (default: 0,0 – 1920,1080). + public WinApp.Cli.Helpers.PointerRect WindowRect { get; set; } = new(0, 0, 1920, 1080); + + /// When , reports the window rect as unreadable (returns false). + public bool WindowRectAllow { get; set; } = true; + + /// Records each call so tests can distinguish the + /// hwnd-0 rejection (no lookup) from the out-of-bounds rejection (lookup happened). + public List WindowRectCalls { get; } = []; + + public bool TryGetWindowRect(long hwnd, out WinApp.Cli.Helpers.PointerRect rect) + { + WindowRectCalls.Add(hwnd); + if (!WindowRectAllow || hwnd == 0) + { + rect = default; + return false; + } + + rect = WindowRect; + return true; + } + public Task InspectAsync(UiSessionInfo session, string? elementId, int depth, CancellationToken ct) => Task.FromResult(InspectResult); @@ -155,6 +178,44 @@ public void ScrollWheel(int screenX, int screenY, int delta, int settleMs = 30) => ScrollWheelCalls.Add(new(screenX, screenY, delta, settleMs)); } +/// +/// Fake pointer input for testing — records injected touch contacts and pen strokes instead of +/// issuing real synthetic-pointer injection. +/// +internal class FakePointerInput : WinApp.Cli.Helpers.IPointerInput +{ + public record TouchCall( + WinApp.Cli.Helpers.TouchGesture Gesture, + IReadOnlyList> ContactPaths, + int HoldMs, + int DurationMs); + + public record PenCall( + IReadOnlyList Path, + float Pressure, + int TiltX, + int TiltY, + bool Eraser); + + public List TouchCalls { get; } = []; + public List PenCalls { get; } = []; + + public void Touch( + WinApp.Cli.Helpers.TouchGesture gesture, + IReadOnlyList> contactPaths, + int holdMs, + int durationMs) + => TouchCalls.Add(new(gesture, contactPaths, holdMs, durationMs)); + + public void Pen( + IReadOnlyList path, + float pressure, + int tiltX, + int tiltY, + bool eraser) + => PenCalls.Add(new(path, pressure, tiltX, tiltY, eraser)); +} + /// /// Fake keyboard input for testing — records the actions/transport instead of issuing real input. /// diff --git a/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Pen.cs b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Pen.cs new file mode 100644 index 00000000..f879ca08 --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Pen.cs @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using WinApp.Cli.Commands; +using WinApp.Cli.Helpers; +using WinApp.Cli.Models; + +namespace WinApp.Cli.Tests; + +public partial class UiCommandTests +{ + // --------------------------------------------------------------------- + // pen — synthetic pen/stylus taps and ink strokes at a selector center, + // explicit --at, or an explicit --path. Exercised via FakePointerInput + // (records strokes), FakeForegroundGuard and FakeUiAutomationService (window rect). + // --------------------------------------------------------------------- + + [TestMethod] + public async Task Pen_Tap_SelectorCenter_InjectsPenPoint() + { + // Element center = (50 + 60/2, 60 + 40/2) = (80, 80). + _fakeUia.FindSingleResult = new UiElement + { + Id = "e0", Type = "Image", Selector = "canvas-1234", + X = 50, Y = 60, Width = 60, Height = 40, WindowHandle = 9001 + }; + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["canvas-1234", "-a", "TestApp", "--json"]); + Assert.AreEqual(0, exitCode); + + Assert.AreEqual(1, _fakePointer.PenCalls.Count); + var call = _fakePointer.PenCalls[0]; + Assert.AreEqual(1, call.Path.Count); + Assert.AreEqual(new PointerPoint(80, 80), call.Path[0]); + + var result = System.Text.Json.JsonSerializer.Deserialize(TestAnsiConsole.Output); + Assert.AreEqual("tap", result.GetProperty("action").GetString()); + Assert.AreEqual(9001, result.GetProperty("hwnd").GetInt64()); + } + + [TestMethod] + public async Task Pen_Path_MultiPoint_InjectsStroke() + { + _fakeSession.SessionResult.WindowHandle = 3300; + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, + ["-a", "TestApp", "--path", "10,10 20,30 40,50", "--json"]); + Assert.AreEqual(0, exitCode); + + Assert.AreEqual(1, _fakePointer.PenCalls.Count); + var call = _fakePointer.PenCalls[0]; + Assert.AreEqual(3, call.Path.Count); + Assert.AreEqual(new PointerPoint(10, 10), call.Path[0]); + Assert.AreEqual(new PointerPoint(20, 30), call.Path[1]); + Assert.AreEqual(new PointerPoint(40, 50), call.Path[2]); + + var result = System.Text.Json.JsonSerializer.Deserialize(TestAnsiConsole.Output); + Assert.AreEqual("draw", result.GetProperty("action").GetString()); + } + + [TestMethod] + public async Task Pen_InvalidPressure_Rejected_NoInjection() + { + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, + ["-a", "TestApp", "--at", "100,100", "--pressure", "1.5", "--json"]); + Assert.AreEqual(1, exitCode); + Assert.AreEqual(0, _fakePointer.PenCalls.Count); + } + + [TestMethod] + public async Task Pen_Eraser_SetsFlag() + { + _fakeSession.SessionResult.WindowHandle = 4400; + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, + ["-a", "TestApp", "--at", "120,120", "--eraser", "--json"]); + Assert.AreEqual(0, exitCode); + + Assert.AreEqual(1, _fakePointer.PenCalls.Count); + Assert.IsTrue(_fakePointer.PenCalls[0].Eraser); + + var result = System.Text.Json.JsonSerializer.Deserialize(TestAnsiConsole.Output); + Assert.AreEqual("erase", result.GetProperty("action").GetString()); + Assert.IsTrue(result.GetProperty("eraser").GetBoolean()); + } + + [TestMethod] + public async Task Pen_PathOutsideWindow_Rejected_NoInjection() + { + _fakeSession.SessionResult.WindowHandle = 6600; + _fakeUia.WindowRect = new PointerRect(0, 0, 800, 600); + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, + ["-a", "TestApp", "--path", "10,10 9000,9000", "--json"]); + Assert.AreEqual(1, exitCode); + Assert.AreEqual(0, _fakePointer.PenCalls.Count); + Assert.IsTrue(_fakeUia.WindowRectCalls.Count >= 1); + Assert.AreEqual(0, _fakeForeground.Calls.Count); + } + + [TestMethod] + public async Task Pen_ZeroTargetHwnd_Rejected_NoInjection() + { + _fakeSession.SessionResult.WindowHandle = 0; + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, + ["-a", "TestApp", "--at", "100,100", "--json"]); + Assert.AreEqual(1, exitCode); + Assert.AreEqual(0, _fakePointer.PenCalls.Count); + Assert.AreEqual(0, _fakeUia.WindowRectCalls.Count); + Assert.AreEqual(0, _fakeForeground.Calls.Count); + } + + [TestMethod] + public async Task Pen_WindowRectUnreadable_Rejected_NoInjection() + { + // Nonzero handle resolves but its rect can't be read → no verifiable target (no_target). + _fakeSession.SessionResult.WindowHandle = 6700; + _fakeUia.WindowRectAllow = false; + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, + ["-a", "TestApp", "--at", "100,100", "--json"]); + Assert.AreEqual(1, exitCode); + Assert.AreEqual(0, _fakePointer.PenCalls.Count); + Assert.IsTrue(_fakeUia.WindowRectCalls.Count >= 1); + Assert.AreEqual(0, _fakeForeground.Calls.Count); + } +} diff --git a/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Touch.cs b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Touch.cs new file mode 100644 index 00000000..4ca11a8e --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Touch.cs @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using WinApp.Cli.Commands; +using WinApp.Cli.Helpers; +using WinApp.Cli.Models; + +namespace WinApp.Cli.Tests; + +public partial class UiCommandTests +{ + // --------------------------------------------------------------------- + // touch — synthetic touch gestures (tap/swipe/pinch/…) at a selector + // center or explicit app x,y coordinates. Exercised via FakePointerInput + // (records contacts), FakeForegroundGuard (proceeds) and + // FakeUiAutomationService (supplies the bounds rect via TryGetWindowRect). + // --------------------------------------------------------------------- + + [TestMethod] + public async Task Touch_Tap_SelectorCenter_InjectsTouch() + { + // Element center = (100 + 40/2, 100 + 20/2) = (120, 110). Window handle non-zero so the + // command has a real target to bounds-check and foreground-verify against. + _fakeUia.FindSingleResult = new UiElement + { + Id = "e0", Type = "Button", Selector = "btn-ok-1234", + X = 100, Y = 100, Width = 40, Height = 20, WindowHandle = 4242 + }; + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["btn-ok-1234", "-a", "TestApp", "--json"]); + Assert.AreEqual(0, exitCode); + + Assert.AreEqual(1, _fakePointer.TouchCalls.Count); + var call = _fakePointer.TouchCalls[0]; + Assert.AreEqual(TouchGesture.Tap, call.Gesture); + Assert.AreEqual(1, call.ContactPaths.Count); + Assert.AreEqual(new PointerPoint(120, 110), call.ContactPaths[0][0]); + + // Bounds-check + foreground gate both ran against the resolved window handle. + Assert.IsTrue(_fakeUia.WindowRectCalls.Count >= 1); + Assert.IsTrue(_fakeForeground.Calls.Count >= 1); + + var result = System.Text.Json.JsonSerializer.Deserialize(TestAnsiConsole.Output); + Assert.AreEqual("tap", result.GetProperty("gesture").GetString()); + Assert.AreEqual(4242, result.GetProperty("hwnd").GetInt64()); + } + + [TestMethod] + public async Task Touch_Swipe_ExplicitPoints_InjectsGlidePath() + { + _fakeSession.SessionResult.WindowHandle = 5150; + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, + ["-a", "TestApp", "--gesture", "swipe", "--at", "100,100", "--to-point", "300,120", "--json"]); + Assert.AreEqual(0, exitCode); + + Assert.AreEqual(1, _fakePointer.TouchCalls.Count); + var call = _fakePointer.TouchCalls[0]; + Assert.AreEqual(TouchGesture.Swipe, call.Gesture); + Assert.AreEqual(new PointerPoint(100, 100), call.ContactPaths[0][0]); + Assert.AreEqual(new PointerPoint(300, 120), call.ContactPaths[0][^1]); + } + + [TestMethod] + public async Task Touch_InvalidGesture_Rejected_NoInjection() + { + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, + ["-a", "TestApp", "--gesture", "frobnicate", "--at", "100,100", "--json"]); + Assert.AreEqual(1, exitCode); + Assert.AreEqual(0, _fakePointer.TouchCalls.Count); + } + + [TestMethod] + public async Task Touch_FingersAboveMax_Rejected_NoInjection() + { + // 11 > MaxContacts (10) must be rejected up front, before planning/injecting. + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, + ["-a", "TestApp", "--at", "100,100", "--fingers", "11", "--json"]); + Assert.AreEqual(1, exitCode); + Assert.AreEqual(0, _fakePointer.TouchCalls.Count); + // Rejected before any window resolution. + Assert.AreEqual(0, _fakeUia.WindowRectCalls.Count); + } + + [TestMethod] + public async Task Touch_ExplicitPointOutsideWindow_Rejected_NoInjection() + { + _fakeSession.SessionResult.WindowHandle = 7000; + _fakeUia.WindowRect = new PointerRect(0, 0, 800, 600); + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, + ["-a", "TestApp", "--at", "5000,5000", "--json"]); + Assert.AreEqual(1, exitCode); + Assert.AreEqual(0, _fakePointer.TouchCalls.Count); + // The window rect WAS consulted (bounds check ran) but the foreground gate never fired. + Assert.IsTrue(_fakeUia.WindowRectCalls.Count >= 1); + Assert.AreEqual(0, _fakeForeground.Calls.Count); + } + + [TestMethod] + public async Task Touch_ZeroTargetHwnd_Rejected_NoInjection() + { + // Session has no window handle (0) and an explicit --at (no element resolves a handle), + // so there is no verifiable target — the command must refuse to inject. + _fakeSession.SessionResult.WindowHandle = 0; + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, + ["-a", "TestApp", "--at", "100,100", "--json"]); + Assert.AreEqual(1, exitCode); + Assert.AreEqual(0, _fakePointer.TouchCalls.Count); + // Refused before the rect lookup / foreground gate. + Assert.AreEqual(0, _fakeUia.WindowRectCalls.Count); + Assert.AreEqual(0, _fakeForeground.Calls.Count); + } + + [TestMethod] + public async Task Touch_WindowRectUnreadable_Rejected_NoInjection() + { + // A nonzero handle resolves, but the window rect can't be read → no verifiable + // target, so the command must refuse (no_target) before the foreground gate. + _fakeSession.SessionResult.WindowHandle = 7100; + _fakeUia.WindowRectAllow = false; + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, + ["-a", "TestApp", "--at", "100,100", "--json"]); + Assert.AreEqual(1, exitCode); + Assert.AreEqual(0, _fakePointer.TouchCalls.Count); + // The rect lookup WAS attempted, but injection/foreground never happened. + Assert.IsTrue(_fakeUia.WindowRectCalls.Count >= 1); + Assert.AreEqual(0, _fakeForeground.Calls.Count); + } +} diff --git a/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.cs b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.cs index 91982263..075671b3 100644 --- a/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.cs +++ b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.cs @@ -16,6 +16,7 @@ public partial class UiCommandTests : BaseCommandTests private FakeMouseInput _fakeMouse = null!; private FakeKeyboardInput _fakeKeyboard = null!; private FakeForegroundGuard _fakeForeground = null!; + private FakePointerInput _fakePointer = null!; protected override IServiceCollection ConfigureServices(IServiceCollection services) { @@ -24,12 +25,14 @@ protected override IServiceCollection ConfigureServices(IServiceCollection servi _fakeMouse = new FakeMouseInput(); _fakeKeyboard = new FakeKeyboardInput(); _fakeForeground = new FakeForegroundGuard(); + _fakePointer = new FakePointerInput(); return services .AddSingleton(_fakeUia) .AddSingleton(_fakeSession) .AddSingleton(_fakeMouse) .AddSingleton(_fakeKeyboard) - .AddSingleton(_fakeForeground); + .AddSingleton(_fakeForeground) + .AddSingleton(_fakePointer); } [TestMethod] diff --git a/src/winapp-CLI/WinApp.Cli.Tests/UiSessionServiceTests.cs b/src/winapp-CLI/WinApp.Cli.Tests/UiSessionServiceTests.cs index 01402e5a..fef40a4b 100644 --- a/src/winapp-CLI/WinApp.Cli.Tests/UiSessionServiceTests.cs +++ b/src/winapp-CLI/WinApp.Cli.Tests/UiSessionServiceTests.cs @@ -66,6 +66,7 @@ private sealed class StubUiAutomation : IUiAutomationService { public List<(nint Hwnd, int Pid, string Title)> FindWindowsByTitle(string titleQuery) => []; public List<(nint Hwnd, int Pid, string Title)> FindWindowsByPid(int pid) => []; + public bool TryGetWindowRect(long hwnd, out WinApp.Cli.Helpers.PointerRect rect) { rect = default; return false; } public Task InspectAsync(UiSessionInfo session, string? elementId, int depth, CancellationToken ct) => Task.FromResult([]); public Task InspectAncestorsAsync(UiSessionInfo session, string elementId, CancellationToken ct) => Task.FromResult([]); diff --git a/src/winapp-CLI/WinApp.Cli/Commands/UiCommand.cs b/src/winapp-CLI/WinApp.Cli/Commands/UiCommand.cs index 57bdaae9..8585919b 100644 --- a/src/winapp-CLI/WinApp.Cli/Commands/UiCommand.cs +++ b/src/winapp-CLI/WinApp.Cli/Commands/UiCommand.cs @@ -19,6 +19,8 @@ public UiCommand( UiInvokeCommand invokeCommand, UiClickCommand clickCommand, UiDragCommand dragCommand, + UiTouchCommand touchCommand, + UiPenCommand penCommand, UiHoverCommand hoverCommand, UiSendKeysCommand sendKeysCommand, UiSetValueCommand setValueCommand, @@ -40,6 +42,8 @@ public UiCommand( Subcommands.Add(invokeCommand); Subcommands.Add(clickCommand); Subcommands.Add(dragCommand); + Subcommands.Add(touchCommand); + Subcommands.Add(penCommand); Subcommands.Add(hoverCommand); Subcommands.Add(sendKeysCommand); Subcommands.Add(setValueCommand); diff --git a/src/winapp-CLI/WinApp.Cli/Commands/UiPenCommand.cs b/src/winapp-CLI/WinApp.Cli/Commands/UiPenCommand.cs new file mode 100644 index 00000000..6a5cb798 --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli/Commands/UiPenCommand.cs @@ -0,0 +1,291 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using System.CommandLine; +using System.CommandLine.Invocation; +using System.CommandLine.Parsing; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Spectre.Console; +using WinApp.Cli.Helpers; +using WinApp.Cli.Models; +using WinApp.Cli.Services; + +namespace WinApp.Cli.Commands; + +internal class UiPenCommand : Command, IShortDescription +{ + public string ShortDescription => "Inject synthetic pen/stylus input (taps and ink strokes with pressure and tilt)"; + + public static Option AtOption { get; } = new("--at") + { + Description = "Pen contact point as app coordinates x,y (as reported by 'ui inspect'). " + + "Defaults to the selector's element center. Ignored when --path is given." + }; + + public static Option PathOption { get; } = new("--path") + { + Description = "Ink stroke path as a whitespace-separated list of x,y pairs, e.g. \"10,10 20,30 40,50\"." + }; + + public static Option PressureOption { get; } = new("--pressure") + { + Description = "Pen pressure from 0.0 to 1.0 (default: 0.5).", + DefaultValueFactory = _ => 0.5f + }; + + public static Option TiltXOption { get; } = new("--tilt-x") + { + Description = "Pen tilt along the x-axis in degrees (-90 to 90, default: 0).", + DefaultValueFactory = _ => 0 + }; + + public static Option TiltYOption { get; } = new("--tilt-y") + { + Description = "Pen tilt along the y-axis in degrees (-90 to 90, default: 0).", + DefaultValueFactory = _ => 0 + }; + + public static Option EraserOption { get; } = new("--eraser") + { + Description = "Use the eraser end of the pen instead of the tip." + }; + + public UiPenCommand() + : base("pen", "Inject synthetic pen/stylus input using the Windows synthetic-pointer API. " + + "Taps or draws ink strokes with configurable pressure, tilt and eraser mode, at an element's " + + "center or explicit app x,y coordinates. Requires an unlocked, interactive desktop with the " + + "target window foregroundable (Windows 10 1809+).") + { + Arguments.Add(SharedUiOptions.SelectorArgument); + Options.Add(SharedUiOptions.AppOption); + Options.Add(SharedUiOptions.WindowOption); + Options.Add(AtOption); + Options.Add(PathOption); + Options.Add(PressureOption); + Options.Add(TiltXOption); + Options.Add(TiltYOption); + Options.Add(EraserOption); + Options.Add(WinAppRootCommand.JsonOption); + } + + public class Handler( + IUiSessionService sessionService, + IUiAutomationService uiAutomation, + ISelectorService selectorService, + IPointerInput pointerInput, + IForegroundGuard foregroundGuard, + IAnsiConsole ansiConsole, + ILogger logger) : AsynchronousCommandLineAction + { + public override async Task InvokeAsync(ParseResult parseResult, CancellationToken cancellationToken = default) + { + var json = parseResult.GetValue(WinAppRootCommand.JsonOption); + var selectorStr = parseResult.GetValue(SharedUiOptions.SelectorArgument); + var app = parseResult.GetValue(SharedUiOptions.AppOption); + var window = parseResult.GetValue(SharedUiOptions.WindowOption); + var atStr = parseResult.GetValue(AtOption); + var pathStr = parseResult.GetValue(PathOption); + var pressure = parseResult.GetValue(PressureOption); + var tiltX = parseResult.GetValue(TiltXOption); + var tiltY = parseResult.GetValue(TiltYOption); + var eraser = parseResult.GetValue(EraserOption); + + if (string.IsNullOrWhiteSpace(app) && window is null) + { + UiErrors.MissingApp(logger, json); + return 1; + } + + if (pressure < 0f || pressure > 1f) + { + logger.LogError("{Symbol} --pressure must be between 0.0 and 1.0.", UiSymbols.Error); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, "--pressure must be between 0.0 and 1.0."); + return 1; + } + + if (tiltX < -90 || tiltX > 90 || tiltY < -90 || tiltY > 90) + { + logger.LogError("{Symbol} --tilt-x and --tilt-y must be between -90 and 90 degrees.", UiSymbols.Error); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, "--tilt-x and --tilt-y must be between -90 and 90 degrees."); + return 1; + } + + List? path = null; + if (!string.IsNullOrWhiteSpace(pathStr)) + { + if (!PointerGesturePlanner.TryParsePath(pathStr, out path)) + { + logger.LogError("{Symbol} --path must be whitespace-separated x,y pairs (e.g. \"10,10 20,30\"). Got '{Path}'.", UiSymbols.Error, pathStr); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, $"--path must be whitespace-separated x,y pairs (e.g. \"10,10 20,30\"). Got '{pathStr}'.", pathStr); + return 1; + } + } + + PointerPoint? at = null; + if (!string.IsNullOrWhiteSpace(atStr)) + { + if (!PointerGesturePlanner.TryParsePoint(atStr, out var atPoint)) + { + logger.LogError("{Symbol} --at must be a valid x,y pair (e.g. 100,200). Got '{At}'.", UiSymbols.Error, atStr); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, $"--at must be a valid x,y pair (e.g. 100,200). Got '{atStr}'.", atStr); + return 1; + } + at = atPoint; + } + + if (path is null && at is null && string.IsNullOrWhiteSpace(selectorStr)) + { + logger.LogError("{Symbol} Provide a target: a selector, --at x,y, or --path.", UiSymbols.Error); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, "Provide a target: a selector, --at x,y, or --path."); + return 1; + } + + try + { + var session = await sessionService.ResolveSessionAsync(app, window, cancellationToken); + + long targetHwnd = session.WindowHandle; + var targetLabel = pathStr ?? selectorStr ?? atStr; + + // Build the ink path: explicit --path wins; else --at; else the selector's center. + if (path is null) + { + PointerPoint contact; + if (at is not null) + { + contact = at.Value; + } + else + { + var selector = selectorService.Parse(selectorStr!); + var element = await uiAutomation.FindSingleElementAsync(session, selector, cancellationToken); + if (element is null) + { + UiErrors.ElementNotFound(logger, selectorStr!, json); + return 1; + } + + if (element.Width == 0 || element.Height == 0) + { + logger.LogError("{Symbol} Element has zero size — cannot use its center as a pen point.", UiSymbols.Error); + UiJsonError.Emit(json, UiJsonError.CodeZeroSize, "Element has zero size — cannot use its center as a pen point.", selectorStr); + return 1; + } + + targetHwnd = element.WindowHandle ?? session.WindowHandle; + + if (targetHwnd != 0) + { + Windows.Win32.PInvoke.SetForegroundWindow(new Windows.Win32.Foundation.HWND((nint)targetHwnd)); + await Task.Delay(100, cancellationToken); + } + + var stable = await GestureTargeting.ResolveStableAsync( + uiAutomation, session, selector, element, + GestureTargeting.DefaultMaxReads, GestureTargeting.DefaultReadDelayMs, null, cancellationToken); + if (!GestureTargeting.TryReport(stable, logger, json, selectorStr!, "pen")) + { + return 1; + } + contact = new PointerPoint(stable.CenterX, stable.CenterY); + } + + path = [contact]; + } + + if (targetHwnd != 0) + { + Windows.Win32.PInvoke.SetForegroundWindow(new Windows.Win32.Foundation.HWND((nint)targetHwnd)); + await Task.Delay(100, cancellationToken); + } + + // Refuse to inject without a resolved target window (F1): with hwnd 0 the foreground + // guard cannot verify the injection lands on the intended window, and the OS-wide pen + // input would hit whatever is foreground. Fail closed instead. + if (targetHwnd == 0) + { + logger.LogError("{Symbol} No target window could be resolved — refusing to inject pen input (it could hit the wrong window).", UiSymbols.Error); + UiJsonError.Emit(json, UiJsonError.CodeNoTarget, + "No target window could be resolved — refusing to inject pen input. Target an app window (via --app/--window) whose element resolves to a window handle."); + return 1; + } + + // Resolve the target window rect to bounds-check every ink point before injecting. + if (!uiAutomation.TryGetWindowRect(targetHwnd, out var windowRect)) + { + logger.LogError("{Symbol} Could not read the target window rectangle — refusing to inject pen input.", UiSymbols.Error); + UiJsonError.Emit(json, UiJsonError.CodeNoTarget, + "Could not read the target window rectangle — refusing to inject pen input."); + return 1; + } + + // Every ink point (selector center, explicit --at, or --path waypoint) must fall inside + // the target window — reject out-of-bounds coordinates and inject nothing. + var outOfBounds = PointerGesturePlanner.FirstOutOfBounds(windowRect, path); + if (outOfBounds is not null) + { + logger.LogError( + "{Symbol} Point ({X}, {Y}) is outside the target window ({Left},{Top})-({Right},{Bottom}) — refusing to inject pen input.", + UiSymbols.Error, outOfBounds.Value.X, outOfBounds.Value.Y, + windowRect.Left, windowRect.Top, windowRect.Right, windowRect.Bottom); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, + $"Point ({outOfBounds.Value.X},{outOfBounds.Value.Y}) is outside the target window " + + $"({windowRect.Left},{windowRect.Top})-({windowRect.Right},{windowRect.Bottom}) — no input injected."); + return 1; + } + + // Final foreground gate before the OS-wide injection (matches click/drag/hover). + if (!foregroundGuard.TryEnsureForeground(targetHwnd, logger, json, "pen")) + { + return 1; + } + + pointerInput.Pen(path, pressure, tiltX, tiltY, eraser); + + var action = eraser ? "erase" : (path.Count > 1 ? "draw" : "tap"); + + if (json) + { + var result = new UiPenResult + { + Action = action, + Target = targetLabel, + Points = path.Select(p => new UiPointResult { X = p.X, Y = p.Y }).ToArray(), + Pressure = pressure, + TiltX = tiltX, + TiltY = tiltY, + Eraser = eraser, + Hwnd = targetHwnd + }; + ansiConsole.Profile.Out.Writer.WriteLine( + JsonSerializer.Serialize(result, UiJsonContext.Default.UiPenResult)); + } + else + { + logger.LogInformation("{Symbol} pen {Action} with {Count} point(s), pressure {Pressure:0.00}", + UiSymbols.Check, action, path.Count, pressure); + } + + return 0; + } + catch (System.Runtime.InteropServices.COMException comEx) + { + logger.LogDebug("COM error: {HResult} {StackTrace}", comEx.HResult, comEx.StackTrace); + UiErrors.StaleElement(logger, json); + return 1; + } + catch (InvalidOperationException injectEx) + { + logger.LogError("{Symbol} {Message}", UiSymbols.Error, injectEx.Message); + UiJsonError.Emit(json, UiJsonError.CodeInjectionUnsupported, injectEx.Message); + return 1; + } + catch (Exception ex) + { + UiErrors.GenericError(logger, ex, json); + return 1; + } + } + } +} diff --git a/src/winapp-CLI/WinApp.Cli/Commands/UiTouchCommand.cs b/src/winapp-CLI/WinApp.Cli/Commands/UiTouchCommand.cs new file mode 100644 index 00000000..0a23d9e8 --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli/Commands/UiTouchCommand.cs @@ -0,0 +1,333 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using System.CommandLine; +using System.CommandLine.Invocation; +using System.CommandLine.Parsing; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Spectre.Console; +using WinApp.Cli.Helpers; +using WinApp.Cli.Models; +using WinApp.Cli.Services; + +namespace WinApp.Cli.Commands; + +internal class UiTouchCommand : Command, IShortDescription +{ + public string ShortDescription => "Inject synthetic touch gestures (tap, swipe, pinch, stretch, long-press)"; + + private static readonly Dictionary Gestures = new(StringComparer.OrdinalIgnoreCase) + { + ["tap"] = TouchGesture.Tap, + ["double-tap"] = TouchGesture.DoubleTap, + ["long-press"] = TouchGesture.LongPress, + ["swipe"] = TouchGesture.Swipe, + ["pinch"] = TouchGesture.Pinch, + ["stretch"] = TouchGesture.Stretch, + }; + + public static Option GestureOption { get; } = new("--gesture", "-g") + { + Description = "Gesture to perform: tap, double-tap, long-press, swipe, pinch, stretch (default: tap).", + DefaultValueFactory = _ => "tap" + }; + + public static Option AtOption { get; } = new("--at") + { + Description = "Explicit start point as app coordinates x,y (as reported by 'ui inspect'). " + + "Defaults to the selector's element center." + }; + + public static Option ToPointOption { get; } = new("--to-point") + { + Description = "End point x,y for a swipe (app coordinates)." + }; + + public static Option DistanceOption { get; } = new("--distance") + { + Description = "Distance in pixels for pinch/stretch (finger spread) or a directionless swipe.", + DefaultValueFactory = _ => 0 + }; + + public static Option HoldOption { get; } = new("--hold-ms") + { + Description = "Milliseconds to hold contacts down before lifting (long-press hold time).", + DefaultValueFactory = _ => 0 + }; + + public static Option DurationOption { get; } = new("--duration-ms") + { + Description = "Glide time in milliseconds for moving gestures (swipe/pinch/stretch).", + DefaultValueFactory = _ => 300 + }; + + public static Option FingersOption { get; } = new("--fingers") + { + Description = "Number of touch contacts (default: 1). Pinch/stretch always use 2.", + DefaultValueFactory = _ => 1 + }; + + public UiTouchCommand() + : base("touch", "Inject synthetic touch input using the Windows touch-injection API. " + + "Supports tap, double-tap, long-press, swipe, pinch and stretch gestures at an element's " + + "center or explicit app x,y coordinates. Requires an unlocked, interactive desktop with the " + + "target window foregroundable.") + { + Arguments.Add(SharedUiOptions.SelectorArgument); + Options.Add(SharedUiOptions.AppOption); + Options.Add(SharedUiOptions.WindowOption); + Options.Add(GestureOption); + Options.Add(AtOption); + Options.Add(ToPointOption); + Options.Add(DistanceOption); + Options.Add(HoldOption); + Options.Add(DurationOption); + Options.Add(FingersOption); + Options.Add(WinAppRootCommand.JsonOption); + } + + public class Handler( + IUiSessionService sessionService, + IUiAutomationService uiAutomation, + ISelectorService selectorService, + IPointerInput pointerInput, + IForegroundGuard foregroundGuard, + IAnsiConsole ansiConsole, + ILogger logger) : AsynchronousCommandLineAction + { + public override async Task InvokeAsync(ParseResult parseResult, CancellationToken cancellationToken = default) + { + var json = parseResult.GetValue(WinAppRootCommand.JsonOption); + var selectorStr = parseResult.GetValue(SharedUiOptions.SelectorArgument); + var app = parseResult.GetValue(SharedUiOptions.AppOption); + var window = parseResult.GetValue(SharedUiOptions.WindowOption); + var gestureStr = parseResult.GetValue(GestureOption) ?? "tap"; + var atStr = parseResult.GetValue(AtOption); + var toStr = parseResult.GetValue(ToPointOption); + var distance = parseResult.GetValue(DistanceOption); + var holdMs = parseResult.GetValue(HoldOption); + var durationMs = parseResult.GetValue(DurationOption); + var fingers = parseResult.GetValue(FingersOption); + + if (string.IsNullOrWhiteSpace(app) && window is null) + { + UiErrors.MissingApp(logger, json); + return 1; + } + + if (!Gestures.TryGetValue(gestureStr, out var gesture)) + { + logger.LogError("{Symbol} Unknown gesture '{Gesture}'. Allowed: tap, double-tap, long-press, swipe, pinch, stretch.", + UiSymbols.Error, gestureStr); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, + $"Unknown gesture '{gestureStr}'. Allowed: tap, double-tap, long-press, swipe, pinch, stretch."); + return 1; + } + + if (holdMs < 0 || durationMs < 0 || distance < 0 || fingers < 1) + { + logger.LogError("{Symbol} --hold-ms/--duration-ms/--distance must be zero or positive and --fingers >= 1.", UiSymbols.Error); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, + "--hold-ms/--duration-ms/--distance must be zero or positive and --fingers >= 1."); + return 1; + } + + if (fingers > PointerGesturePlanner.MaxContacts) + { + logger.LogError("{Symbol} --fingers must be between 1 and {Max} (the touch-injection contact limit).", + UiSymbols.Error, PointerGesturePlanner.MaxContacts); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, + $"--fingers must be between 1 and {PointerGesturePlanner.MaxContacts} (the touch-injection contact limit)."); + return 1; + } + + // Parse an explicit start point up front (mutually independent of selector resolution). + PointerPoint? at = null; + if (!string.IsNullOrWhiteSpace(atStr)) + { + if (!PointerGesturePlanner.TryParsePoint(atStr, out var atPoint)) + { + logger.LogError("{Symbol} --at must be a valid x,y pair (e.g. 100,200). Got '{At}'.", UiSymbols.Error, atStr); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, $"--at must be a valid x,y pair (e.g. 100,200). Got '{atStr}'.", atStr); + return 1; + } + at = atPoint; + } + + PointerPoint? to = null; + if (!string.IsNullOrWhiteSpace(toStr)) + { + if (!PointerGesturePlanner.TryParsePoint(toStr, out var toPoint)) + { + logger.LogError("{Symbol} --to-point must be a valid x,y pair (e.g. 300,400). Got '{To}'.", UiSymbols.Error, toStr); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, $"--to-point must be a valid x,y pair (e.g. 300,400). Got '{toStr}'.", toStr); + return 1; + } + to = toPoint; + } + + if (at is null && string.IsNullOrWhiteSpace(selectorStr)) + { + logger.LogError("{Symbol} Provide a target: a selector, or --at x,y.", UiSymbols.Error); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, "Provide a target: a selector, or --at x,y."); + return 1; + } + + if (gesture is TouchGesture.Pinch or TouchGesture.Stretch && distance <= 0) + { + logger.LogError("{Symbol} {Gesture} requires --distance (finger spread in pixels).", UiSymbols.Error, gestureStr); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, $"{gestureStr} requires --distance (finger spread in pixels)."); + return 1; + } + + if (gesture is TouchGesture.Swipe && to is null && distance <= 0) + { + logger.LogError("{Symbol} swipe requires --to-point x,y or --distance.", UiSymbols.Error); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, "swipe requires --to-point x,y or --distance."); + return 1; + } + + try + { + var session = await sessionService.ResolveSessionAsync(app, window, cancellationToken); + + long targetHwnd = session.WindowHandle; + PointerPoint start; + var targetLabel = selectorStr ?? atStr; + + if (at is not null) + { + start = at.Value; + } + else + { + // Resolve the selector's element center and re-resolve just before injection. + var selector = selectorService.Parse(selectorStr!); + var element = await uiAutomation.FindSingleElementAsync(session, selector, cancellationToken); + if (element is null) + { + UiErrors.ElementNotFound(logger, selectorStr!, json); + return 1; + } + + if (element.Width == 0 || element.Height == 0) + { + logger.LogError("{Symbol} Element has zero size — cannot use its center as a touch point.", UiSymbols.Error); + UiJsonError.Emit(json, UiJsonError.CodeZeroSize, "Element has zero size — cannot use its center as a touch point.", selectorStr); + return 1; + } + + targetHwnd = element.WindowHandle ?? session.WindowHandle; + + if (targetHwnd != 0) + { + Windows.Win32.PInvoke.SetForegroundWindow(new Windows.Win32.Foundation.HWND((nint)targetHwnd)); + await Task.Delay(100, cancellationToken); + } + + var stable = await GestureTargeting.ResolveStableAsync( + uiAutomation, session, selector, element, + GestureTargeting.DefaultMaxReads, GestureTargeting.DefaultReadDelayMs, null, cancellationToken); + if (!GestureTargeting.TryReport(stable, logger, json, selectorStr!, "touch")) + { + return 1; + } + start = new PointerPoint(stable.CenterX, stable.CenterY); + } + + if (at is not null && targetHwnd != 0) + { + Windows.Win32.PInvoke.SetForegroundWindow(new Windows.Win32.Foundation.HWND((nint)targetHwnd)); + await Task.Delay(100, cancellationToken); + } + + // Refuse to inject without a resolved target window (F1): with hwnd 0 the foreground + // guard cannot verify the injection lands on the intended window, and the OS-wide touch + // would hit whatever is foreground. Fail closed instead. + if (targetHwnd == 0) + { + logger.LogError("{Symbol} No target window could be resolved — refusing to inject touch (it could hit the wrong window).", UiSymbols.Error); + UiJsonError.Emit(json, UiJsonError.CodeNoTarget, + "No target window could be resolved — refusing to inject touch. Target an app window (via --app/--window) whose element resolves to a window handle."); + return 1; + } + + // Resolve the target window rect to bounds-check every coordinate before injecting. + if (!uiAutomation.TryGetWindowRect(targetHwnd, out var windowRect)) + { + logger.LogError("{Symbol} Could not read the target window rectangle — refusing to inject touch.", UiSymbols.Error); + UiJsonError.Emit(json, UiJsonError.CodeNoTarget, + "Could not read the target window rectangle — refusing to inject touch."); + return 1; + } + + var (contactPaths, points, effectiveFingers) = + PointerGesturePlanner.PlanTouch(gesture, start, to, distance, fingers); + + // Every planned point (selector center, explicit --at/--to-point, and generated + // waypoints) must fall inside the target window — reject out-of-bounds coordinates + // and inject nothing. + var outOfBounds = PointerGesturePlanner.FirstOutOfBounds(windowRect, points); + if (outOfBounds is not null) + { + logger.LogError( + "{Symbol} Point ({X}, {Y}) is outside the target window ({Left},{Top})-({Right},{Bottom}) — refusing to inject touch.", + UiSymbols.Error, outOfBounds.Value.X, outOfBounds.Value.Y, + windowRect.Left, windowRect.Top, windowRect.Right, windowRect.Bottom); + UiJsonError.Emit(json, UiJsonError.CodeInvalidArguments, + $"Point ({outOfBounds.Value.X},{outOfBounds.Value.Y}) is outside the target window " + + $"({windowRect.Left},{windowRect.Top})-({windowRect.Right},{windowRect.Bottom}) — no input injected."); + return 1; + } + + // Final foreground gate before the OS-wide injection (matches click/drag/hover). + if (!foregroundGuard.TryEnsureForeground(targetHwnd, logger, json, "touch")) + { + return 1; + } + + pointerInput.Touch(gesture, contactPaths, holdMs, durationMs); + + if (json) + { + var result = new UiTouchResult + { + Gesture = gestureStr.ToLowerInvariant(), + Target = targetLabel, + Points = points.Select(p => new UiPointResult { X = p.X, Y = p.Y }).ToArray(), + Fingers = effectiveFingers, + DurationMs = durationMs, + Hwnd = targetHwnd + }; + ansiConsole.Profile.Out.Writer.WriteLine( + JsonSerializer.Serialize(result, UiJsonContext.Default.UiTouchResult)); + } + else + { + logger.LogInformation("{Symbol} {Gesture} at ({X}, {Y}) with {Fingers} finger(s)", + UiSymbols.Check, gestureStr, start.X, start.Y, effectiveFingers); + } + + return 0; + } + catch (System.Runtime.InteropServices.COMException comEx) + { + logger.LogDebug("COM error: {HResult} {StackTrace}", comEx.HResult, comEx.StackTrace); + UiErrors.StaleElement(logger, json); + return 1; + } + catch (InvalidOperationException injectEx) + { + logger.LogError("{Symbol} {Message}", UiSymbols.Error, injectEx.Message); + UiJsonError.Emit(json, UiJsonError.CodeInjectionUnsupported, injectEx.Message); + return 1; + } + catch (Exception ex) + { + UiErrors.GenericError(logger, ex, json); + return 1; + } + } + } +} diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/CliSchema.cs b/src/winapp-CLI/WinApp.Cli/Helpers/CliSchema.cs index 7f83ff0a..b77ae4a2 100644 --- a/src/winapp-CLI/WinApp.Cli/Helpers/CliSchema.cs +++ b/src/winapp-CLI/WinApp.Cli/Helpers/CliSchema.cs @@ -22,6 +22,7 @@ namespace WinApp.Cli.Helpers; [JsonSerializable(typeof(Dictionary))] [JsonSerializable(typeof(IfExists))] [JsonSerializable(typeof(ManifestTemplates))] +[JsonSerializable(typeof(float))] [JsonSourceGenerationOptions( WriteIndented = true, NewLine = "\n", diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/HostBuilderExtensions.cs b/src/winapp-CLI/WinApp.Cli/Helpers/HostBuilderExtensions.cs index a55b1f2e..2171692e 100644 --- a/src/winapp-CLI/WinApp.Cli/Helpers/HostBuilderExtensions.cs +++ b/src/winapp-CLI/WinApp.Cli/Helpers/HostBuilderExtensions.cs @@ -52,6 +52,7 @@ public static IServiceCollection ConfigureServices(this IServiceCollection servi .AddSingleton() // UI Automation services .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() @@ -94,6 +95,8 @@ public static IServiceCollection ConfigureCommands(this IServiceCollection servi .UseCommandHandler() .UseCommandHandler() .UseCommandHandler() + .UseCommandHandler() + .UseCommandHandler() .UseCommandHandler() .UseCommandHandler() .UseCommandHandler() diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/IPointerInput.cs b/src/winapp-CLI/WinApp.Cli/Helpers/IPointerInput.cs new file mode 100644 index 00000000..d39848fe --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli/Helpers/IPointerInput.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +namespace WinApp.Cli.Helpers; + +/// A single point in app/screen pixel space (the same space ui inspect reports). +internal readonly record struct PointerPoint(int X, int Y); + +/// +/// A window rectangle in screen pixels (as returned by GetWindowRect). Used to bounds-check +/// explicit touch/pen coordinates so a gesture can never be injected outside the target window. +/// +internal readonly record struct PointerRect(int Left, int Top, int Right, int Bottom) +{ + /// Whether lies inside (inclusive) this rectangle. + public bool Contains(PointerPoint p) + => p.X >= Left && p.X <= Right && p.Y >= Top && p.Y <= Bottom; +} + +/// The synthetic touch gestures supported by winapp ui touch. +internal enum TouchGesture +{ + Tap, + DoubleTap, + LongPress, + Swipe, + Pinch, + Stretch, +} + +/// +/// Abstraction over synthetic pointer (touch / pen) injection for testability. The real +/// implementation drives InitializeTouchInjection/InjectTouchInput (touch) and +/// CreateSyntheticPointerDevice/InjectSyntheticPointerInput (pen); fakes record the +/// injected contacts and gesture parameters so the ui touch/ui pen commands can be +/// unit-tested without a live, unlocked desktop. +/// +internal interface IPointerInput +{ + /// + /// Injects a synthetic touch gesture. holds one ordered waypoint + /// path per finger (each path has at least the contact's start point; a two-point path glides from + /// start to end). The implementation presses all contacts down, interpolates between waypoints over + /// , then lifts them. + /// + /// Gesture kind — drives double-tap repetition and long-press semantics. + /// Milliseconds to hold the contacts down before lifting (long-press). + /// Glide time in milliseconds for moving gestures (swipe/pinch/stretch). + void Touch( + TouchGesture gesture, + IReadOnlyList> contactPaths, + int holdMs, + int durationMs); + + /// + /// Injects a synthetic pen action along (a single ink stroke; a one-point + /// path is a tap). is 0..1 (mapped to the 0..1024 pen range), + /// / are tilt angles in degrees, and + /// selects the eraser end of the pen. + /// + void Pen( + IReadOnlyList path, + float pressure, + int tiltX, + int tiltY, + bool eraser); +} diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/PointerGesturePlanner.cs b/src/winapp-CLI/WinApp.Cli/Helpers/PointerGesturePlanner.cs new file mode 100644 index 00000000..40a8add0 --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli/Helpers/PointerGesturePlanner.cs @@ -0,0 +1,185 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using System.Globalization; + +namespace WinApp.Cli.Helpers; + +/// +/// Pure geometry/parsing helpers shared by ui touch and ui pen. Parses the x,y +/// coordinate grammar (identical to ui drag's) and expands a high-level touch gesture into the +/// per-finger waypoint paths that replays. +/// +internal static class PointerGesturePlanner +{ + /// Contact spread (px) between extra fingers for multi-finger tap/swipe gestures. + private const int FingerSpacingPx = 24; + + /// Residual gap (px) each pinch finger keeps from the center at full contraction. + private const int PinchCenterGapPx = 4; + + /// + /// Maximum simultaneous touch contacts the pointer-injection subsystem supports + /// (MAX_TOUCH_COUNT / the count registered with InitializeTouchInjection). + /// ui touch --fingers is rejected above this. + /// + public const int MaxContacts = 10; + + /// + /// Returns the first point in that lies outside , + /// or when every point is inside. Used to reject touch/pen gestures whose + /// coordinates fall outside the target window before any OS-wide injection. + /// + public static PointerPoint? FirstOutOfBounds(PointerRect rect, IEnumerable points) + { + foreach (var p in points) + { + if (!rect.Contains(p)) + { + return p; + } + } + + return null; + } + + /// + /// Parses a single x,y integer pair (app coordinates as reported by ui inspect). + /// + public static bool TryParsePoint(string? value, out PointerPoint point) + { + point = default; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var parts = value.Split(',', StringSplitOptions.TrimEntries); + if (parts.Length != 2) + { + return false; + } + + if (int.TryParse(parts[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out int x) + && int.TryParse(parts[1], NumberStyles.Integer, CultureInfo.InvariantCulture, out int y)) + { + point = new PointerPoint(x, y); + return true; + } + + return false; + } + + /// + /// Whether a token that failed to parse as a point was nonetheless meant as coordinates (has a comma + /// and an integer first field). Mirrors ui drag's heuristic so malformed coordinates surface + /// a precise error instead of a misleading "element not found". + /// + public static bool LooksLikeCoordinates(string token) + { + var parts = token.Split(','); + return parts.Length >= 2 + && int.TryParse(parts[0].Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out _); + } + + /// + /// Parses a pen ink path — a whitespace-separated list of x,y pairs + /// ("x1,y1 x2,y2 ..."). Returns if any token is not a valid pair. + /// + public static bool TryParsePath(string? value, out List points) + { + points = []; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + foreach (var token in value.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (!TryParsePoint(token, out var p)) + { + return false; + } + points.Add(p); + } + + return points.Count > 0; + } + + /// + /// Expands a touch gesture into per-finger waypoint paths plus the flattened point list reported in + /// JSON. is the anchor (selector center or --at). For pinch/stretch + /// is coerced to at least 2. + /// + public static (List> ContactPaths, List Points, int Fingers) PlanTouch( + TouchGesture gesture, + PointerPoint start, + PointerPoint? end, + int distance, + int fingers) + { + var contactPaths = new List>(); + + switch (gesture) + { + case TouchGesture.Swipe: + { + var to = end ?? new PointerPoint(start.X + distance, start.Y); + int count = Math.Max(1, fingers); + for (int i = 0; i < count; i++) + { + int dy = i * FingerSpacingPx; + contactPaths.Add([new PointerPoint(start.X, start.Y + dy), new PointerPoint(to.X, to.Y + dy)]); + } + break; + } + + case TouchGesture.Pinch: + case TouchGesture.Stretch: + { + fingers = Math.Max(2, fingers); + int half = Math.Max(PinchCenterGapPx + 1, distance / 2); + + // Two opposing fingers along the x-axis. Pinch converges toward the center; stretch + // diverges away from it. + var leftApart = new PointerPoint(start.X - half, start.Y); + var rightApart = new PointerPoint(start.X + half, start.Y); + var leftNear = new PointerPoint(start.X - PinchCenterGapPx, start.Y); + var rightNear = new PointerPoint(start.X + PinchCenterGapPx, start.Y); + + if (gesture == TouchGesture.Pinch) + { + contactPaths.Add([leftApart, leftNear]); + contactPaths.Add([rightApart, rightNear]); + } + else + { + contactPaths.Add([leftNear, leftApart]); + contactPaths.Add([rightNear, rightApart]); + } + break; + } + + case TouchGesture.Tap: + case TouchGesture.DoubleTap: + case TouchGesture.LongPress: + default: + { + int count = Math.Max(1, fingers); + for (int i = 0; i < count; i++) + { + contactPaths.Add([new PointerPoint(start.X + i * FingerSpacingPx, start.Y)]); + } + break; + } + } + + var points = new List(); + foreach (var path in contactPaths) + { + points.AddRange(path); + } + + return (contactPaths, points, contactPaths.Count); + } +} diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/PointerInput.cs b/src/winapp-CLI/WinApp.Cli/Helpers/PointerInput.cs new file mode 100644 index 00000000..18c9de95 --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli/Helpers/PointerInput.cs @@ -0,0 +1,389 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using System.ComponentModel; +using System.Runtime.InteropServices; +using Windows.Win32; +using Windows.Win32.Foundation; +using Windows.Win32.UI.Input.Pointer; +using Windows.Win32.UI.WindowsAndMessaging; + +namespace WinApp.Cli.Helpers; + +/// +/// Injects synthetic touch and pen input using the Windows pointer-injection APIs. Touch prefers the +/// synthetic-pointer device (CreateSyntheticPointerDevice(PT_TOUCH)/ +/// InjectSyntheticPointerInput) — the same mechanism the pen path uses — and falls back to the +/// legacy InitializeTouchInjection/InjectTouchInput API when a synthetic touch device +/// cannot be created. Pen uses CreateSyntheticPointerDevice(PT_PEN). Coordinates are screen +/// pixels — the same space ui inspect reports. +/// +internal static class PointerInput +{ + /// Maximum simultaneous touch contacts we register with the injection subsystem. + private const uint MaxContacts = 10; + + /// Pen pressure range used by the pointer APIs (0..1024). + private const uint PenPressureMax = 1024; + + /// Steps used to interpolate a moving gesture between two waypoints. + private const int GlideSteps = 20; + + // touchFlags / touchMask / penFlags / penMask are raw DWORD bitmasks in the generated structs. + private const uint TOUCH_MASK_CONTACTAREA = 0x00000001; + private const uint PEN_FLAG_NONE = 0x00000000; + private const uint PEN_FLAG_ERASER = 0x00000004; + private const uint PEN_MASK_PRESSURE = 0x00000001; + private const uint PEN_MASK_TILT_X = 0x00000004; + private const uint PEN_MASK_TILT_Y = 0x00000008; + + private static readonly object InitLock = new(); + private static bool _touchInitialized; + + /// Delegate that submits one frame of touch contacts (synthetic device or legacy API). + private delegate void TouchSender(POINTER_TOUCH_INFO[] contacts); + + /// + /// Registers this process for legacy touch injection the first time it is needed. Idempotent — the + /// OS only allows a single successful InitializeTouchInjection per process. On failure the + /// actual Win32 error is surfaced so callers can tell "unsupported" from "locked desktop". + /// + private static void EnsureTouchInitialized() + { + if (_touchInitialized) + { + return; + } + + lock (InitLock) + { + if (_touchInitialized) + { + return; + } + + // TOUCH_FEEDBACK_NONE — suppress the OS touch-visual so automation stays invisible. + if (!PInvoke.InitializeTouchInjection(MaxContacts, TOUCH_FEEDBACK_MODE.TOUCH_FEEDBACK_NONE)) + { + int err = Marshal.GetLastPInvokeError(); + throw new InvalidOperationException( + $"InitializeTouchInjection failed (Win32 error {err}: {Win32Message(err)}) — touch " + + "injection is unsupported or unavailable on this desktop. This usually means the " + + "device/driver does not support injected touch, or the session is locked / on a secure desktop."); + } + + _touchInitialized = true; + } + } + + /// Formats a Win32 error code into its system message for honest diagnostics. + private static string Win32Message(int error) + { + try { return new Win32Exception(error).Message; } + catch { return "unknown error"; } + } + + public static void Touch( + TouchGesture gesture, + IReadOnlyList> contactPaths, + int holdMs, + int durationMs) + { + // Primary path: a synthetic touch pointer device, mirroring the working pen path. This is the + // modern, better-supported mechanism (Windows 10 1809+). + var device = PInvoke.CreateSyntheticPointerDevice( + POINTER_INPUT_TYPE.PT_TOUCH, MaxContacts, POINTER_FEEDBACK_MODE.POINTER_FEEDBACK_NONE); + + if (!device.IsNull) + { + try + { + RunTouchGesture(gesture, contactPaths, holdMs, durationMs, + contacts => SendSyntheticTouch(device, contacts)); + return; + } + finally + { + PInvoke.DestroySyntheticPointerDevice(device); + } + } + + int createErr = Marshal.GetLastPInvokeError(); + + // Fallback path: the legacy touch-injection API. EnsureTouchInitialized surfaces an honest + // Win32 error if even this is unsupported. + try + { + EnsureTouchInitialized(); + } + catch (InvalidOperationException ex) + { + throw new InvalidOperationException( + $"Synthetic touch injection is unsupported (CreateSyntheticPointerDevice(PT_TOUCH) failed, " + + $"Win32 error {createErr}: {Win32Message(createErr)}). {ex.Message}"); + } + + RunTouchGesture(gesture, contactPaths, holdMs, durationMs, SendLegacyTouch); + } + + private static void RunTouchGesture( + TouchGesture gesture, + IReadOnlyList> contactPaths, + int holdMs, + int durationMs, + TouchSender send) + { + int repeats = gesture == TouchGesture.DoubleTap ? 2 : 1; + for (int r = 0; r < repeats; r++) + { + InjectTouchStroke(contactPaths, holdMs, durationMs, send); + if (r + 1 < repeats) + { + Thread.Sleep(60); // inter-tap gap for double-tap + } + } + } + + private static void InjectTouchStroke( + IReadOnlyList> contactPaths, + int holdMs, + int durationMs, + TouchSender send) + { + int count = contactPaths.Count; + var contacts = new POINTER_TOUCH_INFO[count]; + + // --- Press down --- + for (int i = 0; i < count; i++) + { + var start = contactPaths[i][0]; + contacts[i] = MakeContact( + (uint)i, start.X, start.Y, + POINTER_FLAGS.POINTER_FLAG_DOWN | POINTER_FLAGS.POINTER_FLAG_INRANGE | POINTER_FLAGS.POINTER_FLAG_INCONTACT, + primary: i == 0); + } + send(contacts); + + try + { + if (holdMs > 0) + { + Thread.Sleep(holdMs); + } + + // --- Glide between waypoints (only if any contact has more than one waypoint) --- + int maxWaypoints = 0; + foreach (var path in contactPaths) + { + maxWaypoints = Math.Max(maxWaypoints, path.Count); + } + + if (maxWaypoints > 1) + { + int perStep = Math.Max(1, durationMs / GlideSteps); + for (int step = 1; step <= GlideSteps; step++) + { + double t = step / (double)GlideSteps; + for (int i = 0; i < count; i++) + { + var path = contactPaths[i]; + var (x, y) = Interpolate(path, t); + contacts[i] = MakeContact( + (uint)i, x, y, + POINTER_FLAGS.POINTER_FLAG_UPDATE | POINTER_FLAGS.POINTER_FLAG_INRANGE | POINTER_FLAGS.POINTER_FLAG_INCONTACT, + primary: i == 0); + } + send(contacts); + Thread.Sleep(perStep); + } + } + } + finally + { + // --- Lift (always, even if a glide frame threw, so contacts don't stay stuck down) --- + for (int i = 0; i < count; i++) + { + var last = contactPaths[i][^1]; + contacts[i] = MakeContact((uint)i, last.X, last.Y, POINTER_FLAGS.POINTER_FLAG_UP, primary: i == 0); + } + try { send(contacts); } + catch (InvalidOperationException) { } + } + } + + private static POINTER_TOUCH_INFO MakeContact(uint id, int x, int y, POINTER_FLAGS flags, bool primary) + { + if (primary) + { + flags |= POINTER_FLAGS.POINTER_FLAG_PRIMARY; + } + + return new POINTER_TOUCH_INFO + { + pointerInfo = new POINTER_INFO + { + pointerType = POINTER_INPUT_TYPE.PT_TOUCH, + pointerId = id, + pointerFlags = flags, + ptPixelLocation = new System.Drawing.Point(x, y), + }, + touchFlags = 0, + touchMask = TOUCH_MASK_CONTACTAREA, + rcContact = new RECT { left = x - 2, top = y - 2, right = x + 2, bottom = y + 2 }, + }; + } + + /// Submits one frame of touch contacts via the legacy InjectTouchInput API. + private static void SendLegacyTouch(POINTER_TOUCH_INFO[] contacts) + { + unsafe + { + fixed (POINTER_TOUCH_INFO* p = contacts) + { + if (!PInvoke.InjectTouchInput((uint)contacts.Length, p)) + { + int err = Marshal.GetLastPInvokeError(); + throw new InvalidOperationException( + $"InjectTouchInput failed (Win32 error {err}: {Win32Message(err)}) — touch injection " + + "failed or is unsupported on this desktop. The target may be elevated (run this CLI " + + "as administrator), the desktop may be locked, or injected touch may not be supported here."); + } + } + } + } + + /// + /// Submits one frame of touch contacts via the synthetic-pointer device + /// (InjectSyntheticPointerInput) — the modern path shared with pen injection. + /// + private static void SendSyntheticTouch(HSYNTHETICPOINTERDEVICE device, POINTER_TOUCH_INFO[] contacts) + { + var infos = new POINTER_TYPE_INFO[contacts.Length]; + for (int i = 0; i < contacts.Length; i++) + { + infos[i] = new POINTER_TYPE_INFO { type = POINTER_INPUT_TYPE.PT_TOUCH }; + infos[i].Anonymous.touchInfo = contacts[i]; + } + + unsafe + { + fixed (POINTER_TYPE_INFO* p = infos) + { + if (!PInvoke.InjectSyntheticPointerInput(device, p, (uint)infos.Length)) + { + int err = Marshal.GetLastPInvokeError(); + throw new InvalidOperationException( + $"InjectSyntheticPointerInput (touch) failed (Win32 error {err}: {Win32Message(err)}) — " + + "touch injection failed on this desktop. The target may be elevated (run this CLI as " + + "administrator), or the desktop may be locked."); + } + } + } + } + + public static void Pen( + IReadOnlyList path, + float pressure, + int tiltX, + int tiltY, + bool eraser) + { + var device = PInvoke.CreateSyntheticPointerDevice(POINTER_INPUT_TYPE.PT_PEN, 1, POINTER_FEEDBACK_MODE.POINTER_FEEDBACK_NONE); + if (device.IsNull) + { + int err = Marshal.GetLastPInvokeError(); + throw new InvalidOperationException( + $"CreateSyntheticPointerDevice(PT_PEN) failed (Win32 error {err}: {Win32Message(err)}) — " + + "synthetic pen injection is unavailable on this desktop (requires Windows 10 1809+ and an " + + "unlocked interactive session)."); + } + + try + { + uint mappedPressure = (uint)Math.Clamp((int)Math.Round(pressure * PenPressureMax), 0, (int)PenPressureMax); + if (mappedPressure == 0) + { + mappedPressure = 1; // in-contact frames need non-zero pressure + } + + // Down at the first point. + var first = path[0]; + SendPen(device, first.X, first.Y, mappedPressure, tiltX, tiltY, eraser, + POINTER_FLAGS.POINTER_FLAG_DOWN | POINTER_FLAGS.POINTER_FLAG_INRANGE | POINTER_FLAGS.POINTER_FLAG_INCONTACT); + + try + { + // Glide through the remaining ink points. + for (int i = 1; i < path.Count; i++) + { + var pt = path[i]; + SendPen(device, pt.X, pt.Y, mappedPressure, tiltX, tiltY, eraser, + POINTER_FLAGS.POINTER_FLAG_UPDATE | POINTER_FLAGS.POINTER_FLAG_INRANGE | POINTER_FLAGS.POINTER_FLAG_INCONTACT); + Thread.Sleep(10); + } + } + finally + { + var last = path[^1]; + try { SendPen(device, last.X, last.Y, 0, tiltX, tiltY, eraser, POINTER_FLAGS.POINTER_FLAG_UP); } + catch (InvalidOperationException) { } + } + } + finally + { + PInvoke.DestroySyntheticPointerDevice(device); + } + } + + private static void SendPen( + HSYNTHETICPOINTERDEVICE device, int x, int y, uint pressure, int tiltX, int tiltY, bool eraser, POINTER_FLAGS flags) + { + var penFlags = eraser ? PEN_FLAG_ERASER : PEN_FLAG_NONE; + + var info = new POINTER_TYPE_INFO + { + type = POINTER_INPUT_TYPE.PT_PEN, + }; + info.Anonymous.penInfo = new POINTER_PEN_INFO + { + pointerInfo = new POINTER_INFO + { + pointerType = POINTER_INPUT_TYPE.PT_PEN, + pointerId = 1, + pointerFlags = flags, + ptPixelLocation = new System.Drawing.Point(x, y), + }, + penFlags = penFlags, + penMask = PEN_MASK_PRESSURE | PEN_MASK_TILT_X | PEN_MASK_TILT_Y, + pressure = pressure, + tiltX = tiltX, + tiltY = tiltY, + }; + + unsafe + { + if (!PInvoke.InjectSyntheticPointerInput(device, &info, 1)) + { + int err = Marshal.GetLastPInvokeError(); + throw new InvalidOperationException( + $"InjectSyntheticPointerInput (pen) failed (Win32 error {err}: {Win32Message(err)}) — the " + + "target may be elevated (run this CLI as administrator) or the desktop is locked."); + } + } + } + + private static (int X, int Y) Interpolate(IReadOnlyList path, double t) + { + if (path.Count == 1) + { + return (path[0].X, path[0].Y); + } + + // Treat the path as a single straight segment from first to last waypoint. + var a = path[0]; + var b = path[^1]; + int x = a.X + (int)Math.Round((b.X - a.X) * t); + int y = a.Y + (int)Math.Round((b.Y - a.Y) * t); + return (x, y); + } +} diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/RealPointerInput.cs b/src/winapp-CLI/WinApp.Cli/Helpers/RealPointerInput.cs new file mode 100644 index 00000000..18fc44ed --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli/Helpers/RealPointerInput.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +namespace WinApp.Cli.Helpers; + +/// +/// Production implementation — delegates to the static P/Invoke helpers +/// that drive the Windows touch- and pen-injection APIs. +/// +internal class RealPointerInput : IPointerInput +{ + public void Touch(TouchGesture gesture, IReadOnlyList> contactPaths, int holdMs, int durationMs) + => PointerInput.Touch(gesture, contactPaths, holdMs, durationMs); + + public void Pen(IReadOnlyList path, float pressure, int tiltX, int tiltY, bool eraser) + => PointerInput.Pen(path, pressure, tiltX, tiltY, eraser); +} diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonContext.cs b/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonContext.cs index 9abee25c..d423f379 100644 --- a/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonContext.cs +++ b/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonContext.cs @@ -33,6 +33,10 @@ namespace WinApp.Cli.Helpers; [JsonSerializable(typeof(UiHoverResult))] [JsonSerializable(typeof(UiSendKeysResult))] [JsonSerializable(typeof(UiDragResult))] +[JsonSerializable(typeof(UiTouchResult))] +[JsonSerializable(typeof(UiPenResult))] +[JsonSerializable(typeof(UiPointResult))] +[JsonSerializable(typeof(UiPointResult[]))] [JsonSerializable(typeof(UiErrorResult))] [JsonSerializable(typeof(UiErrorInfo))] [JsonSerializable(typeof(UiFocusedResult))] @@ -249,3 +253,34 @@ internal sealed class UiDragResult public int DwellMs { get; set; } public long Hwnd { get; set; } } + +/// A point in app/screen pixel space for touch/pen JSON output. +internal sealed class UiPointResult +{ + public int X { get; set; } + public int Y { get; set; } +} + +internal sealed class UiTouchResult +{ + public string Gesture { get; set; } = ""; + /// The selector or app x,y the gesture targeted. + public string? Target { get; set; } + public UiPointResult[] Points { get; set; } = []; + public int Fingers { get; set; } + public int DurationMs { get; set; } + public long Hwnd { get; set; } +} + +internal sealed class UiPenResult +{ + public string Action { get; set; } = ""; + /// The selector or app x,y the pen action targeted. + public string? Target { get; set; } + public UiPointResult[] Points { get; set; } = []; + public float Pressure { get; set; } + public int TiltX { get; set; } + public int TiltY { get; set; } + public bool Eraser { get; set; } + public long Hwnd { get; set; } +} diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonError.cs b/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonError.cs index e5bb9c01..a90188ff 100644 --- a/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonError.cs +++ b/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonError.cs @@ -23,6 +23,8 @@ internal static class UiJsonError public const string CodeForegroundNotTarget = "foreground_not_target"; public const string CodeNoInteractiveDesktop = "no_interactive_desktop"; public const string CodeTargetMoved = "target_moved"; + public const string CodeNoTarget = "no_target"; + public const string CodeInjectionUnsupported = "injection_unsupported"; /// Write a JSON error envelope to stderr. No-op when is false. public static void Emit(bool json, string code, string message, diff --git a/src/winapp-CLI/WinApp.Cli/NativeMethods.txt b/src/winapp-CLI/WinApp.Cli/NativeMethods.txt index 613e8e67..aefede79 100644 --- a/src/winapp-CLI/WinApp.Cli/NativeMethods.txt +++ b/src/winapp-CLI/WinApp.Cli/NativeMethods.txt @@ -93,6 +93,18 @@ WM_CHAR WM_SYSKEYDOWN WM_SYSKEYUP GetSystemMetrics +InitializeTouchInjection +InjectTouchInput +CreateSyntheticPointerDevice +DestroySyntheticPointerDevice +InjectSyntheticPointerInput +POINTER_TOUCH_INFO +POINTER_PEN_INFO +POINTER_TYPE_INFO +POINTER_FLAGS +POINTER_INPUT_TYPE +POINTER_FEEDBACK_MODE +TOUCH_FEEDBACK_MODE GetWindow GetClassName IUIAutomationTextPattern diff --git a/src/winapp-CLI/WinApp.Cli/Services/IUiAutomationService.cs b/src/winapp-CLI/WinApp.Cli/Services/IUiAutomationService.cs index ed2effe9..900c925b 100644 --- a/src/winapp-CLI/WinApp.Cli/Services/IUiAutomationService.cs +++ b/src/winapp-CLI/WinApp.Cli/Services/IUiAutomationService.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation and Contributors. All rights reserved. // Licensed under the MIT License. +using WinApp.Cli.Helpers; using WinApp.Cli.Models; namespace WinApp.Cli.Services; @@ -21,6 +22,15 @@ internal interface IUiAutomationService /// Find all top-level windows for a specific process ID. /// List<(nint Hwnd, int Pid, string Title)> FindWindowsByPid(int pid); + + /// + /// Resolve a top-level window's screen rectangle via GetWindowRect. Returns + /// (and a default rect) when the handle is 0, invalid, or unreadable — + /// callers must treat that as "no verifiable target window". Used to bounds-check + /// ui touch/ui pen coordinates before any OS-wide injection. + /// + bool TryGetWindowRect(long hwnd, out PointerRect rect); + Task InspectAsync(UiSessionInfo session, string? elementId, int depth, CancellationToken ct); Task InspectAncestorsAsync(UiSessionInfo session, string elementId, CancellationToken ct); diff --git a/src/winapp-CLI/WinApp.Cli/Services/UiAutomationService.cs b/src/winapp-CLI/WinApp.Cli/Services/UiAutomationService.cs index 25149865..8b66f5af 100644 --- a/src/winapp-CLI/WinApp.Cli/Services/UiAutomationService.cs +++ b/src/winapp-CLI/WinApp.Cli/Services/UiAutomationService.cs @@ -5,6 +5,7 @@ using System.Runtime.InteropServices.Marshalling; using Microsoft.Extensions.Logging; using Windows.Win32.UI.Accessibility; +using WinApp.Cli.Helpers; using WinApp.Cli.Models; namespace WinApp.Cli.Services; @@ -37,6 +38,38 @@ public UiAutomationService(ILogger logger, ISelectorService return EnumerateWindows((pid, title) => pid == targetPid); } + public bool TryGetWindowRect(long hwnd, out PointerRect rect) + { + rect = default; + if (hwnd == 0) + { + return false; + } + + try + { + Windows.Win32.Foundation.RECT r; + bool ok; + unsafe + { + ok = Windows.Win32.PInvoke.GetWindowRect( + new Windows.Win32.Foundation.HWND((nint)hwnd), &r); + } + + if (!ok) + { + return false; + } + + rect = new PointerRect(r.left, r.top, r.right, r.bottom); + return true; + } + catch + { + return false; + } + } + private static List<(nint Hwnd, int Pid, string Title)> EnumerateWindows(Func filter) { var results = new List<(nint, int, string)>(); diff --git a/src/winapp-npm/scripts/generate-commands.mjs b/src/winapp-npm/scripts/generate-commands.mjs index 80f9f9d1..36e7ceaf 100644 --- a/src/winapp-npm/scripts/generate-commands.mjs +++ b/src/winapp-npm/scripts/generate-commands.mjs @@ -108,7 +108,7 @@ function deriveUnionName(valueType) { function tsType(valueType, helpName) { if (!valueType) return 'string'; if (valueType.includes('Boolean')) return 'boolean'; - if (valueType.includes('Int32') || valueType.includes('Int64') || valueType.includes('Double')) return 'number'; + if (valueType.includes('Int32') || valueType.includes('Int64') || valueType.includes('Double') || valueType.includes('Single')) return 'number'; // If helpName has pipe-separated values, it's an enum — derive a named union type if (helpName && helpName.includes('|')) { diff --git a/src/winapp-npm/src/winapp-commands.ts b/src/winapp-npm/src/winapp-commands.ts index c120c800..f1b4ac94 100644 --- a/src/winapp-npm/src/winapp-commands.ts +++ b/src/winapp-npm/src/winapp-commands.ts @@ -861,6 +861,51 @@ export async function uiListWindows(options: UiListWindowsOptions = {}): Promise return execCommand(args, options); } +// --------------------------------------------------------------------------- +// ui pen +// --------------------------------------------------------------------------- + +export interface UiPenOptions extends CommonOptions { + /** Semantic slug (e.g., btn-minimize-d1a0) or text to search by name/automationId */ + selector?: string; + /** Target app (process name, window title, or PID). Lists windows if ambiguous. */ + app?: string; + /** Pen contact point as app coordinates x,y (as reported by 'ui inspect'). Defaults to the selector's element center. Ignored when --path is given. */ + at?: string; + /** Use the eraser end of the pen instead of the tip. */ + eraser?: boolean; + /** Format output as JSON */ + json?: boolean; + /** Ink stroke path as a whitespace-separated list of x,y pairs, e.g. "10,10 20,30 40,50". */ + path?: string; + /** Pen pressure from 0.0 to 1.0 (default: 0.5). */ + pressure?: number; + /** Pen tilt along the x-axis in degrees (-90 to 90, default: 0). */ + tiltX?: number; + /** Pen tilt along the y-axis in degrees (-90 to 90, default: 0). */ + tiltY?: number; + /** Target window by HWND (stable handle from list output). Takes precedence over --app. */ + window?: number; +} + +/** + * Inject synthetic pen/stylus input using the Windows synthetic-pointer API. Taps or draws ink strokes with configurable pressure, tilt and eraser mode, at an element's center or explicit app x,y coordinates. Requires an unlocked, interactive desktop with the target window foregroundable (Windows 10 1809+). + */ +export async function uiPen(options: UiPenOptions = {}): Promise { + const args: string[] = ['ui', 'pen']; + if (options.selector) args.push(options.selector); + if (options.app) args.push('--app', options.app); + if (options.at) args.push('--at', options.at); + if (options.eraser) args.push('--eraser'); + if (options.json) args.push('--json'); + if (options.path) args.push('--path', options.path); + if (options.pressure !== undefined) args.push('--pressure', options.pressure.toString()); + if (options.tiltX !== undefined) args.push('--tilt-x', options.tiltX.toString()); + if (options.tiltY !== undefined) args.push('--tilt-y', options.tiltY.toString()); + if (options.window !== undefined) args.push('--window', options.window.toString()); + return execCommand(args, options); +} + // --------------------------------------------------------------------------- // ui screenshot // --------------------------------------------------------------------------- @@ -1080,6 +1125,54 @@ export async function uiStatus(options: UiStatusOptions = {}): Promise { + const args: string[] = ['ui', 'touch']; + if (options.selector) args.push(options.selector); + if (options.app) args.push('--app', options.app); + if (options.at) args.push('--at', options.at); + if (options.distance !== undefined) args.push('--distance', options.distance.toString()); + if (options.durationMs !== undefined) args.push('--duration-ms', options.durationMs.toString()); + if (options.fingers !== undefined) args.push('--fingers', options.fingers.toString()); + if (options.gesture) args.push('--gesture', options.gesture); + if (options.holdMs !== undefined) args.push('--hold-ms', options.holdMs.toString()); + if (options.json) args.push('--json'); + if (options.toPoint) args.push('--to-point', options.toPoint); + if (options.window !== undefined) args.push('--window', options.window.toString()); + return execCommand(args, options); +} + // --------------------------------------------------------------------------- // ui wait-for // ---------------------------------------------------------------------------