diff --git a/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/src/commands/interaction/runtime/__tests__/test-utils/index.ts new file mode 100644 index 000000000..37487943a --- /dev/null +++ b/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -0,0 +1,431 @@ +import type { AgentDeviceBackend, BackendSnapshotResult } from '../../../../../backend.ts'; +import { createLocalArtifactAdapter } from '../../../../../io.ts'; +import type { SnapshotState } from '../../../../../kernel/snapshot.ts'; +import { + createAgentDevice, + createMemorySessionStore, + localCommandPolicy, + type CommandSessionStore, +} from '../../../../../runtime.ts'; +import { ref } from '../../../../index.ts'; +import { makeSnapshotState } from '../../../../../__tests__/test-utils/index.ts'; + +export function selectorSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Continue', + value: 'Continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: true, + }, + ]); +} + +// Closed-drawer shape shared by the native-ref preflight tests: the only +// interactive node (@e2) sits fully left of the Application viewport. +export function offscreenDrawerSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 400, height: 800 }, + hittable: true, + }, + { + index: 1, + depth: 2, + parentIndex: 0, + type: 'Button', + label: 'Explore', + rect: { x: -320, y: 240, width: 300, height: 50 }, + hittable: true, + }, + ]); +} + +export function runtimeScrollSnapshot(options: { + hiddenBelow: boolean; + message?: string; +}): SnapshotState { + return makeSnapshotState([ + { + index: 1, + depth: 0, + type: 'ScrollView', + label: 'Messages', + hiddenContentBelow: options.hiddenBelow ? true : undefined, + rect: { x: 0, y: 100, width: 400, height: 600 }, + }, + { + index: 2, + depth: 1, + parentIndex: 1, + type: 'Button', + label: options.message ?? 'Latest message', + rect: { x: 0, y: 640, width: 400, height: 56 }, + hittable: true, + }, + ]); +} + +export function fillableSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect: { x: 20, y: 10, width: 60, height: 40 }, + hittable: true, + }, + ]); +} + +export function iosTabBarSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeApplication', + label: 'TabRepro', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: false, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeTabBar', + rect: { x: 0, y: 791, width: 402, height: 83 }, + hittable: true, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'XCUIElementTypeButton', + label: 'Home', + rect: { x: 30, y: 800, width: 91, height: 44 }, + hittable: false, + }, + { + index: 3, + depth: 2, + parentIndex: 1, + type: 'XCUIElementTypeButton', + label: 'Library', + rect: { x: 120, y: 800, width: 92, height: 44 }, + hittable: false, + }, + { + index: 4, + depth: 2, + parentIndex: 1, + type: 'XCUIElementTypeButton', + label: 'Settings', + rect: { x: 211, y: 800, width: 91, height: 44 }, + hittable: false, + }, + { + index: 5, + depth: 2, + parentIndex: 1, + type: 'XCUIElementTypeButton', + label: 'Search', + rect: { x: 304, y: 800, width: 92, height: 44 }, + hittable: false, + }, + ]); +} + +export function coveredByTabBarSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Save draft', + rect: { x: 16, y: 790, width: 140, height: 44 }, + hittable: false, + interactionBlocked: 'covered', + presentationHints: ['covered'], + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'TabBar', + rect: { x: 0, y: 760, width: 390, height: 84 }, + hittable: true, + }, + ]); +} + +export function duplicateCoveredLabelSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Save draft', + rect: { x: 16, y: 120, width: 140, height: 44 }, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Save draft', + rect: { x: 16, y: 790, width: 140, height: 44 }, + hittable: false, + interactionBlocked: 'covered', + presentationHints: ['covered'], + }, + { + index: 3, + depth: 1, + parentIndex: 0, + type: 'TabBar', + rect: { x: 0, y: 760, width: 390, height: 84 }, + hittable: true, + }, + ]); +} + +export function nonHittableCellSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeOther', + label: 'Settings list', + rect: { x: 10, y: 20, width: 300, height: 80 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeCell', + label: 'Account', + rect: { x: 20, y: 10, width: 100, height: 40 }, + hittable: false, + }, + ]); +} + +// Mirrors the #1037 Maps repro: a non-hittable map-pin annotation exact-matches +// `text="Anthropic - Headquarters"` while the real, longer-labeled row is a +// separate node. Fill/press must still proceed against the unique match (no +// stricter resolution) but should flag it as likely non-actionable. +export function mapPinAnnotationSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Other', + label: 'Anthropic - Headquarters', + rect: { x: 177, y: 446, width: 30, height: 30 }, + hittable: false, + }, + { + index: 1, + depth: 1, + type: 'Button', + label: 'Anthropic - Headquarters, 548 Market St', + rect: { x: 0, y: 786, width: 390, height: 60 }, + hittable: false, + }, + ]); +} + +export function nonTouchableGroupSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeOther', + label: 'Clickable group', + rect: { x: 10, y: 20, width: 300, height: 80 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeOther', + label: 'Decorative group', + rect: { x: 30, y: 40, width: 60, height: 20 }, + hittable: false, + }, + ]); +} + +export function snapshotWithOffscreenContent(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 100, height: 100 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Visible', + rect: { x: 10, y: 10, width: 20, height: 20 }, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Offscreen', + rect: { x: 10, y: 900, width: 20, height: 20 }, + hittable: true, + }, + ]); +} + +export function createInteractionDevice( + snapshot: SnapshotState, + overrides: Partial< + Pick< + AgentDeviceBackend, + | 'captureSnapshot' + | 'tap' + | 'tapTarget' + | 'fill' + | 'fillTarget' + | 'typeText' + | 'focus' + | 'longPress' + | 'scroll' + | 'swipe' + | 'pinch' + > + > & { + platform?: AgentDeviceBackend['platform']; + sessionMetadata?: Record; + } = {}, +) { + return createAgentDevice({ + backend: { + platform: overrides.platform ?? 'ios', + captureSnapshot: async (...args) => + overrides.captureSnapshot ? await overrides.captureSnapshot(...args) : { snapshot }, + tap: async (...args) => await overrides.tap?.(...args), + tapTarget: overrides.tapTarget + ? async (...args) => await overrides.tapTarget?.(...args) + : undefined, + fill: async (...args) => await overrides.fill?.(...args), + fillTarget: overrides.fillTarget + ? async (...args) => await overrides.fillTarget?.(...args) + : undefined, + typeText: async (...args) => await overrides.typeText?.(...args), + focus: overrides.focus ? async (...args) => await overrides.focus?.(...args) : undefined, + longPress: overrides.longPress + ? async (...args) => await overrides.longPress?.(...args) + : undefined, + scroll: overrides.scroll ? async (...args) => await overrides.scroll?.(...args) : undefined, + swipe: overrides.swipe ? async (...args) => await overrides.swipe?.(...args) : undefined, + pinch: overrides.pinch ? async (...args) => await overrides.pinch?.(...args) : undefined, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([ + { name: 'default', snapshot, metadata: overrides.sessionMetadata }, + ]), + policy: localCommandPolicy(), + }); +} + +export async function clickRefE2(device: ReturnType) { + return await device.interactions.click(ref('@e2'), { + session: 'default', + }); +} + +export function createFakeClock(stepMs = 300): { + now: () => number; + sleep: (ms: number) => Promise; +} { + let elapsed = 0; + return { + now: () => elapsed, + sleep: async (ms: number) => { + elapsed += ms > 0 ? ms : stepMs; + }, + }; +} + +export function selectorReadSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Continue', + value: 'Continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + }, + ]); +} + +export function createSelectorDevice( + snapshot: SnapshotState, + options: { + readText?: string; + findText?: boolean; + now?: number; + captureSnapshot?: () => BackendSnapshotResult | Promise; + } = {}, +) { + const session = { name: 'default', snapshot }; + const sessions = { + get: () => session, + set: (record) => { + session.snapshot = record.snapshot ?? session.snapshot; + }, + } satisfies CommandSessionStore; + return createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => + options.captureSnapshot ? await options.captureSnapshot() : { snapshot }, + readText: async () => ({ text: options.readText ?? '' }), + findText: async () => ({ found: options.findText ?? false }), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions, + policy: localCommandPolicy(), + clock: { + now: () => options.now ?? 0, + sleep: async () => {}, + }, + }); +} diff --git a/src/commands/interaction/runtime/gestures.test.ts b/src/commands/interaction/runtime/gestures.test.ts new file mode 100644 index 000000000..e6c957771 --- /dev/null +++ b/src/commands/interaction/runtime/gestures.test.ts @@ -0,0 +1,417 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { ref, selector } from '../../index.ts'; +import { AppError } from '../../../kernel/errors.ts'; +import { + createInteractionDevice, + runtimeScrollSnapshot, + selectorSnapshot, + snapshotWithOffscreenContent, +} from './__tests__/test-utils/index.ts'; + +test('runtime focus and longPress share selector/ref target resolution', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(selectorSnapshot(), { + focus: async (_context, point) => { + calls.push({ command: 'focus', point }); + return { focused: true }; + }, + longPress: async (_context, point, options) => { + calls.push({ command: 'longPress', point, durationMs: options?.durationMs }); + }, + }); + + const focused = await device.interactions.focus(selector('label=Continue'), { + session: 'default', + }); + const longPressed = await device.interactions.longPress(ref('@e1'), { + session: 'default', + durationMs: 750, + }); + + assert.equal(focused.kind, 'selector'); + assert.deepEqual(focused.backendResult, { focused: true }); + assert.equal(longPressed.kind, 'ref'); + assert.deepEqual(calls, [ + { command: 'focus', point: { x: 60, y: 40 } }, + { command: 'longPress', point: { x: 60, y: 40 }, durationMs: 750 }, + ]); +}); + +test('runtime scroll resolves selector targets before calling the backend primitive', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(selectorSnapshot(), { + scroll: async (_context, target, options) => { + calls.push({ target, options }); + return { scrolled: true }; + }, + }); + + const selectorResult = await device.interactions.scroll({ + session: 'default', + target: selector('label=Continue'), + direction: 'down', + pixels: 120, + durationMs: 50, + }); + const viewportResult = await device.interactions.scroll({ + direction: 'up', + amount: 0.5, + }); + + assert.equal(selectorResult.kind, 'selector'); + assert.equal(selectorResult.durationMs, undefined); + assert.equal(viewportResult.kind, 'viewport'); + assert.deepEqual(calls, [ + { + target: { kind: 'point', point: { x: 60, y: 40 } }, + options: { direction: 'down', pixels: 120, durationMs: 50 }, + }, + { + target: { kind: 'viewport' }, + options: { direction: 'up', amount: 0.5 }, + }, + ]); +}); + +test('runtime scroll reports duration only when the backend honored it', async () => { + const device = createInteractionDevice(selectorSnapshot(), { + scroll: async (_context, _target, options) => ({ durationMs: options?.durationMs }), + }); + + const result = await device.interactions.scroll({ + direction: 'down', + pixels: 120, + durationMs: 50, + }); + + assert.equal(result.durationMs, 50); + assert.deepEqual(result.backendResult, { durationMs: 50 }); +}); + +test('runtime scroll rejects duration above the shared cap', async () => { + const device = createInteractionDevice(selectorSnapshot(), { + scroll: async () => { + throw new Error('scroll should be rejected before backend call'); + }, + }); + + await assert.rejects( + () => + device.interactions.scroll({ + direction: 'down', + pixels: 120, + durationMs: 10_001, + }), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /durationMs.*at most 10000/i.test(error.message), + ); +}); + +test('runtime scroll bottom rejects blind scrolling without snapshot support', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async () => { + throw new Error('snapshot unavailable'); + }, + scroll: async (_context, target, options) => { + calls.push({ target, options }); + return { pass: calls.length }; + }, + }); + + await assert.rejects( + () => + device.interactions.scroll({ + direction: 'bottom', + }), + /Failed to verify scroll bottom state/, + ); + + assert.equal(calls.length, 0); +}); + +test('runtime scroll bottom does not scroll when no hidden content is below', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(runtimeScrollSnapshot({ hiddenBelow: false }), { + scroll: async (_context, target, options) => { + calls.push({ target, options }); + return { pass: calls.length }; + }, + }); + + const result = await device.interactions.scroll({ + direction: 'bottom', + }); + + assert.equal(result.kind, 'viewport'); + assert.equal(result.edge, 'bottom'); + assert.equal(result.passes, 0); + assert.equal(calls.length, 0); +}); + +test('runtime scroll bottom scrolls only while scoped snapshot confirms hidden content', async () => { + const calls: unknown[] = []; + const snapshotScopes: unknown[] = []; + const snapshots = [ + runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), + runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), + runtimeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), + ]; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async (_context, options) => { + snapshotScopes.push(options?.scope); + return { snapshot: snapshots[Math.min(snapshotScopes.length - 1, snapshots.length - 1)] }; + }, + scroll: async (_context, target, options) => { + calls.push({ target, options }); + return { pass: calls.length }; + }, + }); + + const result = await device.interactions.scroll({ + direction: 'bottom', + }); + + assert.equal(result.kind, 'viewport'); + assert.equal(result.edge, 'bottom'); + assert.equal(result.passes, 1); + assert.equal(result.backendResult?.pass, 1); + assert.deepEqual(calls, [ + { + target: { kind: 'viewport' }, + options: { direction: 'down' }, + }, + ]); + assert.deepEqual(snapshotScopes, [undefined, 'Messages', 'Messages']); +}); + +test('runtime scroll bottom tolerates unchanged signatures while hidden content advances', async () => { + const calls: unknown[] = []; + const snapshots = [ + runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), + runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), + runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), + runtimeScrollSnapshot({ hiddenBelow: false, message: 'Repeated row' }), + ]; + let snapshotIndex = 0; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async () => ({ + snapshot: snapshots[Math.min(snapshotIndex++, snapshots.length - 1)], + }), + scroll: async (_context, target, options) => { + calls.push({ target, options }); + return { pass: calls.length }; + }, + }); + + const result = await device.interactions.scroll({ + direction: 'bottom', + }); + + assert.equal(result.passes, 2); + assert.equal(calls.length, 2); +}); + +test('runtime scroll bottom keeps scoped snapshot failures scoped', async () => { + let snapshotCount = 0; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async (_context, options) => { + snapshotCount += 1; + if (options?.scope) throw new Error('scoped snapshot failed'); + return { snapshot: runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }) }; + }, + scroll: async () => ({}), + }); + + await assert.rejects( + () => + device.interactions.scroll({ + direction: 'bottom', + }), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + /scoped container/i.test(error.message) && + error.details?.scope === 'Messages', + ); + assert.equal(snapshotCount, 2); +}); + +test('runtime swipe supports explicit and viewport-derived targets', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(selectorSnapshot(), { + swipe: async (_context, from, to, options) => { + calls.push({ from, to, durationMs: options?.durationMs }); + }, + }); + + const explicit = await device.interactions.swipe({ + from: selector('label=Continue'), + to: { x: 200, y: 40 }, + durationMs: 300, + session: 'default', + }); + const directional = await device.interactions.swipe({ + direction: 'left', + distance: 25, + session: 'default', + }); + + assert.deepEqual(explicit.from, { x: 60, y: 40 }); + assert.deepEqual(directional.from, { x: 60, y: 40 }); + assert.deepEqual(directional.to, { x: 35, y: 40 }); + assert.deepEqual(calls, [ + { from: { x: 60, y: 40 }, to: { x: 200, y: 40 }, durationMs: 300 }, + { from: { x: 60, y: 40 }, to: { x: 35, y: 40 }, durationMs: undefined }, + ]); +}); + +test('runtime directional swipe uses the visible viewport instead of off-screen content bounds', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(snapshotWithOffscreenContent(), { + swipe: async (_context, from, to) => { + calls.push({ from, to }); + }, + }); + + const result = await device.interactions.swipe({ + direction: 'left', + distance: 25, + session: 'default', + }); + + assert.deepEqual(result.from, { x: 50, y: 50 }); + assert.deepEqual(result.to, { x: 25, y: 50 }); + assert.deepEqual(calls, [{ from: { x: 50, y: 50 }, to: { x: 25, y: 50 } }]); +}); + +test('runtime gesture swipe presets use stable viewport lanes', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(snapshotWithOffscreenContent(), { + platform: 'android', + swipe: async (_context, from, to, options) => { + calls.push({ from, to, durationMs: options?.durationMs }); + }, + }); + + const pageSwipe = await device.interactions.swipe({ + preset: 'left', + durationMs: 300, + session: 'default', + }); + const edgeSwipe = await device.interactions.swipe({ + preset: 'right-edge', + durationMs: 350, + session: 'default', + }); + + assert.deepEqual(pageSwipe.from, { x: 85, y: 50 }); + assert.deepEqual(pageSwipe.to, { x: 15, y: 50 }); + assert.deepEqual(edgeSwipe.from, { x: 8, y: 50 }); + assert.deepEqual(edgeSwipe.to, { x: 85, y: 50 }); + assert.deepEqual(calls, [ + { from: { x: 85, y: 50 }, to: { x: 15, y: 50 }, durationMs: 300 }, + { from: { x: 8, y: 50 }, to: { x: 85, y: 50 }, durationMs: 350 }, + ]); +}); + +test('runtime iOS in-page swipe presets avoid edge-navigation lanes', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(snapshotWithOffscreenContent(), { + platform: 'ios', + swipe: async (_context, from, to, options) => { + calls.push({ from, to, durationMs: options?.durationMs }); + }, + }); + + const pageSwipe = await device.interactions.swipe({ + preset: 'right', + durationMs: 300, + session: 'default', + }); + + assert.deepEqual(pageSwipe.from, { x: 15, y: 50 }); + assert.deepEqual(pageSwipe.to, { x: 85, y: 50 }); + assert.deepEqual(calls, [{ from: { x: 15, y: 50 }, to: { x: 85, y: 50 }, durationMs: 300 }]); +}); + +test('runtime viewport gestures reject inspect-only macOS surfaces', async () => { + for (const surface of ['desktop', 'menubar'] as const) { + const device = createInteractionDevice(selectorSnapshot(), { + platform: 'macos', + sessionMetadata: { surface }, + scroll: async () => { + throw new Error(`${surface} scroll should be rejected before backend call`); + }, + swipe: async () => { + throw new Error(`${surface} swipe should be rejected before backend call`); + }, + pinch: async () => { + throw new Error(`${surface} pinch should be rejected before backend call`); + }, + }); + + await assert.rejects( + () => + device.interactions.scroll({ + direction: 'down', + target: { kind: 'viewport' }, + session: 'default', + }), + new RegExp(`scroll is not supported on macOS ${surface}`), + ); + await assert.rejects( + () => + device.interactions.swipe({ + direction: 'left', + session: 'default', + }), + new RegExp(`swipe is not supported on macOS ${surface}`), + ); + await assert.rejects( + () => + device.interactions.swipe({ + from: { x: 10, y: 20 }, + to: { x: 30, y: 20 }, + session: 'default', + }), + new RegExp(`swipe is not supported on macOS ${surface}`), + ); + await assert.rejects( + () => + device.interactions.pinch({ + scale: 1.2, + session: 'default', + }), + new RegExp(`pinch is not supported on macOS ${surface}`), + ); + } +}); + +test('runtime pinch is backend-gated and resolves optional center targets', async () => { + const calls: unknown[] = []; + const unsupported = createInteractionDevice(selectorSnapshot()); + await assert.rejects( + () => unsupported.interactions.pinch({ scale: 1.2 }), + /pinch is not supported/, + ); + + const device = createInteractionDevice(selectorSnapshot(), { + pinch: async (_context, options) => { + calls.push(options); + }, + }); + + const result = await device.interactions.pinch({ + scale: 0.8, + center: ref('@e1'), + session: 'default', + }); + + assert.equal(result.kind, 'pinch'); + assert.deepEqual(result.center, { x: 60, y: 40 }); + assert.deepEqual(calls, [{ scale: 0.8, center: { x: 60, y: 40 } }]); +}); diff --git a/src/commands/interaction/runtime/index.test.ts b/src/commands/interaction/runtime/index.test.ts new file mode 100644 index 000000000..eec255cc0 --- /dev/null +++ b/src/commands/interaction/runtime/index.test.ts @@ -0,0 +1,17 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { commands, selector } from '../../index.ts'; +import { createInteractionDevice, selectorSnapshot } from './__tests__/test-utils/index.ts'; + +test('runtime interaction commands are available from the command namespace', async () => { + const device = createInteractionDevice(selectorSnapshot(), { + tap: async () => {}, + }); + + const result = await commands.interactions.click(device, { + session: 'default', + target: selector('label=Continue'), + }); + + assert.equal(result.kind, 'selector'); +}); diff --git a/src/commands/interaction/runtime/interactions.test.ts b/src/commands/interaction/runtime/interactions.test.ts index 3a6f83a38..7f7e7e98d 100644 --- a/src/commands/interaction/runtime/interactions.test.ts +++ b/src/commands/interaction/runtime/interactions.test.ts @@ -1,17 +1,23 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import type { AgentDeviceBackend, BackendSnapshotOptions } from '../../../backend.ts'; -import { commands, ref, selector } from '../../index.ts'; -import { resolveActionableTouchResolution } from '../../../core/interaction-targeting.ts'; +import type { AgentDeviceBackend } from '../../../backend.ts'; +import { ref, selector } from '../../index.ts'; import { createLocalArtifactAdapter } from '../../../io.ts'; import { createAgentDevice, createMemorySessionStore, localCommandPolicy, } from '../../../runtime.ts'; -import { AppError } from '../../../kernel/errors.ts'; -import type { Point, SnapshotState } from '../../../kernel/snapshot.ts'; +import type { Point } from '../../../kernel/snapshot.ts'; import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts'; +import { + coveredByTabBarSnapshot, + createInteractionDevice, + fillableSnapshot, + nonHittableCellSnapshot, + offscreenDrawerSnapshot, + selectorSnapshot, +} from './__tests__/test-utils/index.ts'; test('runtime click taps an explicit point without requiring a snapshot', async () => { const calls: Array<{ point: Point; count?: number }> = []; @@ -287,113 +293,6 @@ test('runtime interactions pass runtime signal to backend primitives', async () assert.equal(signal, controller.signal); }); -test('runtime press resolves selector targets to the actionable node center', async () => { - const calls: Point[] = []; - const device = createInteractionDevice(selectorSnapshot(), { - tap: async (_context, point) => { - calls.push(point); - return { ok: true }; - }, - }); - - const result = await device.interactions.press(selector('label=Continue'), { - session: 'default', - }); - - assert.deepEqual(calls, [{ x: 60, y: 40 }]); - assert.equal(result.kind, 'selector'); - assert.deepEqual(result.target, { kind: 'selector', selector: 'label=Continue' }); - assert.equal(result.node?.label, 'Continue'); - assert.deepEqual(result.selectorChain, [ - 'role="button" label="Continue"', - 'label="Continue"', - 'value="Continue"', - ]); - assert.deepEqual(result.backendResult, { ok: true }); -}); - -test('runtime selector interactions fall back to a full snapshot when interactive refresh misses', async () => { - const calls: Point[] = []; - const captureOptions: Array = []; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async (_context, options) => { - captureOptions.push(options); - return { - snapshot: options?.interactiveOnly - ? makeSnapshotState([]) - : makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'XCUIElementTypeCell', - label: 'General', - rect: { x: 0, y: 100, width: 320, height: 44 }, - hittable: true, - }, - ]), - }; - }, - tap: async (_context, point) => { - calls.push(point); - }, - }); - - const result = await device.interactions.click(selector('label=General'), { - session: 'default', - }); - - assert.equal(result.kind, 'selector'); - assert.equal(result.node?.label, 'General'); - assert.deepEqual(calls, [{ x: 160, y: 122 }]); - assert.deepEqual(captureOptions, [ - { interactiveOnly: true, includeRects: true }, - { interactiveOnly: false, includeRects: true }, - ]); -}); - -test('runtime press refuses a selector that resolves to an off-screen element', async () => { - // Closed-drawer shape: the only match sits fully left of the viewport. The - // @ref path already refuses this; the selector path must not silently tap - // out-of-viewport coordinates. - const offscreenSnapshot = makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Application', - rect: { x: 0, y: 0, width: 400, height: 800 }, - hittable: true, - }, - { - index: 1, - depth: 2, - parentIndex: 0, - type: 'Button', - label: 'Explore', - rect: { x: -320, y: 240, width: 300, height: 50 }, - hittable: true, - }, - ]); - const taps: unknown[] = []; - const device = createInteractionDevice(offscreenSnapshot, { - tap: async (_context, point) => { - taps.push(point); - }, - }); - - await assert.rejects( - () => device.interactions.press(selector('label=Explore'), { session: 'default' }), - (error: unknown) => { - assert.ok(error instanceof Error); - assert.match(error.message, /off-screen element and is not safe to press/); - const details = (error as { details?: Record }).details; - assert.equal(details?.reason, 'offscreen_selector'); - assert.ok(typeof details?.hint === 'string'); - return true; - }, - ); - assert.equal(taps.length, 0); -}); - test('runtime press with verify drops the non-hittable hint when evidence proves a change', async () => { let captureCount = 0; const nonHittableSnapshot = () => @@ -594,245 +493,6 @@ test('runtime fill with verify reports evidence and detects a changed post-actio assert.equal(result.evidence?.nodeCount, 1); }); -test('runtime click keeps distinct tab button centers when iOS reports the tab bar as hittable', async () => { - const calls: Point[] = []; - const device = createInteractionDevice(iosTabBarSnapshot(), { - tap: async (_context, point) => { - calls.push(point); - }, - }); - - const refResult = await device.interactions.click(ref('@e4'), { - session: 'default', - }); - const selectorResult = await device.interactions.click(selector('label=Settings'), { - session: 'default', - }); - - assert.deepEqual(calls, [ - { x: 166, y: 822 }, - { x: 257, y: 822 }, - ]); - assert.equal(refResult.kind, 'ref'); - assert.equal(refResult.node?.label, 'Library'); - assert.equal(selectorResult.kind, 'selector'); - assert.equal(selectorResult.node?.label, 'Settings'); -}); - -test('runtime click rejects refs covered by floating overlays', async () => { - const calls: Point[] = []; - const device = createInteractionDevice(coveredByTabBarSnapshot(), { - tap: async (_context, point) => { - calls.push(point); - }, - }); - - await assert.rejects( - () => device.interactions.click(ref('@e2'), { session: 'default' }), - /Ref @e2 is covered by another visible element/, - ); - assert.deepEqual(calls, []); -}); - -test('runtime selector interactions skip covered matches when an uncovered duplicate exists', async () => { - const calls: Point[] = []; - const device = createInteractionDevice(duplicateCoveredLabelSnapshot(), { - tap: async (_context, point) => { - calls.push(point); - }, - }); - - const result = await device.interactions.click(selector('label="Save draft"'), { - session: 'default', - }); - - assert.equal(result.kind, 'selector'); - assert.equal(result.node?.ref, 'e2'); - assert.deepEqual(calls, [{ x: 86, y: 142 }]); -}); - -test('runtime click keeps non-button semantic targets at their own center', async () => { - const calls: Point[] = []; - const device = createInteractionDevice(nonHittableCellSnapshot(), { - tap: async (_context, point) => { - calls.push(point); - }, - }); - - const result = await clickRefE2(device); - - assert.deepEqual(calls, [{ x: 70, y: 30 }]); - assert.equal(result.kind, 'ref'); - assert.equal(result.node?.label, 'Account'); -}); - -test('runtime press surfaces targetHittable and a hint when the final tap node is non-hittable (#1037)', async () => { - const calls: Point[] = []; - const device = createInteractionDevice(nonHittableCellSnapshot(), { - tap: async (_context, point) => { - calls.push(point); - }, - }); - - const result = await device.interactions.press(ref('@e2'), { session: 'default' }); - - // Press still proceeds and reports success — non-hittable is informational only. - assert.deepEqual(calls, [{ x: 70, y: 30 }]); - assert.equal(result.kind, 'ref'); - assert.equal(result.node?.label, 'Account'); - assert.equal(result.targetHittable, false); - assert.match(result.hint ?? '', /hittable: false/); - assert.match(result.hint ?? '', /@ref/); -}); - -test('runtime press omits targetHittable and hint when the resolved node is hittable', async () => { - const device = createInteractionDevice(selectorSnapshot(), { - tap: async () => {}, - }); - - const result = await device.interactions.press(selector('label=Continue'), { - session: 'default', - }); - - assert.equal(result.kind, 'selector'); - assert.equal(result.targetHittable, undefined); - assert.equal(result.hint, undefined); -}); - -test('runtime fill surfaces targetHittable and a hint for a non-hittable selector match (Maps pin case, #1037)', async () => { - const calls: Array<{ point: Point; text: string }> = []; - const device = createInteractionDevice(mapPinAnnotationSnapshot(), { - fill: async (_context, point, text) => { - calls.push({ point, text }); - }, - }); - - const result = await device.interactions.fill( - selector('text="Anthropic - Headquarters"'), - 'ignored', - { session: 'default' }, - ); - - assert.equal(result.kind, 'selector'); - assert.equal(result.node?.label, 'Anthropic - Headquarters'); - assert.equal(result.targetHittable, false); - assert.match(result.hint ?? '', /hittable: false/); - assert.deepEqual(calls, [{ point: { x: 192, y: 461 }, text: 'ignored' }]); -}); - -test('runtime click still promotes non-touchable nodes to hittable ancestors', async () => { - const calls: Point[] = []; - const device = createInteractionDevice(nonTouchableGroupSnapshot(), { - tap: async (_context, point) => { - calls.push(point); - }, - }); - - const result = await clickRefE2(device); - - assert.deepEqual(calls, [{ x: 160, y: 60 }]); - assert.equal(result.kind, 'ref'); - assert.equal(result.node?.label, 'Clickable group'); -}); - -test('touch resolution promotes static text inside a hittable row to the row', () => { - const snapshot = makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'XCUIElementTypeCell', - label: 'Account row', - rect: { x: 10, y: 20, width: 300, height: 60 }, - hittable: true, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeStaticText', - label: 'Account', - rect: { x: 24, y: 32, width: 80, height: 20 }, - hittable: false, - }, - ]); - - const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[1]!); - - assert.equal(resolution.reason, 'hittable-ancestor'); - assert.equal(resolution.node.label, 'Account row'); -}); - -test('touch resolution prefers same-rect hittable descendants over semantic targets', () => { - const snapshot = makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'XCUIElementTypeButton', - label: 'Profile', - rect: { x: 30, y: 40, width: 120, height: 50 }, - hittable: false, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeImage', - identifier: 'profile-hit-area', - rect: { x: 30, y: 40, width: 120, height: 50 }, - hittable: true, - }, - ]); - - const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[0]!); - - assert.equal(resolution.reason, 'same-rect-descendant'); - assert.equal(resolution.node.identifier, 'profile-hit-area'); -}); - -test('touch resolution prevents full-screen window-like ancestors from stealing taps', () => { - const snapshot = makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'XCUIElementTypeApplication', - label: 'Example', - rect: { x: 0, y: 0, width: 390, height: 844 }, - hittable: true, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeStaticText', - label: 'Status', - rect: { x: 24, y: 72, width: 80, height: 24 }, - hittable: false, - }, - ]); - - const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[1]!); - - assert.equal(resolution.reason, 'overly-broad-ancestor'); - assert.equal(resolution.node.label, 'Status'); -}); - -test('touch resolution falls back to the original node when no usable touch target exists', () => { - const snapshot = makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'XCUIElementTypeOther', - label: 'Virtual item', - hittable: false, - }, - ]); - - const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[0]!); - - assert.equal(resolution.reason, 'original'); - assert.equal(resolution.node.label, 'Virtual item'); -}); - test('runtime fill resolves refs and forwards text to the backend primitive', async () => { const calls: Array<{ point: Point; text: string; delayMs?: number }> = []; const device = createInteractionDevice(fillableSnapshot(), { @@ -856,84 +516,6 @@ test('runtime fill resolves refs and forwards text to the backend primitive', as assert.equal(result.warning, undefined); }); -test('runtime interactions reject unsupported macOS desktop and menubar surfaces', async () => { - const desktop = createInteractionDevice(selectorSnapshot(), { - platform: 'macos', - sessionMetadata: { surface: 'desktop' }, - tap: async () => { - throw new Error('desktop click should be rejected before backend tap'); - }, - }); - await assert.rejects( - () => desktop.interactions.click({ kind: 'point', x: 1, y: 2 }, { session: 'default' }), - /click is not supported on macOS desktop sessions yet/, - ); - await assert.rejects( - () => - desktop.interactions.click( - { kind: 'point', x: 1, y: 2 }, - { session: 'default', metadata: { surface: 'app' } }, - ), - /click is not supported on macOS desktop sessions yet/, - ); - - const menubar = createInteractionDevice(fillableSnapshot(), { - platform: 'macos', - sessionMetadata: { surface: 'menubar' }, - fill: async () => { - throw new Error('menubar fill should be rejected before backend fill'); - }, - }); - await assert.rejects( - () => menubar.interactions.fill(ref('@e1'), 'hello', { session: 'default' }), - /fill is not supported on macOS menubar sessions yet/, - ); - - let pressed = false; - const menubarPress = createInteractionDevice(fillableSnapshot(), { - platform: 'macos', - sessionMetadata: { surface: 'menubar' }, - tap: async () => { - pressed = true; - }, - }); - - await menubarPress.interactions.press(ref('@e1'), { session: 'default' }); - - assert.equal(pressed, true); -}); - -test('runtime ref interactions refresh the snapshot when a stored ref has no usable rect', async () => { - const staleSnapshot = makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Button', - label: 'Continue', - hittable: true, - }, - ]); - const freshSnapshot = selectorSnapshot(); - const calls: Point[] = []; - let captures = 0; - const device = createInteractionDevice(staleSnapshot, { - captureSnapshot: async () => { - captures += 1; - return { snapshot: freshSnapshot }; - }, - tap: async (_context, point) => { - calls.push(point); - }, - }); - - const result = await device.interactions.click(ref('@e1'), { session: 'default' }); - - assert.equal(captures, 1); - assert.deepEqual(calls, [{ x: 60, y: 40 }]); - assert.equal(result.kind, 'ref'); - assert.equal(result.node?.rect?.width, 100); -}); - test('runtime typeText validates refs and forwards text to the backend primitive', async () => { const calls: Array<{ text: string; delayMs?: number }> = []; const device = createInteractionDevice(selectorSnapshot(), { @@ -959,426 +541,6 @@ test('runtime typeText validates refs and forwards text to the backend primitive ); }); -test('runtime focus and longPress share selector/ref target resolution', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(selectorSnapshot(), { - focus: async (_context, point) => { - calls.push({ command: 'focus', point }); - return { focused: true }; - }, - longPress: async (_context, point, options) => { - calls.push({ command: 'longPress', point, durationMs: options?.durationMs }); - }, - }); - - const focused = await device.interactions.focus(selector('label=Continue'), { - session: 'default', - }); - const longPressed = await device.interactions.longPress(ref('@e1'), { - session: 'default', - durationMs: 750, - }); - - assert.equal(focused.kind, 'selector'); - assert.deepEqual(focused.backendResult, { focused: true }); - assert.equal(longPressed.kind, 'ref'); - assert.deepEqual(calls, [ - { command: 'focus', point: { x: 60, y: 40 } }, - { command: 'longPress', point: { x: 60, y: 40 }, durationMs: 750 }, - ]); -}); - -test('runtime scroll resolves selector targets before calling the backend primitive', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(selectorSnapshot(), { - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { scrolled: true }; - }, - }); - - const selectorResult = await device.interactions.scroll({ - session: 'default', - target: selector('label=Continue'), - direction: 'down', - pixels: 120, - durationMs: 50, - }); - const viewportResult = await device.interactions.scroll({ - direction: 'up', - amount: 0.5, - }); - - assert.equal(selectorResult.kind, 'selector'); - assert.equal(selectorResult.durationMs, undefined); - assert.equal(viewportResult.kind, 'viewport'); - assert.deepEqual(calls, [ - { - target: { kind: 'point', point: { x: 60, y: 40 } }, - options: { direction: 'down', pixels: 120, durationMs: 50 }, - }, - { - target: { kind: 'viewport' }, - options: { direction: 'up', amount: 0.5 }, - }, - ]); -}); - -test('runtime scroll reports duration only when the backend honored it', async () => { - const device = createInteractionDevice(selectorSnapshot(), { - scroll: async (_context, _target, options) => ({ durationMs: options?.durationMs }), - }); - - const result = await device.interactions.scroll({ - direction: 'down', - pixels: 120, - durationMs: 50, - }); - - assert.equal(result.durationMs, 50); - assert.deepEqual(result.backendResult, { durationMs: 50 }); -}); - -test('runtime scroll rejects duration above the shared cap', async () => { - const device = createInteractionDevice(selectorSnapshot(), { - scroll: async () => { - throw new Error('scroll should be rejected before backend call'); - }, - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'down', - pixels: 120, - durationMs: 10_001, - }), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /durationMs.*at most 10000/i.test(error.message), - ); -}); - -test('runtime scroll bottom rejects blind scrolling without snapshot support', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => { - throw new Error('snapshot unavailable'); - }, - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'bottom', - }), - /Failed to verify scroll bottom state/, - ); - - assert.equal(calls.length, 0); -}); - -test('runtime scroll bottom does not scroll when no hidden content is below', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(runtimeScrollSnapshot({ hiddenBelow: false }), { - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - const result = await device.interactions.scroll({ - direction: 'bottom', - }); - - assert.equal(result.kind, 'viewport'); - assert.equal(result.edge, 'bottom'); - assert.equal(result.passes, 0); - assert.equal(calls.length, 0); -}); - -test('runtime scroll bottom scrolls only while scoped snapshot confirms hidden content', async () => { - const calls: unknown[] = []; - const snapshotScopes: unknown[] = []; - const snapshots = [ - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), - runtimeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), - ]; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async (_context, options) => { - snapshotScopes.push(options?.scope); - return { snapshot: snapshots[Math.min(snapshotScopes.length - 1, snapshots.length - 1)] }; - }, - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - const result = await device.interactions.scroll({ - direction: 'bottom', - }); - - assert.equal(result.kind, 'viewport'); - assert.equal(result.edge, 'bottom'); - assert.equal(result.passes, 1); - assert.equal(result.backendResult?.pass, 1); - assert.deepEqual(calls, [ - { - target: { kind: 'viewport' }, - options: { direction: 'down' }, - }, - ]); - assert.deepEqual(snapshotScopes, [undefined, 'Messages', 'Messages']); -}); - -test('runtime scroll bottom tolerates unchanged signatures while hidden content advances', async () => { - const calls: unknown[] = []; - const snapshots = [ - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - runtimeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - runtimeScrollSnapshot({ hiddenBelow: false, message: 'Repeated row' }), - ]; - let snapshotIndex = 0; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async () => ({ - snapshot: snapshots[Math.min(snapshotIndex++, snapshots.length - 1)], - }), - scroll: async (_context, target, options) => { - calls.push({ target, options }); - return { pass: calls.length }; - }, - }); - - const result = await device.interactions.scroll({ - direction: 'bottom', - }); - - assert.equal(result.passes, 2); - assert.equal(calls.length, 2); -}); - -test('runtime scroll bottom keeps scoped snapshot failures scoped', async () => { - let snapshotCount = 0; - const device = createInteractionDevice(selectorSnapshot(), { - captureSnapshot: async (_context, options) => { - snapshotCount += 1; - if (options?.scope) throw new Error('scoped snapshot failed'); - return { snapshot: runtimeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }) }; - }, - scroll: async () => ({}), - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'bottom', - }), - (error: unknown) => - error instanceof AppError && - error.code === 'COMMAND_FAILED' && - /scoped container/i.test(error.message) && - error.details?.scope === 'Messages', - ); - assert.equal(snapshotCount, 2); -}); - -test('runtime swipe supports explicit and viewport-derived targets', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(selectorSnapshot(), { - swipe: async (_context, from, to, options) => { - calls.push({ from, to, durationMs: options?.durationMs }); - }, - }); - - const explicit = await device.interactions.swipe({ - from: selector('label=Continue'), - to: { x: 200, y: 40 }, - durationMs: 300, - session: 'default', - }); - const directional = await device.interactions.swipe({ - direction: 'left', - distance: 25, - session: 'default', - }); - - assert.deepEqual(explicit.from, { x: 60, y: 40 }); - assert.deepEqual(directional.from, { x: 60, y: 40 }); - assert.deepEqual(directional.to, { x: 35, y: 40 }); - assert.deepEqual(calls, [ - { from: { x: 60, y: 40 }, to: { x: 200, y: 40 }, durationMs: 300 }, - { from: { x: 60, y: 40 }, to: { x: 35, y: 40 }, durationMs: undefined }, - ]); -}); - -test('runtime directional swipe uses the visible viewport instead of off-screen content bounds', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(snapshotWithOffscreenContent(), { - swipe: async (_context, from, to) => { - calls.push({ from, to }); - }, - }); - - const result = await device.interactions.swipe({ - direction: 'left', - distance: 25, - session: 'default', - }); - - assert.deepEqual(result.from, { x: 50, y: 50 }); - assert.deepEqual(result.to, { x: 25, y: 50 }); - assert.deepEqual(calls, [{ from: { x: 50, y: 50 }, to: { x: 25, y: 50 } }]); -}); - -test('runtime gesture swipe presets use stable viewport lanes', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(snapshotWithOffscreenContent(), { - platform: 'android', - swipe: async (_context, from, to, options) => { - calls.push({ from, to, durationMs: options?.durationMs }); - }, - }); - - const pageSwipe = await device.interactions.swipe({ - preset: 'left', - durationMs: 300, - session: 'default', - }); - const edgeSwipe = await device.interactions.swipe({ - preset: 'right-edge', - durationMs: 350, - session: 'default', - }); - - assert.deepEqual(pageSwipe.from, { x: 85, y: 50 }); - assert.deepEqual(pageSwipe.to, { x: 15, y: 50 }); - assert.deepEqual(edgeSwipe.from, { x: 8, y: 50 }); - assert.deepEqual(edgeSwipe.to, { x: 85, y: 50 }); - assert.deepEqual(calls, [ - { from: { x: 85, y: 50 }, to: { x: 15, y: 50 }, durationMs: 300 }, - { from: { x: 8, y: 50 }, to: { x: 85, y: 50 }, durationMs: 350 }, - ]); -}); - -test('runtime iOS in-page swipe presets avoid edge-navigation lanes', async () => { - const calls: unknown[] = []; - const device = createInteractionDevice(snapshotWithOffscreenContent(), { - platform: 'ios', - swipe: async (_context, from, to, options) => { - calls.push({ from, to, durationMs: options?.durationMs }); - }, - }); - - const pageSwipe = await device.interactions.swipe({ - preset: 'right', - durationMs: 300, - session: 'default', - }); - - assert.deepEqual(pageSwipe.from, { x: 15, y: 50 }); - assert.deepEqual(pageSwipe.to, { x: 85, y: 50 }); - assert.deepEqual(calls, [{ from: { x: 15, y: 50 }, to: { x: 85, y: 50 }, durationMs: 300 }]); -}); - -test('runtime viewport gestures reject inspect-only macOS surfaces', async () => { - for (const surface of ['desktop', 'menubar'] as const) { - const device = createInteractionDevice(selectorSnapshot(), { - platform: 'macos', - sessionMetadata: { surface }, - scroll: async () => { - throw new Error(`${surface} scroll should be rejected before backend call`); - }, - swipe: async () => { - throw new Error(`${surface} swipe should be rejected before backend call`); - }, - pinch: async () => { - throw new Error(`${surface} pinch should be rejected before backend call`); - }, - }); - - await assert.rejects( - () => - device.interactions.scroll({ - direction: 'down', - target: { kind: 'viewport' }, - session: 'default', - }), - new RegExp(`scroll is not supported on macOS ${surface}`), - ); - await assert.rejects( - () => - device.interactions.swipe({ - direction: 'left', - session: 'default', - }), - new RegExp(`swipe is not supported on macOS ${surface}`), - ); - await assert.rejects( - () => - device.interactions.swipe({ - from: { x: 10, y: 20 }, - to: { x: 30, y: 20 }, - session: 'default', - }), - new RegExp(`swipe is not supported on macOS ${surface}`), - ); - await assert.rejects( - () => - device.interactions.pinch({ - scale: 1.2, - session: 'default', - }), - new RegExp(`pinch is not supported on macOS ${surface}`), - ); - } -}); - -test('runtime pinch is backend-gated and resolves optional center targets', async () => { - const calls: unknown[] = []; - const unsupported = createInteractionDevice(selectorSnapshot()); - await assert.rejects( - () => unsupported.interactions.pinch({ scale: 1.2 }), - /pinch is not supported/, - ); - - const device = createInteractionDevice(selectorSnapshot(), { - pinch: async (_context, options) => { - calls.push(options); - }, - }); - - const result = await device.interactions.pinch({ - scale: 0.8, - center: ref('@e1'), - session: 'default', - }); - - assert.equal(result.kind, 'pinch'); - assert.deepEqual(result.center, { x: 60, y: 40 }); - assert.deepEqual(calls, [{ scale: 0.8, center: { x: 60, y: 40 } }]); -}); - -test('runtime interaction commands are available from the command namespace', async () => { - const device = createInteractionDevice(selectorSnapshot(), { - tap: async () => {}, - }); - - const result = await commands.interactions.click(device, { - session: 'default', - target: selector('label=Continue'), - }); - - assert.equal(result.kind, 'selector'); -}); - test('coordinate tap with out-of-bounds point warns when session has viewport', async () => { const device = createInteractionDevice( makeSnapshotState([ @@ -1457,360 +619,3 @@ test('coordinate tap with no session snapshot has no warning', async () => { assert.equal(result.kind, 'point'); assert.equal('warning' in result, false); }); - -function selectorSnapshot(): SnapshotState { - return makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Button', - label: 'Continue', - value: 'Continue', - rect: { x: 10, y: 20, width: 100, height: 40 }, - hittable: true, - }, - ]); -} - -// Closed-drawer shape shared by the native-ref preflight tests: the only -// interactive node (@e2) sits fully left of the Application viewport. -function offscreenDrawerSnapshot(): SnapshotState { - return makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Application', - rect: { x: 0, y: 0, width: 400, height: 800 }, - hittable: true, - }, - { - index: 1, - depth: 2, - parentIndex: 0, - type: 'Button', - label: 'Explore', - rect: { x: -320, y: 240, width: 300, height: 50 }, - hittable: true, - }, - ]); -} - -function runtimeScrollSnapshot(options: { hiddenBelow: boolean; message?: string }): SnapshotState { - return makeSnapshotState([ - { - index: 1, - depth: 0, - type: 'ScrollView', - label: 'Messages', - hiddenContentBelow: options.hiddenBelow ? true : undefined, - rect: { x: 0, y: 100, width: 400, height: 600 }, - }, - { - index: 2, - depth: 1, - parentIndex: 1, - type: 'Button', - label: options.message ?? 'Latest message', - rect: { x: 0, y: 640, width: 400, height: 56 }, - hittable: true, - }, - ]); -} - -function fillableSnapshot(): SnapshotState { - return makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'XCUIElementTypeTextField', - label: 'Email', - rect: { x: 20, y: 10, width: 60, height: 40 }, - hittable: true, - }, - ]); -} - -function iosTabBarSnapshot(): SnapshotState { - return makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'XCUIElementTypeApplication', - label: 'TabRepro', - rect: { x: 0, y: 0, width: 402, height: 874 }, - hittable: false, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeTabBar', - rect: { x: 0, y: 791, width: 402, height: 83 }, - hittable: true, - }, - { - index: 2, - depth: 2, - parentIndex: 1, - type: 'XCUIElementTypeButton', - label: 'Home', - rect: { x: 30, y: 800, width: 91, height: 44 }, - hittable: false, - }, - { - index: 3, - depth: 2, - parentIndex: 1, - type: 'XCUIElementTypeButton', - label: 'Library', - rect: { x: 120, y: 800, width: 92, height: 44 }, - hittable: false, - }, - { - index: 4, - depth: 2, - parentIndex: 1, - type: 'XCUIElementTypeButton', - label: 'Settings', - rect: { x: 211, y: 800, width: 91, height: 44 }, - hittable: false, - }, - { - index: 5, - depth: 2, - parentIndex: 1, - type: 'XCUIElementTypeButton', - label: 'Search', - rect: { x: 304, y: 800, width: 92, height: 44 }, - hittable: false, - }, - ]); -} - -function coveredByTabBarSnapshot(): SnapshotState { - return makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Application', - label: 'Example', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Save draft', - rect: { x: 16, y: 790, width: 140, height: 44 }, - hittable: false, - interactionBlocked: 'covered', - presentationHints: ['covered'], - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'TabBar', - rect: { x: 0, y: 760, width: 390, height: 84 }, - hittable: true, - }, - ]); -} - -function duplicateCoveredLabelSnapshot(): SnapshotState { - return makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Application', - label: 'Example', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Save draft', - rect: { x: 16, y: 120, width: 140, height: 44 }, - hittable: true, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Save draft', - rect: { x: 16, y: 790, width: 140, height: 44 }, - hittable: false, - interactionBlocked: 'covered', - presentationHints: ['covered'], - }, - { - index: 3, - depth: 1, - parentIndex: 0, - type: 'TabBar', - rect: { x: 0, y: 760, width: 390, height: 84 }, - hittable: true, - }, - ]); -} - -function nonHittableCellSnapshot(): SnapshotState { - return makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'XCUIElementTypeOther', - label: 'Settings list', - rect: { x: 10, y: 20, width: 300, height: 80 }, - hittable: true, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeCell', - label: 'Account', - rect: { x: 20, y: 10, width: 100, height: 40 }, - hittable: false, - }, - ]); -} - -// Mirrors the #1037 Maps repro: a non-hittable map-pin annotation exact-matches -// `text="Anthropic - Headquarters"` while the real, longer-labeled row is a -// separate node. Fill/press must still proceed against the unique match (no -// stricter resolution) but should flag it as likely non-actionable. -function mapPinAnnotationSnapshot(): SnapshotState { - return makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Other', - label: 'Anthropic - Headquarters', - rect: { x: 177, y: 446, width: 30, height: 30 }, - hittable: false, - }, - { - index: 1, - depth: 1, - type: 'Button', - label: 'Anthropic - Headquarters, 548 Market St', - rect: { x: 0, y: 786, width: 390, height: 60 }, - hittable: false, - }, - ]); -} - -function nonTouchableGroupSnapshot(): SnapshotState { - return makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'XCUIElementTypeOther', - label: 'Clickable group', - rect: { x: 10, y: 20, width: 300, height: 80 }, - hittable: true, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeOther', - label: 'Decorative group', - rect: { x: 30, y: 40, width: 60, height: 20 }, - hittable: false, - }, - ]); -} - -function snapshotWithOffscreenContent(): SnapshotState { - return makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Application', - label: 'Example', - rect: { x: 0, y: 0, width: 100, height: 100 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Visible', - rect: { x: 10, y: 10, width: 20, height: 20 }, - hittable: true, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Offscreen', - rect: { x: 10, y: 900, width: 20, height: 20 }, - hittable: true, - }, - ]); -} - -function createInteractionDevice( - snapshot: SnapshotState, - overrides: Partial< - Pick< - AgentDeviceBackend, - | 'captureSnapshot' - | 'tap' - | 'tapTarget' - | 'fill' - | 'fillTarget' - | 'typeText' - | 'focus' - | 'longPress' - | 'scroll' - | 'swipe' - | 'pinch' - > - > & { - platform?: AgentDeviceBackend['platform']; - sessionMetadata?: Record; - } = {}, -) { - return createAgentDevice({ - backend: { - platform: overrides.platform ?? 'ios', - captureSnapshot: async (...args) => - overrides.captureSnapshot ? await overrides.captureSnapshot(...args) : { snapshot }, - tap: async (...args) => await overrides.tap?.(...args), - tapTarget: overrides.tapTarget - ? async (...args) => await overrides.tapTarget?.(...args) - : undefined, - fill: async (...args) => await overrides.fill?.(...args), - fillTarget: overrides.fillTarget - ? async (...args) => await overrides.fillTarget?.(...args) - : undefined, - typeText: async (...args) => await overrides.typeText?.(...args), - focus: overrides.focus ? async (...args) => await overrides.focus?.(...args) : undefined, - longPress: overrides.longPress - ? async (...args) => await overrides.longPress?.(...args) - : undefined, - scroll: overrides.scroll ? async (...args) => await overrides.scroll?.(...args) : undefined, - swipe: overrides.swipe ? async (...args) => await overrides.swipe?.(...args) : undefined, - pinch: overrides.pinch ? async (...args) => await overrides.pinch?.(...args) : undefined, - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions: createMemorySessionStore([ - { name: 'default', snapshot, metadata: overrides.sessionMetadata }, - ]), - policy: localCommandPolicy(), - }); -} - -async function clickRefE2(device: ReturnType) { - return await device.interactions.click(ref('@e2'), { - session: 'default', - }); -} diff --git a/src/commands/interaction/runtime/resolution.test.ts b/src/commands/interaction/runtime/resolution.test.ts new file mode 100644 index 000000000..90948bc7e --- /dev/null +++ b/src/commands/interaction/runtime/resolution.test.ts @@ -0,0 +1,443 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { BackendSnapshotOptions } from '../../../backend.ts'; +import { ref, selector } from '../../index.ts'; +import { resolveActionableTouchResolution } from '../../../core/interaction-targeting.ts'; +import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts'; +import type { Point } from '../../../kernel/snapshot.ts'; +import { + clickRefE2, + coveredByTabBarSnapshot, + createInteractionDevice, + duplicateCoveredLabelSnapshot, + fillableSnapshot, + iosTabBarSnapshot, + mapPinAnnotationSnapshot, + nonHittableCellSnapshot, + nonTouchableGroupSnapshot, + selectorSnapshot, +} from './__tests__/test-utils/index.ts'; + +test('runtime press resolves selector targets to the actionable node center', async () => { + const calls: Point[] = []; + const device = createInteractionDevice(selectorSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + return { ok: true }; + }, + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + }); + + assert.deepEqual(calls, [{ x: 60, y: 40 }]); + assert.equal(result.kind, 'selector'); + assert.deepEqual(result.target, { kind: 'selector', selector: 'label=Continue' }); + assert.equal(result.node?.label, 'Continue'); + assert.deepEqual(result.selectorChain, [ + 'role="button" label="Continue"', + 'label="Continue"', + 'value="Continue"', + ]); + assert.deepEqual(result.backendResult, { ok: true }); +}); + +test('runtime selector interactions fall back to a full snapshot when interactive refresh misses', async () => { + const calls: Point[] = []; + const captureOptions: Array = []; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async (_context, options) => { + captureOptions.push(options); + return { + snapshot: options?.interactiveOnly + ? makeSnapshotState([]) + : makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeCell', + label: 'General', + rect: { x: 0, y: 100, width: 320, height: 44 }, + hittable: true, + }, + ]), + }; + }, + tap: async (_context, point) => { + calls.push(point); + }, + }); + + const result = await device.interactions.click(selector('label=General'), { + session: 'default', + }); + + assert.equal(result.kind, 'selector'); + assert.equal(result.node?.label, 'General'); + assert.deepEqual(calls, [{ x: 160, y: 122 }]); + assert.deepEqual(captureOptions, [ + { interactiveOnly: true, includeRects: true }, + { interactiveOnly: false, includeRects: true }, + ]); +}); + +test('runtime press refuses a selector that resolves to an off-screen element', async () => { + // Closed-drawer shape: the only match sits fully left of the viewport. The + // @ref path already refuses this; the selector path must not silently tap + // out-of-viewport coordinates. + const offscreenSnapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 400, height: 800 }, + hittable: true, + }, + { + index: 1, + depth: 2, + parentIndex: 0, + type: 'Button', + label: 'Explore', + rect: { x: -320, y: 240, width: 300, height: 50 }, + hittable: true, + }, + ]); + const taps: unknown[] = []; + const device = createInteractionDevice(offscreenSnapshot, { + tap: async (_context, point) => { + taps.push(point); + }, + }); + + await assert.rejects( + () => device.interactions.press(selector('label=Explore'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.match(error.message, /off-screen element and is not safe to press/); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'offscreen_selector'); + assert.ok(typeof details?.hint === 'string'); + return true; + }, + ); + assert.equal(taps.length, 0); +}); + +test('runtime click keeps distinct tab button centers when iOS reports the tab bar as hittable', async () => { + const calls: Point[] = []; + const device = createInteractionDevice(iosTabBarSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + }, + }); + + const refResult = await device.interactions.click(ref('@e4'), { + session: 'default', + }); + const selectorResult = await device.interactions.click(selector('label=Settings'), { + session: 'default', + }); + + assert.deepEqual(calls, [ + { x: 166, y: 822 }, + { x: 257, y: 822 }, + ]); + assert.equal(refResult.kind, 'ref'); + assert.equal(refResult.node?.label, 'Library'); + assert.equal(selectorResult.kind, 'selector'); + assert.equal(selectorResult.node?.label, 'Settings'); +}); + +test('runtime click rejects refs covered by floating overlays', async () => { + const calls: Point[] = []; + const device = createInteractionDevice(coveredByTabBarSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + }, + }); + + await assert.rejects( + () => device.interactions.click(ref('@e2'), { session: 'default' }), + /Ref @e2 is covered by another visible element/, + ); + assert.deepEqual(calls, []); +}); + +test('runtime selector interactions skip covered matches when an uncovered duplicate exists', async () => { + const calls: Point[] = []; + const device = createInteractionDevice(duplicateCoveredLabelSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + }, + }); + + const result = await device.interactions.click(selector('label="Save draft"'), { + session: 'default', + }); + + assert.equal(result.kind, 'selector'); + assert.equal(result.node?.ref, 'e2'); + assert.deepEqual(calls, [{ x: 86, y: 142 }]); +}); + +test('runtime click keeps non-button semantic targets at their own center', async () => { + const calls: Point[] = []; + const device = createInteractionDevice(nonHittableCellSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + }, + }); + + const result = await clickRefE2(device); + + assert.deepEqual(calls, [{ x: 70, y: 30 }]); + assert.equal(result.kind, 'ref'); + assert.equal(result.node?.label, 'Account'); +}); + +test('runtime press surfaces targetHittable and a hint when the final tap node is non-hittable (#1037)', async () => { + const calls: Point[] = []; + const device = createInteractionDevice(nonHittableCellSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + }, + }); + + const result = await device.interactions.press(ref('@e2'), { session: 'default' }); + + // Press still proceeds and reports success — non-hittable is informational only. + assert.deepEqual(calls, [{ x: 70, y: 30 }]); + assert.equal(result.kind, 'ref'); + assert.equal(result.node?.label, 'Account'); + assert.equal(result.targetHittable, false); + assert.match(result.hint ?? '', /hittable: false/); + assert.match(result.hint ?? '', /@ref/); +}); + +test('runtime press omits targetHittable and hint when the resolved node is hittable', async () => { + const device = createInteractionDevice(selectorSnapshot(), { + tap: async () => {}, + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + }); + + assert.equal(result.kind, 'selector'); + assert.equal(result.targetHittable, undefined); + assert.equal(result.hint, undefined); +}); + +test('runtime fill surfaces targetHittable and a hint for a non-hittable selector match (Maps pin case, #1037)', async () => { + const calls: Array<{ point: Point; text: string }> = []; + const device = createInteractionDevice(mapPinAnnotationSnapshot(), { + fill: async (_context, point, text) => { + calls.push({ point, text }); + }, + }); + + const result = await device.interactions.fill( + selector('text="Anthropic - Headquarters"'), + 'ignored', + { session: 'default' }, + ); + + assert.equal(result.kind, 'selector'); + assert.equal(result.node?.label, 'Anthropic - Headquarters'); + assert.equal(result.targetHittable, false); + assert.match(result.hint ?? '', /hittable: false/); + assert.deepEqual(calls, [{ point: { x: 192, y: 461 }, text: 'ignored' }]); +}); + +test('runtime click still promotes non-touchable nodes to hittable ancestors', async () => { + const calls: Point[] = []; + const device = createInteractionDevice(nonTouchableGroupSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + }, + }); + + const result = await clickRefE2(device); + + assert.deepEqual(calls, [{ x: 160, y: 60 }]); + assert.equal(result.kind, 'ref'); + assert.equal(result.node?.label, 'Clickable group'); +}); + +test('touch resolution promotes static text inside a hittable row to the row', () => { + const snapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeCell', + label: 'Account row', + rect: { x: 10, y: 20, width: 300, height: 60 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeStaticText', + label: 'Account', + rect: { x: 24, y: 32, width: 80, height: 20 }, + hittable: false, + }, + ]); + + const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[1]!); + + assert.equal(resolution.reason, 'hittable-ancestor'); + assert.equal(resolution.node.label, 'Account row'); +}); + +test('touch resolution prefers same-rect hittable descendants over semantic targets', () => { + const snapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeButton', + label: 'Profile', + rect: { x: 30, y: 40, width: 120, height: 50 }, + hittable: false, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeImage', + identifier: 'profile-hit-area', + rect: { x: 30, y: 40, width: 120, height: 50 }, + hittable: true, + }, + ]); + + const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[0]!); + + assert.equal(resolution.reason, 'same-rect-descendant'); + assert.equal(resolution.node.identifier, 'profile-hit-area'); +}); + +test('touch resolution prevents full-screen window-like ancestors from stealing taps', () => { + const snapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeApplication', + label: 'Example', + rect: { x: 0, y: 0, width: 390, height: 844 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeStaticText', + label: 'Status', + rect: { x: 24, y: 72, width: 80, height: 24 }, + hittable: false, + }, + ]); + + const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[1]!); + + assert.equal(resolution.reason, 'overly-broad-ancestor'); + assert.equal(resolution.node.label, 'Status'); +}); + +test('touch resolution falls back to the original node when no usable touch target exists', () => { + const snapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeOther', + label: 'Virtual item', + hittable: false, + }, + ]); + + const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[0]!); + + assert.equal(resolution.reason, 'original'); + assert.equal(resolution.node.label, 'Virtual item'); +}); + +test('runtime interactions reject unsupported macOS desktop and menubar surfaces', async () => { + const desktop = createInteractionDevice(selectorSnapshot(), { + platform: 'macos', + sessionMetadata: { surface: 'desktop' }, + tap: async () => { + throw new Error('desktop click should be rejected before backend tap'); + }, + }); + await assert.rejects( + () => desktop.interactions.click({ kind: 'point', x: 1, y: 2 }, { session: 'default' }), + /click is not supported on macOS desktop sessions yet/, + ); + await assert.rejects( + () => + desktop.interactions.click( + { kind: 'point', x: 1, y: 2 }, + { session: 'default', metadata: { surface: 'app' } }, + ), + /click is not supported on macOS desktop sessions yet/, + ); + + const menubar = createInteractionDevice(fillableSnapshot(), { + platform: 'macos', + sessionMetadata: { surface: 'menubar' }, + fill: async () => { + throw new Error('menubar fill should be rejected before backend fill'); + }, + }); + await assert.rejects( + () => menubar.interactions.fill(ref('@e1'), 'hello', { session: 'default' }), + /fill is not supported on macOS menubar sessions yet/, + ); + + let pressed = false; + const menubarPress = createInteractionDevice(fillableSnapshot(), { + platform: 'macos', + sessionMetadata: { surface: 'menubar' }, + tap: async () => { + pressed = true; + }, + }); + + await menubarPress.interactions.press(ref('@e1'), { session: 'default' }); + + assert.equal(pressed, true); +}); + +test('runtime ref interactions refresh the snapshot when a stored ref has no usable rect', async () => { + const staleSnapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Continue', + hittable: true, + }, + ]); + const freshSnapshot = selectorSnapshot(); + const calls: Point[] = []; + let captures = 0; + const device = createInteractionDevice(staleSnapshot, { + captureSnapshot: async () => { + captures += 1; + return { snapshot: freshSnapshot }; + }, + tap: async (_context, point) => { + calls.push(point); + }, + }); + + const result = await device.interactions.click(ref('@e1'), { session: 'default' }); + + assert.equal(captures, 1); + assert.deepEqual(calls, [{ x: 60, y: 40 }]); + assert.equal(result.kind, 'ref'); + assert.equal(result.node?.rect?.width, 100); +}); diff --git a/src/commands/interaction/runtime/selector-read-stable.test.ts b/src/commands/interaction/runtime/selector-read-stable.test.ts new file mode 100644 index 000000000..1a7997f9c --- /dev/null +++ b/src/commands/interaction/runtime/selector-read-stable.test.ts @@ -0,0 +1,270 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { AgentDeviceBackend, BackendSnapshotResult } from '../../../backend.ts'; +import { createLocalArtifactAdapter } from '../../../io.ts'; +import { + createAgentDevice, + createMemorySessionStore, + localCommandPolicy, +} from '../../../runtime.ts'; +import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts'; +import { createFakeClock, selectorReadSnapshot } from './__tests__/test-utils/index.ts'; + +test('runtime wait stable settles after two unchanged captures', async () => { + const snapshot = selectorReadSnapshot(); + const clock = createFakeClock(); + let captures = 0; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => { + captures += 1; + return { snapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + clock, + }); + + const result = await device.selectors.wait({ + session: 'default', + target: { kind: 'stable', quietMs: 500, timeoutMs: 10_000 }, + }); + + // Poll cadence is 300ms; a 500ms quiet window needs a 3rd identical capture to + // accumulate enough elapsed time since the first (baseline) capture. + assert.equal(result.kind, 'stable'); + if (result.kind === 'stable') { + assert.equal(result.captures, 3); + assert.equal(result.nodeCount, snapshot.nodes.length); + } + assert.equal(captures, 3); +}); + +test('runtime wait stable hints when it settles on a nearly-empty tree', async () => { + const tinySnapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'One', + rect: { x: 0, y: 0, width: 10, height: 10 }, + }, + { + index: 1, + depth: 0, + type: 'Button', + label: 'Two', + rect: { x: 0, y: 20, width: 10, height: 10 }, + }, + { + index: 2, + depth: 0, + type: 'Button', + label: 'Three', + rect: { x: 0, y: 40, width: 10, height: 10 }, + }, + ]); + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => ({ snapshot: tinySnapshot }), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot: tinySnapshot }]), + policy: localCommandPolicy(), + clock: createFakeClock(), + }); + + const result = await device.selectors.wait({ + session: 'default', + target: { kind: 'stable', quietMs: 500, timeoutMs: 10_000 }, + }); + + assert.equal(result.kind, 'stable'); + if (result.kind === 'stable') { + assert.equal(result.nodeCount, 3); + assert.equal( + result.hint, + 'Settled on a nearly-empty tree — the app may still be loading. Wait for specific content (wait text ...) before interacting.', + ); + } +}); + +test('runtime wait stable omits the loading hint for a normal-sized tree', async () => { + const normalSnapshot = makeSnapshotState( + Array.from({ length: 6 }, (_, index) => ({ + index, + depth: 0, + type: 'Button', + label: `Item ${index}`, + rect: { x: 0, y: index * 20, width: 10, height: 10 }, + })), + ); + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => ({ snapshot: normalSnapshot }), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot: normalSnapshot }]), + policy: localCommandPolicy(), + clock: createFakeClock(), + }); + + const result = await device.selectors.wait({ + session: 'default', + target: { kind: 'stable', quietMs: 500, timeoutMs: 10_000 }, + }); + + assert.equal(result.kind, 'stable'); + if (result.kind === 'stable') { + assert.equal(result.nodeCount, 6); + assert.equal('hint' in result, false); + } +}); + +test('runtime wait stable requires quiet captures after instability before settling', async () => { + const snapshot = selectorReadSnapshot(); + const changedSnapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Loading', + rect: { x: 0, y: 0, width: 1, height: 1 }, + }, + ]); + const clock = createFakeClock(); + let captures = 0; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => { + captures += 1; + // First capture unstable/changed relative to the initial nothing-seen-yet state, + // second capture changes again (still unstable), then two identical captures settle. + if (captures <= 2) return { snapshot: captures === 1 ? snapshot : changedSnapshot }; + return { snapshot: changedSnapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + clock, + }); + + const result = await device.selectors.wait({ + session: 'default', + target: { kind: 'stable', quietMs: 500, timeoutMs: 10_000 }, + }); + + assert.equal(result.kind, 'stable'); + // capture 1: baseline; capture 2: changed (reset quiet window); capture 3+4: unchanged -> settle. + assert.equal(captures, 4); +}); + +test('runtime wait stable times out with capture stats when never settling', async () => { + const clock = createFakeClock(); + let captures = 0; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => { + captures += 1; + return { + snapshot: makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: `Item ${captures}`, + rect: { x: 0, y: 0, width: 1, height: 1 }, + }, + ]), + }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default' }]), + policy: localCommandPolicy(), + clock, + }); + + await assert.rejects( + () => + device.selectors.wait({ + session: 'default', + target: { kind: 'stable', quietMs: 500, timeoutMs: 1_000 }, + }), + (error: unknown) => + error instanceof Error && + error.message === 'wait timed out waiting for a stable UI' && + (error as { details?: { reason?: string; captures?: number } }).details?.reason === + 'wait_stable_timeout' && + typeof (error as { details?: { captures?: number } }).details?.captures === 'number', + ); + assert.ok(captures > 1); +}); + +test('runtime wait stable times out when a backend capture stalls past the wait budget', async () => { + const clock = createFakeClock(); + const device = createAgentDevice({ + backend: { + platform: 'macos', + // Simulates the stalled macOS AX capture: the promise never settles, so + // only the real-timer deadline can end the wait. + captureSnapshot: () => new Promise(() => {}), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default' }]), + policy: localCommandPolicy(), + clock, + }); + + await assert.rejects( + () => + device.selectors.wait({ + session: 'default', + target: { kind: 'stable', quietMs: 100, timeoutMs: 250 }, + }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal(error.message, 'wait timed out waiting for a stable UI'); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'wait_stable_timeout'); + assert.equal(details?.captureStalled, true); + assert.equal(details?.captures, 0); + return true; + }, + ); +}); + +test('runtime wait stable uses provided defaults when quietMs/timeoutMs are omitted', async () => { + const snapshot = selectorReadSnapshot(); + const clock = createFakeClock(); + let captures = 0; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => { + captures += 1; + return { snapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + clock, + }); + + const result = await device.selectors.wait({ + session: 'default', + target: { kind: 'stable' }, + }); + + assert.equal(result.kind, 'stable'); + assert.equal(captures, 3); +}); diff --git a/src/commands/interaction/runtime/selector-read.test.ts b/src/commands/interaction/runtime/selector-read.test.ts index 49e0d3658..6142ffd4d 100644 --- a/src/commands/interaction/runtime/selector-read.test.ts +++ b/src/commands/interaction/runtime/selector-read.test.ts @@ -1,10 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import type { - AgentDeviceBackend, - BackendSnapshotOptions, - BackendSnapshotResult, -} from '../../../backend.ts'; +import type { AgentDeviceBackend, BackendSnapshotOptions } from '../../../backend.ts'; import { createLocalArtifactAdapter } from '../../../io.ts'; import { createAgentDevice, @@ -13,11 +9,11 @@ import { type CommandSessionStore, } from '../../../runtime.ts'; import { ref, selector } from '../../index.ts'; -import type { SnapshotState } from '../../../kernel/snapshot.ts'; import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts'; +import { createSelectorDevice, selectorReadSnapshot } from './__tests__/test-utils/index.ts'; test('runtime get reads text from a selector target', async () => { - const snapshot = selectorSnapshot(); + const snapshot = selectorReadSnapshot(); const device = createSelectorDevice(snapshot, { readText: 'Backend expanded text', }); @@ -40,7 +36,7 @@ test('runtime get reads text from a selector target', async () => { }); test('runtime get selector target captures fresh snapshot without a stored session snapshot', async () => { - const snapshot = selectorSnapshot(); + const snapshot = selectorReadSnapshot(); const sessions = createMemorySessionStore([{ name: 'default' }]); let captures = 0; const device = createAgentDevice({ @@ -68,7 +64,7 @@ test('runtime get selector target captures fresh snapshot without a stored sessi }); test('runtime get returns attrs for a ref target without recapturing', async () => { - const snapshot = selectorSnapshot(); + const snapshot = selectorReadSnapshot(); let captures = 0; const device = createSelectorDevice(snapshot, { captureSnapshot: () => { @@ -90,7 +86,7 @@ test('runtime get returns attrs for a ref target without recapturing', async () }); test('runtime selectors pass runtime signal to backend snapshot capture', async () => { - const snapshot = selectorSnapshot(); + const snapshot = selectorReadSnapshot(); const controller = new AbortController(); let signal: AbortSignal | undefined; const device = createAgentDevice({ @@ -116,7 +112,7 @@ test('runtime selectors pass runtime signal to backend snapshot capture', async }); test('runtime selectors forward public snapshot options to backend capture', async () => { - const snapshot = selectorSnapshot(); + const snapshot = selectorReadSnapshot(); let captureOptions: BackendSnapshotOptions | undefined; const device = createAgentDevice({ backend: { @@ -150,7 +146,7 @@ test('runtime selectors forward public snapshot options to backend capture', asy }); test('runtime visibility predicates request snapshot rects', async () => { - const snapshot = selectorSnapshot(); + const snapshot = selectorReadSnapshot(); let captureOptions: BackendSnapshotOptions | undefined; const device = createAgentDevice({ backend: { @@ -173,7 +169,7 @@ test('runtime visibility predicates request snapshot rects', async () => { }); test('runtime is validates selector predicates', async () => { - const device = createSelectorDevice(selectorSnapshot()); + const device = createSelectorDevice(selectorReadSnapshot()); const result = await device.selectors.is({ session: 'default', @@ -191,7 +187,7 @@ test('runtime is validates selector predicates', async () => { }); test('runtime find get_text reads the matched node', async () => { - const device = createSelectorDevice(selectorSnapshot(), { + const device = createSelectorDevice(selectorReadSnapshot(), { readText: 'Continue', }); @@ -209,7 +205,7 @@ test('runtime find get_text reads the matched node', async () => { }); test('runtime find accepts selector expression queries', async () => { - const device = createSelectorDevice(selectorSnapshot()); + const device = createSelectorDevice(selectorReadSnapshot()); const result = await device.selectors.find({ session: 'default', @@ -221,7 +217,7 @@ test('runtime find accepts selector expression queries', async () => { }); test('runtime web find text does not pass locator text as browser selector scope', async () => { - const snapshot = selectorSnapshot(); + const snapshot = selectorReadSnapshot(); let captureOptions: BackendSnapshotOptions | undefined; const device = createAgentDevice({ backend: { @@ -248,7 +244,7 @@ test('runtime web find text does not pass locator text as browser selector scope }); test('runtime find wait reports sparse snapshot verdicts on the selector-read route', async () => { - const initialSnapshot = selectorSnapshot(); + const initialSnapshot = selectorReadSnapshot(); const session = { name: 'default', snapshot: initialSnapshot }; const sessions = { get: () => session, @@ -303,7 +299,7 @@ test('runtime find wait reports sparse snapshot verdicts on the selector-read ro }); test('runtime wait can use backend text search', async () => { - const device = createSelectorDevice(selectorSnapshot(), { + const device = createSelectorDevice(selectorReadSnapshot(), { findText: true, now: 10, }); @@ -317,7 +313,7 @@ test('runtime wait can use backend text search', async () => { }); test('runtime selector convenience methods use explicit target helpers', async () => { - const device = createSelectorDevice(selectorSnapshot(), { + const device = createSelectorDevice(selectorReadSnapshot(), { readText: 'Continue', findText: true, }); @@ -337,322 +333,3 @@ test('runtime selector convenience methods use explicit target helpers', async ( assert.equal(visible.pass, true); assert.deepEqual(waited, { kind: 'text', text: 'Ready', waitedMs: 0 }); }); - -test('runtime wait stable settles after two unchanged captures', async () => { - const snapshot = selectorSnapshot(); - const clock = createFakeClock(); - let captures = 0; - const device = createAgentDevice({ - backend: { - platform: 'ios', - captureSnapshot: async () => { - captures += 1; - return { snapshot }; - }, - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions: createMemorySessionStore([{ name: 'default', snapshot }]), - policy: localCommandPolicy(), - clock, - }); - - const result = await device.selectors.wait({ - session: 'default', - target: { kind: 'stable', quietMs: 500, timeoutMs: 10_000 }, - }); - - // Poll cadence is 300ms; a 500ms quiet window needs a 3rd identical capture to - // accumulate enough elapsed time since the first (baseline) capture. - assert.equal(result.kind, 'stable'); - if (result.kind === 'stable') { - assert.equal(result.captures, 3); - assert.equal(result.nodeCount, snapshot.nodes.length); - } - assert.equal(captures, 3); -}); - -test('runtime wait stable hints when it settles on a nearly-empty tree', async () => { - const tinySnapshot = makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Button', - label: 'One', - rect: { x: 0, y: 0, width: 10, height: 10 }, - }, - { - index: 1, - depth: 0, - type: 'Button', - label: 'Two', - rect: { x: 0, y: 20, width: 10, height: 10 }, - }, - { - index: 2, - depth: 0, - type: 'Button', - label: 'Three', - rect: { x: 0, y: 40, width: 10, height: 10 }, - }, - ]); - const device = createAgentDevice({ - backend: { - platform: 'ios', - captureSnapshot: async () => ({ snapshot: tinySnapshot }), - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions: createMemorySessionStore([{ name: 'default', snapshot: tinySnapshot }]), - policy: localCommandPolicy(), - clock: createFakeClock(), - }); - - const result = await device.selectors.wait({ - session: 'default', - target: { kind: 'stable', quietMs: 500, timeoutMs: 10_000 }, - }); - - assert.equal(result.kind, 'stable'); - if (result.kind === 'stable') { - assert.equal(result.nodeCount, 3); - assert.equal( - result.hint, - 'Settled on a nearly-empty tree — the app may still be loading. Wait for specific content (wait text ...) before interacting.', - ); - } -}); - -test('runtime wait stable omits the loading hint for a normal-sized tree', async () => { - const normalSnapshot = makeSnapshotState( - Array.from({ length: 6 }, (_, index) => ({ - index, - depth: 0, - type: 'Button', - label: `Item ${index}`, - rect: { x: 0, y: index * 20, width: 10, height: 10 }, - })), - ); - const device = createAgentDevice({ - backend: { - platform: 'ios', - captureSnapshot: async () => ({ snapshot: normalSnapshot }), - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions: createMemorySessionStore([{ name: 'default', snapshot: normalSnapshot }]), - policy: localCommandPolicy(), - clock: createFakeClock(), - }); - - const result = await device.selectors.wait({ - session: 'default', - target: { kind: 'stable', quietMs: 500, timeoutMs: 10_000 }, - }); - - assert.equal(result.kind, 'stable'); - if (result.kind === 'stable') { - assert.equal(result.nodeCount, 6); - assert.equal('hint' in result, false); - } -}); - -test('runtime wait stable requires quiet captures after instability before settling', async () => { - const snapshot = selectorSnapshot(); - const changedSnapshot = makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Button', - label: 'Loading', - rect: { x: 0, y: 0, width: 1, height: 1 }, - }, - ]); - const clock = createFakeClock(); - let captures = 0; - const device = createAgentDevice({ - backend: { - platform: 'ios', - captureSnapshot: async () => { - captures += 1; - // First capture unstable/changed relative to the initial nothing-seen-yet state, - // second capture changes again (still unstable), then two identical captures settle. - if (captures <= 2) return { snapshot: captures === 1 ? snapshot : changedSnapshot }; - return { snapshot: changedSnapshot }; - }, - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions: createMemorySessionStore([{ name: 'default', snapshot }]), - policy: localCommandPolicy(), - clock, - }); - - const result = await device.selectors.wait({ - session: 'default', - target: { kind: 'stable', quietMs: 500, timeoutMs: 10_000 }, - }); - - assert.equal(result.kind, 'stable'); - // capture 1: baseline; capture 2: changed (reset quiet window); capture 3+4: unchanged -> settle. - assert.equal(captures, 4); -}); - -test('runtime wait stable times out with capture stats when never settling', async () => { - const clock = createFakeClock(); - let captures = 0; - const device = createAgentDevice({ - backend: { - platform: 'ios', - captureSnapshot: async () => { - captures += 1; - return { - snapshot: makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Button', - label: `Item ${captures}`, - rect: { x: 0, y: 0, width: 1, height: 1 }, - }, - ]), - }; - }, - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions: createMemorySessionStore([{ name: 'default' }]), - policy: localCommandPolicy(), - clock, - }); - - await assert.rejects( - () => - device.selectors.wait({ - session: 'default', - target: { kind: 'stable', quietMs: 500, timeoutMs: 1_000 }, - }), - (error: unknown) => - error instanceof Error && - error.message === 'wait timed out waiting for a stable UI' && - (error as { details?: { reason?: string; captures?: number } }).details?.reason === - 'wait_stable_timeout' && - typeof (error as { details?: { captures?: number } }).details?.captures === 'number', - ); - assert.ok(captures > 1); -}); - -test('runtime wait stable times out when a backend capture stalls past the wait budget', async () => { - const clock = createFakeClock(); - const device = createAgentDevice({ - backend: { - platform: 'macos', - // Simulates the stalled macOS AX capture: the promise never settles, so - // only the real-timer deadline can end the wait. - captureSnapshot: () => new Promise(() => {}), - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions: createMemorySessionStore([{ name: 'default' }]), - policy: localCommandPolicy(), - clock, - }); - - await assert.rejects( - () => - device.selectors.wait({ - session: 'default', - target: { kind: 'stable', quietMs: 100, timeoutMs: 250 }, - }), - (error: unknown) => { - assert.ok(error instanceof Error); - assert.equal(error.message, 'wait timed out waiting for a stable UI'); - const details = (error as { details?: Record }).details; - assert.equal(details?.reason, 'wait_stable_timeout'); - assert.equal(details?.captureStalled, true); - assert.equal(details?.captures, 0); - return true; - }, - ); -}); - -test('runtime wait stable uses provided defaults when quietMs/timeoutMs are omitted', async () => { - const snapshot = selectorSnapshot(); - const clock = createFakeClock(); - let captures = 0; - const device = createAgentDevice({ - backend: { - platform: 'ios', - captureSnapshot: async () => { - captures += 1; - return { snapshot }; - }, - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions: createMemorySessionStore([{ name: 'default', snapshot }]), - policy: localCommandPolicy(), - clock, - }); - - const result = await device.selectors.wait({ - session: 'default', - target: { kind: 'stable' }, - }); - - assert.equal(result.kind, 'stable'); - assert.equal(captures, 3); -}); - -function createFakeClock(stepMs = 300): { - now: () => number; - sleep: (ms: number) => Promise; -} { - let elapsed = 0; - return { - now: () => elapsed, - sleep: async (ms: number) => { - elapsed += ms > 0 ? ms : stepMs; - }, - }; -} - -function selectorSnapshot(): SnapshotState { - return makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Button', - label: 'Continue', - value: 'Continue', - rect: { x: 10, y: 20, width: 100, height: 40 }, - }, - ]); -} - -function createSelectorDevice( - snapshot: SnapshotState, - options: { - readText?: string; - findText?: boolean; - now?: number; - captureSnapshot?: () => BackendSnapshotResult | Promise; - } = {}, -) { - const session = { name: 'default', snapshot }; - const sessions = { - get: () => session, - set: (record) => { - session.snapshot = record.snapshot ?? session.snapshot; - }, - } satisfies CommandSessionStore; - return createAgentDevice({ - backend: { - platform: 'ios', - captureSnapshot: async () => - options.captureSnapshot ? await options.captureSnapshot() : { snapshot }, - readText: async () => ({ text: options.readText ?? '' }), - findText: async () => ({ found: options.findText ?? false }), - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions, - policy: localCommandPolicy(), - clock: { - now: () => options.now ?? 0, - sleep: async () => {}, - }, - }); -}