From 68f9a856452027d8fcbb8f5b806437fbf7a134a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 4 Jul 2026 15:31:48 +0200 Subject: [PATCH 1/2] feat: direct iOS selector falls back to tree resolution on semantic failures (ADR 0011) Runner ELEMENT_NOT_FOUND/AMBIGUOUS_MATCH on the direct iOS selector dispatch now delegate to the tree-based runtime path, which supplies runtime disambiguation, occlusion refusal, non-hittable promotion/annotation, and rich selector diagnostics/hints. Maestro replay dispatches (allowNonHittableCoordinateFallback) keep the runner-native error shapes: the new delegateSemanticFailures option is threaded from the dispatch flag, and the query path (allowElementNotFound at selector-runtime) keeps its exact semantics. Registry: direct-ios-selector errorTaxonomy/nonHittable/occlusion flip from gap waivers to delegated cells; the pinned gap list shrinks 6 -> 3. Success-path parity cells (disambiguation, responseIdentity) intentionally remain gaps per ADR 0011. Refs #1081 --- .../__tests__/interaction-guarantees.test.ts | 3 - src/contracts/interaction-guarantees.ts | 21 +- .../__tests__/direct-ios-selector.test.ts | 39 ++- src/daemon/direct-ios-selector.ts | 31 ++- .../handlers/__tests__/interaction.test.ts | 125 +++++++-- src/daemon/handlers/interaction-touch.ts | 8 +- ...teraction-direct-selector-fallback.test.ts | 248 ++++++++++++++++++ 7 files changed, 437 insertions(+), 38 deletions(-) create mode 100644 test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts diff --git a/src/contracts/__tests__/interaction-guarantees.test.ts b/src/contracts/__tests__/interaction-guarantees.test.ts index f36c51a70..74b53f8a0 100644 --- a/src/contracts/__tests__/interaction-guarantees.test.ts +++ b/src/contracts/__tests__/interaction-guarantees.test.ts @@ -180,9 +180,6 @@ test('acknowledged gaps are visible and bounded', () => { // (umbrella: https://github.com/callstack/agent-device/issues/1081). assert.deepEqual(gaps.sort(), [ 'direct-ios-selector/disambiguation', - 'direct-ios-selector/errorTaxonomy', - 'direct-ios-selector/nonHittable', - 'direct-ios-selector/occlusion', 'direct-ios-selector/responseIdentity', 'maestro-non-hittable-fallback/errorTaxonomy', ]); diff --git a/src/contracts/interaction-guarantees.ts b/src/contracts/interaction-guarantees.ts index 027a37c27..3770a4baf 100644 --- a/src/contracts/interaction-guarantees.ts +++ b/src/contracts/interaction-guarantees.ts @@ -215,10 +215,9 @@ export const INTERACTION_DISPATCH_PATHS: Record { +test('runner ELEMENT_NOT_FOUND falls back for query callers that allow it', () => { const error = new AppError('ELEMENT_NOT_FOUND', 'element not found'); assert.equal(isDirectIosSelectorFallbackError(error), false); assert.equal(isDirectIosSelectorFallbackError(error, { allowElementNotFound: true }), true); }); +test('semantic failures delegate to the runtime path for interaction dispatches (ADR 0011)', () => { + const notFound = new AppError('ELEMENT_NOT_FOUND', 'element not found'); + const ambiguous = new AppError('AMBIGUOUS_MATCH', 'multiple'); + assert.equal( + isDirectIosSelectorFallbackError(notFound, { delegateSemanticFailures: true }), + true, + ); + assert.equal( + isDirectIosSelectorFallbackError(ambiguous, { delegateSemanticFailures: true }), + true, + ); +}); + +test('maestro replay dispatches preserve the runner semantic error shapes (no fallback)', () => { + const notFound = new AppError('ELEMENT_NOT_FOUND', 'element not found'); + const ambiguous = new AppError('AMBIGUOUS_MATCH', 'multiple'); + assert.equal( + isDirectIosSelectorFallbackError(notFound, { delegateSemanticFailures: false }), + false, + ); + assert.equal( + isDirectIosSelectorFallbackError(ambiguous, { delegateSemanticFailures: false }), + false, + ); +}); + +test('AMBIGUOUS_MATCH does not fall back on the query path (allowElementNotFound callers)', () => { + const ambiguous = new AppError('AMBIGUOUS_MATCH', 'multiple'); + assert.equal(isDirectIosSelectorFallbackError(ambiguous), false); + assert.equal(isDirectIosSelectorFallbackError(ambiguous, { allowElementNotFound: true }), false); +}); + test('transport-level COMMAND_FAILED errors fall back, semantic ones do not', () => { assert.equal( isDirectIosSelectorFallbackError(new AppError('COMMAND_FAILED', 'fetch failed')), @@ -30,8 +63,4 @@ test('transport-level COMMAND_FAILED errors fall back, semantic ones do not', () isDirectIosSelectorFallbackError(new AppError('COMMAND_FAILED', 'element covered by overlay')), false, ); - assert.equal( - isDirectIosSelectorFallbackError(new AppError('AMBIGUOUS_MATCH', 'multiple')), - false, - ); }); diff --git a/src/daemon/direct-ios-selector.ts b/src/daemon/direct-ios-selector.ts index 317571835..7edfebb37 100644 --- a/src/daemon/direct-ios-selector.ts +++ b/src/daemon/direct-ios-selector.ts @@ -31,16 +31,43 @@ function isRunnerNativeSelectorKey(key: string): key is DirectIosSelectorTarget[ export function isDirectIosSelectorFallbackError( error: unknown, - options: { allowElementNotFound?: boolean } = {}, + options: { + /** + * Read/query callers (wait/get/is): a runner ELEMENT_NOT_FOUND re-resolves + * against the daemon tree, but AMBIGUOUS_MATCH still surfaces as-is. + */ + allowElementNotFound?: boolean; + /** + * ADR 0011 delegation-on-error for interaction dispatches: the runner's + * semantic failure shapes (ELEMENT_NOT_FOUND, AMBIGUOUS_MATCH) fall back + * to the tree-based runtime path, which supplies runtime disambiguation, + * non-hittable promotion/annotation, occlusion refusal, and rich selector + * diagnostics/hints. Must stay OFF for Maestro replay dispatches + * (allowNonHittableCoordinateFallback): replay matching is intentionally + * runner-native, so those error shapes must surface unchanged. + */ + delegateSemanticFailures?: boolean; + } = {}, ): boolean { const appError = asAppError(error); - if (appError.code === 'ELEMENT_NOT_FOUND') return options.allowElementNotFound === true; + if (appError.code === 'ELEMENT_NOT_FOUND') { + return options.delegateSemanticFailures === true || options.allowElementNotFound === true; + } + if (appError.code === 'AMBIGUOUS_MATCH') return options.delegateSemanticFailures === true; // The runner refuses to tap a hittable match whose frame is outside the app // frame (closed drawer, off-viewport carousel). The tree-based path either // prefers an on-screen candidate or raises the actionable offscreen_selector // error, so always fall back. if (appError.code === 'ELEMENT_OFFSCREEN') return true; if (appError.code !== 'COMMAND_FAILED') return false; + // Transport-failure classification stays message-based deliberately: the + // sniffed shapes originate at 4+ scattered throw sites (runner-transport + // deadline errors, runner-contract connect errors, runner-session's invalid + // response) plus raw undici "fetch failed" TypeErrors that are only wrapped + // into AppError at this boundary — and isRetryableRunnerError performs the + // same message sniffing for retry policy. A typed transport marker needs a + // wrapping layer around all of them in one change; tracked as Tier-3 error + // cleanup, not worth entangling with this fallback decision. const message = appError.message.toLowerCase(); return ( message.includes('fetch failed') || diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index edd5271df..d79bfff2d 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -657,17 +657,38 @@ test('click simple iOS id selector falls back to snapshot coordinates on transpo } }); -test('click simple iOS id selector does not snapshot-fallback on runner element miss', async () => { +test('click simple iOS id selector falls back to snapshot resolution on runner element miss', async () => { const sessionStore = makeSessionStore(); const sessionName = 'ios-direct-selector-element-miss'; sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); - mockDispatch.mockImplementation(async (_device, command, _positionals, _out, context) => { + mockDispatch.mockImplementation(async (_device, command, positionals, _out, context) => { if (command === 'press' && (context as Record)?.directElementSelector) { throw new AppError('ELEMENT_NOT_FOUND', 'element not found'); } if (command === 'snapshot') { - throw new Error('snapshot fallback should not run'); + return { + nodes: attachRefs([ + { + index: 0, + type: 'Window', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + identifier: 'submit', + rect: { x: 20, y: 80, width: 120, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + backend: 'xctest', + }; + } + if (command === 'press') { + return { x: Number(positionals[0]), y: Number(positionals[1]), pressed: true }; } return {}; }); @@ -685,24 +706,58 @@ test('click simple iOS id selector does not snapshot-fallback on runner element contextFromFlags, }); - expect(response?.ok).toBe(false); - if (response?.ok === false) { - expect(response.error.code).toBe('ELEMENT_NOT_FOUND'); + expect(response?.ok).toBe(true); + const pressCalls = mockDispatch.mock.calls.filter((call) => call[1] === 'press'); + expect(pressCalls.length).toBe(2); + expect(pressCalls[1]?.[2]).toEqual(['80', '100']); + if (response?.ok) { + expect(response.data?.selectorChain).toContain('id="submit"'); } - expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(0); }); -test('click simple iOS id selector does not snapshot-fallback on ambiguous runner match', async () => { +test('click simple iOS id selector falls back to runtime disambiguation on ambiguous runner match', async () => { const sessionStore = makeSessionStore(); const sessionName = 'ios-direct-selector-ambiguous'; sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); - mockDispatch.mockImplementation(async (_device, command, _positionals, _out, context) => { + mockDispatch.mockImplementation(async (_device, command, positionals, _out, context) => { if (command === 'press' && (context as Record)?.directElementSelector) { throw new AppError('AMBIGUOUS_MATCH', 'Selector matched multiple elements'); } if (command === 'snapshot') { - throw new Error('snapshot fallback should not run'); + return { + nodes: attachRefs([ + { + index: 0, + type: 'Window', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + // Off-screen drawer twin: runtime disambiguation prefers the + // visible candidate instead of failing like the runner did. + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + identifier: 'submit', + rect: { x: -300, y: 80, width: 120, height: 40 }, + enabled: true, + hittable: true, + }, + { + index: 2, + parentIndex: 0, + type: 'XCUIElementTypeButton', + identifier: 'submit', + rect: { x: 20, y: 80, width: 120, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + backend: 'xctest', + }; + } + if (command === 'press') { + return { x: Number(positionals[0]), y: Number(positionals[1]), pressed: true }; } return {}; }); @@ -720,13 +775,53 @@ test('click simple iOS id selector does not snapshot-fallback on ambiguous runne contextFromFlags, }); - expect(response?.ok).toBe(false); - if (response?.ok === false) { - expect(response.error.code).toBe('AMBIGUOUS_MATCH'); - } - expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(0); + expect(response?.ok).toBe(true); + const pressCalls = mockDispatch.mock.calls.filter((call) => call[1] === 'press'); + expect(pressCalls.length).toBe(2); + expect(pressCalls[1]?.[2]).toEqual(['80', '100']); }); +test.each([ + ['ELEMENT_NOT_FOUND', 'element not found'], + ['AMBIGUOUS_MATCH', 'Selector matched multiple elements'], +] as const)( + 'maestro-flagged click keeps runner %s error without snapshot fallback', + async (code, message) => { + const sessionStore = makeSessionStore(); + const sessionName = `ios-maestro-direct-selector-${code}`; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + + mockDispatch.mockImplementation(async (_device, command, _positionals, _out, context) => { + if (command === 'press' && (context as Record)?.directElementSelector) { + throw new AppError(code, message); + } + if (command === 'snapshot') { + throw new Error('snapshot fallback should not run for maestro replay dispatches'); + } + return {}; + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'click', + positionals: ['id="submit"'], + flags: { maestro: { allowNonHittableCoordinateFallback: true } }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(false); + if (response?.ok === false) { + expect(response.error.code).toBe(code); + } + expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(0); + }, +); + test('click simple iOS id selector waits for snapshot path after pending gesture stabilization', async () => { const sessionStore = makeSessionStore(); const sessionName = 'ios-direct-selector-after-swipe'; diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 4aa96e515..6e37e618c 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -352,7 +352,13 @@ async function dispatchDirectIosSelectorInteraction(params: { actionFinishedAt, }); } catch (error) { - if (!isDirectIosSelectorFallbackError(error)) { + // ADR 0011 delegation-on-error: semantic runner failures fall back to the + // tree-based runtime path — except for Maestro replay dispatches, whose + // runner-native error shapes must be preserved. + const fallback = isDirectIosSelectorFallbackError(error, { + delegateSemanticFailures: selector.allowNonHittableCoordinateFallback !== true, + }); + if (!fallback) { return { ok: false, error: normalizeError(error) }; } emitDiagnostic({ diff --git a/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts b/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts new file mode 100644 index 000000000..de3106c9b --- /dev/null +++ b/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts @@ -0,0 +1,248 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { AppError } from '../../../src/kernel/errors.ts'; +import { assertRpcError, assertRpcOk } from './assertions.ts'; +import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { + createProviderScenarioHarness, + withProviderScenarioResource, + type ProviderScenarioHarness, +} from './harness.ts'; +import { + createAppleRunnerProviderFromTranscript, + createRecordingAppleToolProvider, + simctlListDevicesHandler, +} from './providers.ts'; +import { + createProviderTranscript, + type ProviderScenarioProviderEntry, + type ProviderScenarioTranscript, +} from './transcript.ts'; + +// ADR 0011 delegation-on-error for the direct iOS selector path: when the +// runner fails with a semantic shape (ELEMENT_NOT_FOUND / AMBIGUOUS_MATCH), +// the dispatch falls back to tree-based runtime resolution, which supplies +// runtime disambiguation, occlusion refusal, and non-hittable +// promotion/annotation. Maestro replay dispatches keep the runner-native +// error shapes (no fallback). + +const APP = 'com.example.app'; +const DEVICE_ID = PROVIDER_SCENARIO_IOS_SIMULATOR.id; + +const APPLICATION_NODE = { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, +}; + +// Drawer twin: same label off-screen and on-screen — runtime disambiguation +// prefers the visible candidate where the runner raised AMBIGUOUS_MATCH. +const AMBIGUOUS_NODES = [ + APPLICATION_NODE, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + hittable: true, + rect: { x: -300, y: 300, width: 200, height: 44 }, + }, + { + index: 2, + parentIndex: 0, + type: 'Button', + label: 'Continue', + hittable: true, + rect: { x: 100, y: 300, width: 200, height: 44 }, + }, +]; + +// A later overlay-like sibling covering the target's center: the runtime path +// annotates the target as covered and refuses the interaction with a hint. +const COVERED_NODES = [ + APPLICATION_NODE, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + hittable: true, + rect: { x: 100, y: 300, width: 200, height: 44 }, + }, + { + index: 2, + parentIndex: 0, + type: 'Sheet', + label: 'Cookie consent', + hittable: true, + rect: { x: 0, y: 200, width: 400, height: 400 }, + }, +]; + +// The runner skips non-hittable matches (ELEMENT_NOT_FOUND); the runtime path +// still resolves the element and annotates targetHittable/hint. +const NON_HITTABLE_NODES = [ + APPLICATION_NODE, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + hittable: false, + rect: { x: 100, y: 300, width: 200, height: 44 }, + }, +]; + +function snapshotEntry(nodes: unknown[]): ProviderScenarioProviderEntry { + return { + command: 'ios.runner.snapshot', + deviceId: DEVICE_ID, + platform: 'apple', + result: { nodes, truncated: false }, + }; +} + +async function withDirectSelectorScenario( + transcript: ProviderScenarioTranscript, + run: (daemon: ProviderScenarioHarness) => Promise, +): Promise { + const appleRunnerProvider = createAppleRunnerProviderFromTranscript(transcript, 'ios.runner'); + const appleTool = createRecordingAppleToolProvider({ + simctl: simctlListDevicesHandler('com.apple.CoreSimulator.SimRuntime.iOS-18-0', [ + { name: PROVIDER_SCENARIO_IOS_SIMULATOR.name, udid: DEVICE_ID }, + ]), + }); + + await withProviderScenarioResource( + async () => + await createProviderScenarioHarness({ + appleRunnerProvider: () => appleRunnerProvider, + appleToolProvider: () => appleTool.provider, + deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], + }), + async (daemon) => { + const open = await daemon.callCommand('open', [APP], { + platform: 'ios', + udid: DEVICE_ID, + }); + assertRpcOk(open); + await run(daemon); + transcript.assertComplete(); + }, + ); +} + +test('Provider-backed integration runner AMBIGUOUS_MATCH falls back to runtime disambiguation', async () => { + const transcript = createProviderTranscript([ + // Direct selector tap attempt fails with the runner's semantic shape. + { + command: 'ios.runner.tap', + deviceId: DEVICE_ID, + platform: 'apple', + error: new AppError('AMBIGUOUS_MATCH', 'Selector matched multiple elements'), + }, + // Fallback: tree capture, disambiguation picks the visible twin, taps it. + snapshotEntry(AMBIGUOUS_NODES), + { + command: 'ios.runner.tap', + deviceId: DEVICE_ID, + platform: 'apple', + result: { x: 200, y: 322 }, + }, + ]); + + await withDirectSelectorScenario(transcript, async (daemon) => { + const click = await daemon.callCommand('click', ['label="Continue"']); + const data = assertRpcOk(click); + assert.equal(data.x, 200); + assert.equal(data.y, 322); + assert.ok( + Array.isArray(data.selectorChain) && data.selectorChain.includes('label="Continue"'), + `selectorChain must include the resolved selector, got ${JSON.stringify(data.selectorChain)}`, + ); + + const tapCalls = transcript.calls.filter((call) => call.command === 'ios.runner.tap'); + assert.equal(tapCalls.length, 2); + const fallbackTap = tapCalls[1]?.request as Record; + assert.equal(fallbackTap.x, 200); + assert.equal(fallbackTap.y, 322); + }); +}); + +test('Provider-backed integration runner ELEMENT_NOT_FOUND on a covered element surfaces the occlusion refusal', async () => { + const transcript = createProviderTranscript([ + { + command: 'ios.runner.tap', + deviceId: DEVICE_ID, + platform: 'apple', + error: new AppError('ELEMENT_NOT_FOUND', 'element not found'), + }, + // Interactive capture filters the covered node, so the runtime retries + // with a full capture before raising the covered-element refusal. + snapshotEntry(COVERED_NODES), + snapshotEntry(COVERED_NODES), + ]); + + await withDirectSelectorScenario(transcript, async (daemon) => { + const click = await daemon.callCommand('click', ['label="Continue"']); + const details = assertRpcError( + click, + 'COMMAND_FAILED', + /covered by another visible element and cannot be tapped safely/, + ); + assert.match(String(details.hint ?? ''), /scroll it clear of the overlay/); + }); +}); + +test('Provider-backed integration runner ELEMENT_NOT_FOUND on a non-hittable element annotates targetHittable', async () => { + const transcript = createProviderTranscript([ + { + command: 'ios.runner.tap', + deviceId: DEVICE_ID, + platform: 'apple', + error: new AppError('ELEMENT_NOT_FOUND', 'element not found'), + }, + snapshotEntry(NON_HITTABLE_NODES), + { + command: 'ios.runner.tap', + deviceId: DEVICE_ID, + platform: 'apple', + result: { x: 200, y: 322 }, + }, + ]); + + await withDirectSelectorScenario(transcript, async (daemon) => { + const click = await daemon.callCommand('click', ['label="Continue"']); + const data = assertRpcOk(click); + assert.equal(data.targetHittable, false); + assert.match(String(data.hint ?? ''), /hittable: false/); + }); +}); + +test('Provider-backed integration maestro replay dispatch keeps runner AMBIGUOUS_MATCH without fallback', async () => { + const transcript = createProviderTranscript([ + { + command: 'ios.runner.tap', + deviceId: DEVICE_ID, + platform: 'apple', + // Proves the maestro flag rode along on the direct dispatch AND that no + // snapshot fallback follows: this is the only transcript entry. + request: { + command: 'tap', + selectorKey: 'label', + selectorValue: 'Continue', + allowNonHittableCoordinateFallback: true, + appBundleId: APP, + }, + error: new AppError('AMBIGUOUS_MATCH', 'Selector matched multiple elements'), + }, + ]); + + await withDirectSelectorScenario(transcript, async (daemon) => { + const click = await daemon.callCommand('click', ['label="Continue"'], { + maestro: { allowNonHittableCoordinateFallback: true }, + }); + assertRpcError(click, 'AMBIGUOUS_MATCH', /matched multiple/); + }); +}); From 97cd23147e0120a19bc06c5ee742cd077b5356a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 4 Jul 2026 16:09:44 +0200 Subject: [PATCH 2/2] test: contract coverage for the three newly-delegated direct-iOS cells MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The coverage gate demanded scenarios the moment the rebase flipped errorTaxonomy/nonHittable/occlusion to delegated — as designed. Three transcript scenarios prove the delegation end to end: runner ELEMENT_NOT_FOUND falls back to runtime no-match diagnostics, to the covered refusal, and to an annotated coordinate tap (the transcript asserts the fallback tap is coordinate-keyed). --- .../direct-ios-selector.contract.test.ts | 57 +++++++++++++++++++ .../direct-ios-selector.coverage.ts | 6 ++ .../interaction-contract/fixtures.ts | 41 +++++++++++++ 3 files changed, 104 insertions(+) diff --git a/test/integration/interaction-contract/direct-ios-selector.contract.test.ts b/test/integration/interaction-contract/direct-ios-selector.contract.test.ts index 8dd7586f0..e682e6771 100644 --- a/test/integration/interaction-contract/direct-ios-selector.contract.test.ts +++ b/test/integration/interaction-contract/direct-ios-selector.contract.test.ts @@ -9,6 +9,8 @@ import { RUNNER_CHANGED_NODES, RUNNER_CLOSED_DRAWER_NODES, RUNNER_CONTINUE_NODES, + RUNNER_COVERED_NODES, + RUNNER_NON_HITTABLE_NODES, } from './fixtures.ts'; import { runnerSnapshotEntry, @@ -99,3 +101,58 @@ test(scenario('verifyEvidence'), async () => { }, ); }); + +test(scenario('errorTaxonomy'), async () => { + await withIosContractDaemon( + [ + runnerTapErrorEntry(new AppError('ELEMENT_NOT_FOUND', 'element not found')), + // Runtime fallback resolution: interactive-only capture, then the + // full-tree retry — neither contains the selector. + runnerSnapshotEntry(RUNNER_CONTINUE_NODES), + runnerSnapshotEntry(RUNNER_CONTINUE_NODES), + ], + async (daemon) => { + const click = await daemon.callCommand('click', ['label=Missing']); + const error = assertRpcError(click, 'COMMAND_FAILED', /did not match/); + // The delegated taxonomy: runtime diagnostics and the actionable hint, + // not the runner's bare "element not found". + assert.ok(typeof error.hint === 'string' && error.hint.length > 0); + }, + ); +}); + +test(scenario('occlusion'), async () => { + await withIosContractDaemon( + [ + runnerTapErrorEntry(new AppError('ELEMENT_NOT_FOUND', 'element not found')), + runnerSnapshotEntry(RUNNER_COVERED_NODES), + runnerSnapshotEntry(RUNNER_COVERED_NODES), + ], + async (daemon) => { + const click = await daemon.callCommand('click', ['label="Save draft"']); + const error = assertRpcError(click, 'COMMAND_FAILED', /covered by another visible element/); + const details = error.details as Record | undefined; + assert.equal(details?.interactionBlocked, 'covered'); + }, + ); +}); + +test(scenario('nonHittable'), async () => { + await withIosContractDaemon( + [ + runnerTapErrorEntry(new AppError('ELEMENT_NOT_FOUND', 'element not found')), + runnerSnapshotEntry(RUNNER_NON_HITTABLE_NODES), + runnerTapEntry({ x: 200, y: 330 }), + ], + async (daemon, transcript) => { + const click = await daemon.callCommand('click', ['label="Recents row"']); + const data = assertRpcOk(click); + // Delegation really happened: the second tap is coordinate-keyed. + const fallbackTap = transcript.calls.at(-1)?.request as Record | undefined; + assert.equal(transcript.calls.at(-1)?.command, 'ios.runner.tap'); + assert.equal(fallbackTap?.selectorKey, undefined); + assert.equal(data.targetHittable, false); + assert.match(String(data.hint ?? ''), /hittable: false/); + }, + ); +}); diff --git a/test/integration/interaction-contract/direct-ios-selector.coverage.ts b/test/integration/interaction-contract/direct-ios-selector.coverage.ts index 4cb7b2622..0df1e1f80 100644 --- a/test/integration/interaction-contract/direct-ios-selector.coverage.ts +++ b/test/integration/interaction-contract/direct-ios-selector.coverage.ts @@ -1,6 +1,12 @@ import { definePathCoverage } from './coverage-manifest.ts'; export const DIRECT_IOS_SELECTOR_COVERAGE = definePathCoverage('direct-ios-selector', { + errorTaxonomy: + 'direct-ios-selector errorTaxonomy: runner ELEMENT_NOT_FOUND falls back to runtime no-match diagnostics and hint', + nonHittable: + 'direct-ios-selector nonHittable: runner ELEMENT_NOT_FOUND on a non-hittable target falls back to an annotated coordinate tap', + occlusion: + 'direct-ios-selector occlusion: runner ELEMENT_NOT_FOUND on a covered target falls back to the runtime covered refusal', offscreen: 'direct-ios-selector offscreen: runner ELEMENT_OFFSCREEN falls back to the runtime refusal', responseConstruction: diff --git a/test/integration/interaction-contract/fixtures.ts b/test/integration/interaction-contract/fixtures.ts index 48ae932d0..ab4438370 100644 --- a/test/integration/interaction-contract/fixtures.ts +++ b/test/integration/interaction-contract/fixtures.ts @@ -249,3 +249,44 @@ export const RUNNER_CLOSED_DRAWER_NODES = [ rect: { x: -320, y: 240, width: 300, height: 50 }, }, ] as const; + +// Runner-side covered control (#1091 delegation): the runner skips it as +// non-hittable (ELEMENT_NOT_FOUND) and the runtime fallback must refuse with +// the covered shape instead of tapping through the overlay. +export const RUNNER_COVERED_NODES = [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Save draft', + hittable: false, + interactionBlocked: 'covered', + rect: { x: 16, y: 700, width: 140, height: 44 }, + }, +] as const; + +// Runner-side visible-but-non-hittable cell (#1037 shape): the runner reports +// ELEMENT_NOT_FOUND, the runtime fallback proceeds by coordinates and +// annotates the result instead of failing. +export const RUNNER_NON_HITTABLE_NODES = [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + parentIndex: 0, + type: 'Cell', + label: 'Recents row', + hittable: false, + rect: { x: 20, y: 300, width: 360, height: 60 }, + }, +] as const;