From ac971f034f44824104ed13d80b1e4222a12b673e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 5 Jul 2026 11:53:42 +0200 Subject: [PATCH 1/2] fix: normalize interaction response wire shapes --- .../handlers/__tests__/interaction.test.ts | 9 +- .../handlers/interaction-touch-response.ts | 116 ++++-- src/daemon/handlers/interaction-touch.ts | 8 +- src/mcp/command-output-schemas.ts | 127 ++---- .../interaction-contract/daemon-harness.ts | 34 +- ...nteraction-response-shape.contract.test.ts | 380 ++++++++++++++++++ .../runtime-ref.contract.test.ts | 6 +- 7 files changed, 530 insertions(+), 150 deletions(-) create mode 100644 test/integration/interaction-contract/interaction-response-shape.contract.test.ts diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 1ce291c99..d7fd43b7f 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -2214,7 +2214,7 @@ test('press @ref does not promote to a full-screen hittable ancestor', async () expect(mockDispatch.mock.calls[0]?.[2]).toEqual(['201', '319']); }); -test('fill @ref preserves fallback coordinates for recording when platform result is sparse', async () => { +test('fill @ref returns canonical wire coordinates and records fallback coordinates when platform result is sparse', async () => { const sessionStore = makeSessionStore(); const sessionName = 'default'; const session = makeSession(sessionName); @@ -2262,7 +2262,12 @@ test('fill @ref preserves fallback coordinates for recording when platform resul expect(response?.ok).toBe(true); if (response?.ok) { expect(response.data?.filled).toBe(true); - expect(response.data?.x).toBeUndefined(); + expect(response.data?.targetKind).toBe('ref'); + expect(response.data?.ref).toBe('e1'); + expect(response.data?.x).toBe(60); + expect(response.data?.y).toBe(40); + expect(response.data?.text).toBe('hello@example.com'); + expect(response.data?.message).toBe('Filled 17 chars'); } const stored = sessionStore.get(sessionName); diff --git a/src/daemon/handlers/interaction-touch-response.ts b/src/daemon/handlers/interaction-touch-response.ts index e6290b256..395d63a09 100644 --- a/src/daemon/handlers/interaction-touch-response.ts +++ b/src/daemon/handlers/interaction-touch-response.ts @@ -5,7 +5,7 @@ import type { PressCommandResult, } from '../../contracts/interaction.ts'; import { successText } from '../../utils/success-text.ts'; -import { interactionResultExtra, stripAtPrefix } from './interaction-touch-targets.ts'; +import { interactionResultExtra } from './interaction-touch-targets.ts'; /** * The single construction site for interaction response payloads (ADR 0011 @@ -25,17 +25,12 @@ export type InteractionResponseSource = | { kind: 'runtime'; result: InteractionRuntimeResult; - /** - * Wire-compat for fill @ref: the response echoes backendResult (or a - * minimal ref/point fallback) plus the identity extras instead of the - * visualization shape. Only applies when the result resolved as a ref. - */ - refBackendWireShape?: boolean; } | { // Direct iOS selector dispatch: no runtime result exists, only the raw // runner payload; identity extras are a declared gap on that path. kind: 'runner-payload'; + targetKind: InteractionRuntimeResult['kind']; data: Record; point: { x: number; y: number }; }; @@ -67,37 +62,48 @@ export function buildInteractionResponseData(params: { }): InteractionResponsePayloads { const { source, referenceFrame, extra } = params; if (source.kind === 'runner-payload') { - const payload = buildTouchVisualizationResult({ + const commonExtra = { + targetKind: source.targetKind, + ...(extra ?? {}), + }; + const result = buildTouchPayload({ data: source.data, fallbackX: source.point.x, fallbackY: source.point.y, referenceFrame, - extra, + extra: commonExtra, }); - return { result: payload, responseData: payload }; + const responseData = buildTouchPayload({ + data: sanitizeWireBackendData(source.data), + fallbackX: source.point.x, + fallbackY: source.point.y, + referenceFrame, + extra: commonExtra, + }); + return { result, responseData }; } const { result } = source; - const visualization = buildTouchVisualizationResult({ + const resultExtra = interactionResultExtra(result); + const commonExtra = { + targetKind: result.kind, + ...resultExtra, + ...(extra ?? {}), + }; + const visualization = buildTouchPayload({ data: result.backendResult, fallbackX: result.point?.x, fallbackY: result.point?.y, referenceFrame, - extra: { - ...interactionResultExtra(result), - ...(extra ?? {}), - }, + extra: commonExtra, + }); + const responseData = buildTouchPayload({ + data: sanitizeWireBackendData(result.backendResult), + fallbackX: result.point?.x, + fallbackY: result.point?.y, + referenceFrame, + extra: commonExtra, }); - const responseData = - source.refBackendWireShape && result.kind === 'ref' - ? { - ...(result.backendResult ?? { - ref: stripAtPrefix(result.target?.kind === 'ref' ? result.target.ref : undefined), - ...(result.point ? { x: result.point.x, y: result.point.y } : {}), - }), - ...interactionResultExtra(result), - } - : visualization; const warning = composeResponseWarning( 'warning' in result ? result.warning : undefined, params.staleRefsWarning, @@ -117,7 +123,7 @@ function composeResponseWarning( return resultWarning ? `${resultWarning} ${staleRefsWarning}` : staleRefsWarning; } -function buildTouchVisualizationResult(params: { +function buildTouchPayload(params: { data: Record | undefined; fallbackX?: number; fallbackY?: number; @@ -128,13 +134,46 @@ function buildTouchVisualizationResult(params: { const message = buildTouchMessage(extra, fallbackX, fallbackY) ?? (typeof data?.message === 'string' ? data.message : undefined); - return { + return stripUndefinedFields({ + ...(data ?? {}), ...(fallbackX === undefined || fallbackY === undefined ? {} : { x: fallbackX, y: fallbackY }), ...(referenceFrame ?? {}), ...(extra ?? {}), - ...(data ?? {}), ...successText(message), - }; + }); +} + +function stripUndefinedFields(data: Record): Record { + return Object.fromEntries(Object.entries(data).filter(([, value]) => value !== undefined)); +} + +function sanitizeWireBackendData( + data: Record | undefined, +): Record | undefined { + if (!data) return undefined; + const sanitized = Object.fromEntries( + Object.entries(data).filter(([key, value]) => shouldKeepWireBackendField(key, value)), + ); + return Object.keys(sanitized).length > 0 ? sanitized : undefined; +} + +function shouldKeepWireBackendField(key: string, value: unknown): boolean { + switch (key) { + case 'gestureStartUptimeMs': + case 'gestureEndUptimeMs': + case 'sequenceResults': + return false; + case 'count': + return value !== 1; + case 'intervalMs': + case 'holdMs': + case 'jitterPx': + return value !== 0; + case 'doubleTap': + return value !== false; + default: + return true; + } } function buildTouchMessage( @@ -146,22 +185,27 @@ function buildTouchMessage( return `Filled ${Array.from(extra.text).length} chars`; } const ref = typeof extra?.ref === 'string' ? extra.ref : undefined; - if (!ref) return undefined; + const selector = typeof extra?.selector === 'string' ? extra.selector : undefined; const pointSuffix = x === undefined || y === undefined ? '' : ` (${x}, ${y})`; - return buildRefTouchMessage(ref, extra ?? {}, pointSuffix); + const label = ref ? `@${ref}` : (selector ?? ''); + if (!label) { + if (x === undefined || y === undefined) return undefined; + return extra?.gesture === 'longpress' ? `Long pressed (${x}, ${y})` : `Tapped (${x}, ${y})`; + } + return buildTouchTargetMessage(label, extra ?? {}, pointSuffix); } -function buildRefTouchMessage( - ref: string, +function buildTouchTargetMessage( + label: string, extra: Record, pointSuffix: string, ): string { const button = typeof extra.button === 'string' ? extra.button : undefined; if (extra.gesture === 'longpress') { - return `Long pressed @${ref}${pointSuffix}`; + return `Long pressed ${label}${pointSuffix}`; } if (button && button !== 'primary') { - return `Clicked ${button} @${ref}${pointSuffix}`; + return `Clicked ${button} ${label}${pointSuffix}`; } - return `Tapped @${ref}${pointSuffix}`; + return `Tapped ${label}${pointSuffix}`; } diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 8973aeadc..ce6e364b1 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -349,7 +349,7 @@ async function dispatchDirectIosSelectorInteraction(params: { const actionFinishedAt = Date.now(); const point = readPointFromDirectSelectorTapResult(data); const { result, responseData } = buildInteractionResponseData({ - source: { kind: 'runner-payload', data, point }, + source: { kind: 'runner-payload', targetKind: 'selector', data, point }, referenceFrame: readReferenceFrameFromDirectSelectorTapResult(data), extra: { ...extra, @@ -484,11 +484,7 @@ async function dispatchFillViaRuntime( ? undefined : readSnapshotNodesReferenceFrame(session.snapshot?.nodes ?? []); return buildInteractionResponseData({ - // refBackendWireShape keeps the historical fill @ref wire response - // (backendResult + identity extras) while the shared builder owns the - // extras — the hand-rolled version of this branch dropped evidence - // (PR #1064 review). - source: { kind: 'runtime', result, refBackendWireShape: true }, + source: { kind: 'runtime', result }, referenceFrame, extra: { text: parsedTarget.text }, staleRefsWarning, diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index 0df701371..5c74329c2 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -58,113 +58,44 @@ function objectSchema( const stringArraySchema: JsonSchema = { type: 'array', items: { type: 'string' } }; -const rectSchema: JsonSchema = objectSchema( - { - x: numberSchema(), - y: numberSchema(), - width: numberSchema(), - height: numberSchema(), - }, - ['x', 'y', 'width', 'height'], -); - -const pointSchema: JsonSchema = objectSchema({ x: numberSchema(), y: numberSchema() }, ['x', 'y']); - -// SnapshotNode = RawSnapshotNode & { ref } (src/kernel/snapshot.ts). `index` and -// `ref` are the only always-present fields; all others are optional. -const snapshotNodeSchema: JsonSchema = objectSchema( - { - index: numberSchema(), - ref: stringSchema('Stable snapshot ref such as e12.'), - type: stringSchema(), - role: stringSchema(), - subrole: stringSchema(), - label: stringSchema(), - value: stringSchema(), - identifier: stringSchema(), - rect: rectSchema, - enabled: booleanSchema(), - selected: booleanSchema(), - focused: booleanSchema(), - visibleToUser: booleanSchema(), - hittable: booleanSchema(), - depth: numberSchema(), - parentIndex: numberSchema(), - pid: numberSchema(), - bundleId: stringSchema(), - appName: stringSchema(), - windowTitle: stringSchema(), - surface: stringSchema(), - hiddenContentAbove: booleanSchema(), - hiddenContentBelow: booleanSchema(), - interactionBlocked: enumSchema(['covered']), - presentationHints: stringArraySchema, - }, - ['index', 'ref'], - 'Resolved snapshot node for the matched element.', -); - -const resolvedRefTargetSchema: JsonSchema = objectSchema( - { kind: constSchema('ref'), ref: stringSchema() }, - ['kind', 'ref'], -); - -const resolvedSelectorTargetSchema: JsonSchema = objectSchema( - { kind: constSchema('selector'), selector: stringSchema() }, - ['kind', 'selector'], -); - type InteractionExtra = { properties?: Record; required?: readonly string[]; }; /** - * `ResolvedInteractionTarget & extra` — a `kind` discriminated union (point / ref - * / selector) shared by press / fill / longpress. The `const` discriminant keeps - * the branches mutually exclusive, so the additive `cost` field never breaks the - * exactly-one-of contract. + * Canonical interaction wire response built by buildInteractionResponseData: + * shared target/coordinate/evidence fields plus per-command extras. The runtime + * result still has richer internal node/backend data; this schema documents the + * JSON payload returned to clients. */ -function interactionResultSchema(extra: InteractionExtra = {}): JsonSchema { +function interactionWireResultSchema(extra: InteractionExtra = {}): JsonSchema { const extraProperties = extra.properties ?? {}; const extraRequired = extra.required ?? []; - const pointBranch = objectSchema( - { kind: constSchema('point'), point: pointSchema, ...extraProperties }, - ['kind', 'point', ...extraRequired], - ); - const refBranch = objectSchema( + return objectSchema( { - kind: constSchema('ref'), - point: pointSchema, - target: resolvedRefTargetSchema, - node: snapshotNodeSchema, - selectorChain: stringArraySchema, - refLabel: stringSchema(), - targetHittable: booleanSchema(), - hint: stringSchema(), - ...extraProperties, - }, - ['kind', 'target', ...extraRequired], - ); - const selectorBranch = objectSchema( - { - kind: constSchema('selector'), - point: pointSchema, - target: resolvedSelectorTargetSchema, - node: snapshotNodeSchema, + targetKind: enumSchema(['point', 'ref', 'selector'], 'Resolved interaction target kind.'), + x: numberSchema('Resolved interaction x coordinate when available.'), + y: numberSchema('Resolved interaction y coordinate when available.'), + referenceWidth: numberSchema('Reference frame width for visualizing the interaction point.'), + referenceHeight: numberSchema( + 'Reference frame height for visualizing the interaction point.', + ), + ref: stringSchema('Snapshot ref without the @ prefix when the target was an @ref.'), + selector: stringSchema('Selector expression when the target was a selector.'), selectorChain: stringArraySchema, refLabel: stringSchema(), targetHittable: booleanSchema(), hint: stringSchema(), + warning: stringSchema(), + message: stringSchema(), + evidence: interactionEvidenceSchema, ...extraProperties, }, - ['kind', 'point', 'target', 'node', 'selectorChain', ...extraRequired], + ['targetKind', ...extraRequired], ); - return { type: 'object', oneOf: [pointBranch, refBranch, selectorBranch] }; } -const backendResultSchema = looseObjectSchema('Raw backend result passthrough.'); - // InteractionEvidence (src/contracts/interaction.ts) — opt-in `--verify` cheap // post-condition evidence (#1047). const interactionEvidenceSchema: JsonSchema = objectSchema( @@ -203,31 +134,25 @@ const targetShutdownResultSchema: JsonSchema = objectSchema( ); export const COMMAND_OUTPUT_SCHEMAS = { - // src/contracts/interaction.ts - press: interactionResultSchema({ + // buildInteractionResponseData wire payloads for interaction commands. + press: interactionWireResultSchema({ properties: { - backendResult: backendResultSchema, - message: stringSchema(), - warning: stringSchema(), evidence: interactionEvidenceSchema, + button: enumSchema(['secondary', 'middle']), }, }), - fill: interactionResultSchema({ + fill: interactionWireResultSchema({ properties: { text: stringSchema('Text submitted to the field.'), - warning: stringSchema(), - backendResult: backendResultSchema, - message: stringSchema(), + delayMs: numberSchema('Delay between typed characters in milliseconds.'), evidence: interactionEvidenceSchema, }, required: ['text'], }), - longpress: interactionResultSchema({ + longpress: interactionWireResultSchema({ properties: { durationMs: numberSchema(), - backendResult: backendResultSchema, - message: stringSchema(), - warning: stringSchema(), + gesture: constSchema('longpress'), }, }), diff --git a/test/integration/interaction-contract/daemon-harness.ts b/test/integration/interaction-contract/daemon-harness.ts index d12bbaeb0..3c609a4d7 100644 --- a/test/integration/interaction-contract/daemon-harness.ts +++ b/test/integration/interaction-contract/daemon-harness.ts @@ -16,7 +16,7 @@ import { type ProviderScenarioTranscript, } from '../provider-scenarios/transcript.ts'; -const CONTRACT_APP = 'com.example.app'; +export const CONTRACT_APP = 'com.example.app'; const CONTRACT_DEVICE_ID = PROVIDER_SCENARIO_IOS_SIMULATOR.id; /** @@ -67,11 +67,41 @@ export function runnerSnapshotEntry(nodes: readonly unknown[]): ProviderScenario }; } -export function runnerTapEntry(result: Record): ProviderScenarioProviderEntry { +export function runnerTapEntry( + result: Record, + request?: Record, +): ProviderScenarioProviderEntry { return { command: 'ios.runner.tap', deviceId: CONTRACT_DEVICE_ID, platform: 'apple', + ...(request ? { request } : {}), + result, + }; +} + +export function runnerTypeEntry( + result: Record, + request?: Record, +): ProviderScenarioProviderEntry { + return { + command: 'ios.runner.type', + deviceId: CONTRACT_DEVICE_ID, + platform: 'apple', + ...(request ? { request } : {}), + result, + }; +} + +export function runnerLongPressEntry( + result: Record, + request?: Record, +): ProviderScenarioProviderEntry { + return { + command: 'ios.runner.longPress', + deviceId: CONTRACT_DEVICE_ID, + platform: 'apple', + ...(request ? { request } : {}), result, }; } diff --git a/test/integration/interaction-contract/interaction-response-shape.contract.test.ts b/test/integration/interaction-contract/interaction-response-shape.contract.test.ts new file mode 100644 index 000000000..328e2b7a0 --- /dev/null +++ b/test/integration/interaction-contract/interaction-response-shape.contract.test.ts @@ -0,0 +1,380 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { assertRpcOk } from '../provider-scenarios/assertions.ts'; +import type { ProviderScenarioTranscript } from '../provider-scenarios/transcript.ts'; +import { RUNNER_CONTINUE_NODES } from './fixtures.ts'; +import { + CONTRACT_APP, + runnerLongPressEntry, + runnerSnapshotEntry, + runnerTapEntry, + runnerTypeEntry, + withIosContractDaemon, +} from './daemon-harness.ts'; + +type WireData = Record; + +const REF_KEYS = [ + 'message', + 'ref', + 'refLabel', + 'referenceHeight', + 'referenceWidth', + 'selectorChain', + 'targetKind', + 'x', + 'y', +] as const; + +const SELECTOR_KEYS = [ + 'message', + 'refLabel', + 'referenceHeight', + 'referenceWidth', + 'selector', + 'selectorChain', + 'targetKind', + 'x', + 'y', +] as const; + +const POINT_KEYS = ['message', 'targetKind', 'x', 'y'] as const; + +test('interaction response shape: press @ref uses the canonical ref envelope', async () => { + await withIosContractDaemon( + [runnerSnapshotEntry(RUNNER_CONTINUE_NODES), runnerTapEntry({})], + async (daemon, transcript) => { + assertRpcOk(await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true })); + + const data = assertRpcOk(await daemon.callCommand('press', ['@e2'])); + assertRunnerRequest(transcript, 'ios.runner.tap', tapRequest(200, 322)); + + assertCanonicalRef(data, { + message: 'Tapped @e2 (200, 322)', + x: 200, + y: 322, + }); + }, + ); +}); + +test('interaction response shape: fill @ref uses the canonical ref envelope', async () => { + await withIosContractDaemon( + [runnerSnapshotEntry(RUNNER_CONTINUE_NODES), runnerTypeEntry({})], + async (daemon, transcript) => { + assertRpcOk(await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true })); + + const data = assertRpcOk(await daemon.callCommand('fill', ['@e2', 'Hello'])); + assertRunnerRequest(transcript, 'ios.runner.type', typeRequest(200, 322, 'Hello')); + + assertCanonicalRef(data, { + message: 'Filled 5 chars', + x: 200, + y: 322, + extra: { + delayMs: 0, + text: 'Hello', + warning: 'fill target @e2 resolved to "Button", attempting fill anyway.', + }, + }); + }, + ); +}); + +test('interaction response shape: longpress @ref uses the canonical ref envelope', async () => { + await withIosContractDaemon( + [runnerSnapshotEntry(RUNNER_CONTINUE_NODES), runnerLongPressEntry({})], + async (daemon, transcript) => { + assertRpcOk(await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true })); + + const data = assertRpcOk(await daemon.callCommand('longpress', ['@e2', '700'])); + assertRunnerRequest(transcript, 'ios.runner.longPress', longPressRequest(200, 322, 700)); + + assertCanonicalRef(data, { + message: 'Long pressed @e2 (200, 322)', + x: 200, + y: 322, + extra: { durationMs: 700, gesture: 'longpress' }, + }); + }, + ); +}); + +test('interaction response shape: press selector uses the canonical selector envelope', async () => { + await withIosContractDaemon( + [runnerSnapshotEntry(RUNNER_CONTINUE_NODES), runnerTapEntry({})], + async (daemon, transcript) => { + const data = assertRpcOk(await daemon.callCommand('press', ['label=Continue'])); + assertRunnerRequest(transcript, 'ios.runner.tap', tapRequest(200, 322)); + + assertCanonicalSelector(data, { + message: 'Tapped label=Continue (200, 322)', + x: 200, + y: 322, + }); + }, + ); +}); + +test('interaction response shape: fill selector uses the canonical selector envelope', async () => { + await withIosContractDaemon([runnerTypeEntry({ x: 200, y: 322 })], async (daemon, transcript) => { + const data = assertRpcOk(await daemon.callCommand('fill', ['label=Continue', 'Hello'])); + assertRunnerRequest(transcript, 'ios.runner.type', selectorTypeRequest('Hello')); + + assertCanonicalDirectSelector(data, { + message: 'Filled 5 chars', + x: 200, + y: 322, + extra: { delayMs: 0, text: 'Hello' }, + }); + }); +}); + +test('interaction response shape: longpress selector uses the canonical selector envelope', async () => { + await withIosContractDaemon( + [runnerSnapshotEntry(RUNNER_CONTINUE_NODES), runnerLongPressEntry({})], + async (daemon, transcript) => { + const data = assertRpcOk(await daemon.callCommand('longpress', ['label=Continue', '700'])); + assertRunnerRequest(transcript, 'ios.runner.longPress', longPressRequest(200, 322, 700)); + + assertCanonicalSelector(data, { + message: 'Long pressed label=Continue (200, 322)', + x: 200, + y: 322, + extra: { durationMs: 700, gesture: 'longpress' }, + }); + }, + ); +}); + +test('interaction response shape: press point uses the canonical point envelope', async () => { + await withIosContractDaemon([runnerTapEntry({})], async (daemon, transcript) => { + const data = assertRpcOk(await daemon.callCommand('press', ['100', '200'])); + assertRunnerRequest(transcript, 'ios.runner.tap', tapRequest(100, 200)); + + assertCanonicalPoint(data, { + message: 'Tapped (100, 200)', + x: 100, + y: 200, + }); + }); +}); + +test('interaction response shape: fill point uses the canonical point envelope', async () => { + await withIosContractDaemon([runnerTypeEntry({})], async (daemon, transcript) => { + const data = assertRpcOk(await daemon.callCommand('fill', ['100', '200', 'Hello'])); + assertRunnerRequest(transcript, 'ios.runner.type', typeRequest(100, 200, 'Hello')); + + assertCanonicalPoint(data, { + message: 'Filled 5 chars', + x: 100, + y: 200, + extra: { delayMs: 0, text: 'Hello' }, + }); + }); +}); + +test('interaction response shape: longpress point uses the canonical point envelope', async () => { + await withIosContractDaemon([runnerLongPressEntry({})], async (daemon, transcript) => { + const data = assertRpcOk(await daemon.callCommand('longpress', ['100', '200', '700'])); + assertRunnerRequest(transcript, 'ios.runner.longPress', longPressRequest(100, 200, 700)); + + assertCanonicalPoint(data, { + message: 'Long pressed (100, 200)', + x: 100, + y: 200, + extra: { durationMs: 700, gesture: 'longpress' }, + }); + }); +}); + +function assertCanonicalRef( + data: WireData, + expected: { message: string; x: number; y: number; extra?: WireData }, +): void { + assertExactKeys(data, [...REF_KEYS, ...Object.keys(expected.extra ?? {})]); + assert.deepEqual( + pick(data, [ + 'message', + 'ref', + 'refLabel', + 'referenceHeight', + 'referenceWidth', + 'targetKind', + 'x', + 'y', + ...Object.keys(expected.extra ?? {}), + ]), + { + message: expected.message, + ref: 'e2', + refLabel: 'Continue', + referenceHeight: 800, + referenceWidth: 400, + targetKind: 'ref', + x: expected.x, + y: expected.y, + ...(expected.extra ?? {}), + }, + ); + assertSelectorChain(data); + assertNoWireNoise(data); +} + +function assertCanonicalSelector( + data: WireData, + expected: { message: string; x: number; y: number; extra?: WireData }, +): void { + assertExactKeys(data, [...SELECTOR_KEYS, ...Object.keys(expected.extra ?? {})]); + assert.deepEqual( + pick(data, [ + 'message', + 'refLabel', + 'referenceHeight', + 'referenceWidth', + 'selector', + 'targetKind', + 'x', + 'y', + ...Object.keys(expected.extra ?? {}), + ]), + { + message: expected.message, + refLabel: 'Continue', + referenceHeight: 800, + referenceWidth: 400, + selector: 'label=Continue', + targetKind: 'selector', + x: expected.x, + y: expected.y, + ...(expected.extra ?? {}), + }, + ); + assertSelectorChain(data); + assertNoWireNoise(data); +} + +function assertCanonicalDirectSelector( + data: WireData, + expected: { message: string; x: number; y: number; extra?: WireData }, +): void { + assertExactKeys(data, [ + 'message', + 'selector', + 'targetKind', + 'x', + 'y', + ...Object.keys(expected.extra ?? {}), + ]); + assert.deepEqual( + pick(data, [ + 'message', + 'selector', + 'targetKind', + 'x', + 'y', + ...Object.keys(expected.extra ?? {}), + ]), + { + message: expected.message, + selector: 'label=Continue', + targetKind: 'selector', + x: expected.x, + y: expected.y, + ...(expected.extra ?? {}), + }, + ); + assertNoWireNoise(data); +} + +function assertCanonicalPoint( + data: WireData, + expected: { message: string; x: number; y: number; extra?: WireData }, +): void { + assertExactKeys(data, [...POINT_KEYS, ...Object.keys(expected.extra ?? {})]); + assert.deepEqual( + pick(data, ['message', 'targetKind', 'x', 'y', ...Object.keys(expected.extra ?? {})]), + { + message: expected.message, + targetKind: 'point', + x: expected.x, + y: expected.y, + ...(expected.extra ?? {}), + }, + ); + assertNoWireNoise(data); +} + +function assertSelectorChain(data: WireData): void { + assert.ok(Array.isArray(data.selectorChain), 'selectorChain must be present'); + assert.ok(data.selectorChain.length > 0, 'selectorChain must be non-empty'); +} + +function assertExactKeys(data: WireData, expectedKeys: readonly string[]): void { + assert.deepEqual(Object.keys(data).sort(), [...expectedKeys].sort()); +} + +function assertNoWireNoise(data: WireData): void { + for (const key of [ + 'count', + 'doubleTap', + 'gestureEndUptimeMs', + 'gestureStartUptimeMs', + 'holdMs', + 'intervalMs', + 'jitterPx', + 'sequenceResults', + ]) { + assert.equal(data[key], undefined, `${key} should not be in the wire response`); + } +} + +function assertRunnerRequest( + transcript: ProviderScenarioTranscript, + command: string, + expected: WireData, +): void { + const call = [...transcript.calls].reverse().find((entry) => entry.command === command); + assert.ok(call, `Expected transcript to include ${command}`); + assert.deepEqual(stripUndefined(call.request as WireData), expected); +} + +function pick(data: WireData, keys: readonly string[]): WireData { + return Object.fromEntries(keys.map((key) => [key, data[key]])); +} + +function stripUndefined(data: WireData): WireData { + return Object.fromEntries(Object.entries(data).filter(([, value]) => value !== undefined)); +} + +function tapRequest(x: number, y: number): WireData { + return { command: 'tap', x, y, synthesized: true, appBundleId: CONTRACT_APP }; +} + +function typeRequest(x: number, y: number, text: string): WireData { + return { + command: 'type', + x, + y, + text, + delayMs: 0, + textEntryMode: 'replace', + appBundleId: CONTRACT_APP, + }; +} + +function selectorTypeRequest(text: string): WireData { + return { + command: 'type', + selectorKey: 'label', + selectorValue: 'Continue', + text, + delayMs: 0, + textEntryMode: 'replace', + appBundleId: CONTRACT_APP, + }; +} + +function longPressRequest(x: number, y: number, durationMs: number): WireData { + return { command: 'longPress', x, y, durationMs, appBundleId: CONTRACT_APP }; +} diff --git a/test/integration/interaction-contract/runtime-ref.contract.test.ts b/test/integration/interaction-contract/runtime-ref.contract.test.ts index ee642b066..6da162feb 100644 --- a/test/integration/interaction-contract/runtime-ref.contract.test.ts +++ b/test/integration/interaction-contract/runtime-ref.contract.test.ts @@ -130,13 +130,13 @@ test(scenario('responseConstruction'), async () => { const snapshot = await daemon.callCommand('snapshot', [], { snapshotInteractiveOnly: true }); assertRpcOk(snapshot); - const press = await daemon.callCommand('press', ['@e1']); + const press = await daemon.callCommand('press', ['@e2']); const data = assertRpcOk(press); // Canonical ref response set from the shared construction site. - assert.equal(data.ref, 'e1'); + assert.equal(data.ref, 'e2'); assert.equal(data.x, 200); assert.equal(data.y, 322); - assert.match(String(data.message), /Tapped @e1/); + assert.match(String(data.message), /Tapped @e2/); }, ); }); From bd4cdd8c034e2ffdcd1b3416180ffdcd01f8a7b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 5 Jul 2026 12:25:08 +0200 Subject: [PATCH 2/2] fix: reduce interaction response complexity --- .../handlers/__tests__/interaction.test.ts | 8 +--- .../handlers/interaction-touch-response.ts | 42 +++++++++++++------ 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index d7fd43b7f..03a6fbbe4 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -2214,7 +2214,7 @@ test('press @ref does not promote to a full-screen hittable ancestor', async () expect(mockDispatch.mock.calls[0]?.[2]).toEqual(['201', '319']); }); -test('fill @ref returns canonical wire coordinates and records fallback coordinates when platform result is sparse', async () => { +test('fill @ref preserves fallback coordinates for recording when platform result is sparse', async () => { const sessionStore = makeSessionStore(); const sessionName = 'default'; const session = makeSession(sessionName); @@ -2262,12 +2262,6 @@ test('fill @ref returns canonical wire coordinates and records fallback coordina expect(response?.ok).toBe(true); if (response?.ok) { expect(response.data?.filled).toBe(true); - expect(response.data?.targetKind).toBe('ref'); - expect(response.data?.ref).toBe('e1'); - expect(response.data?.x).toBe(60); - expect(response.data?.y).toBe(40); - expect(response.data?.text).toBe('hello@example.com'); - expect(response.data?.message).toBe('Filled 17 chars'); } const stored = sessionStore.get(sessionName); diff --git a/src/daemon/handlers/interaction-touch-response.ts b/src/daemon/handlers/interaction-touch-response.ts index 395d63a09..2504c0ecf 100644 --- a/src/daemon/handlers/interaction-touch-response.ts +++ b/src/daemon/handlers/interaction-touch-response.ts @@ -181,18 +181,36 @@ function buildTouchMessage( x: number | undefined, y: number | undefined, ): string | undefined { - if (typeof extra?.text === 'string') { - return `Filled ${Array.from(extra.text).length} chars`; - } - const ref = typeof extra?.ref === 'string' ? extra.ref : undefined; - const selector = typeof extra?.selector === 'string' ? extra.selector : undefined; - const pointSuffix = x === undefined || y === undefined ? '' : ` (${x}, ${y})`; - const label = ref ? `@${ref}` : (selector ?? ''); - if (!label) { - if (x === undefined || y === undefined) return undefined; - return extra?.gesture === 'longpress' ? `Long pressed (${x}, ${y})` : `Tapped (${x}, ${y})`; - } - return buildTouchTargetMessage(label, extra ?? {}, pointSuffix); + const fillText = readString(extra, 'text'); + if (fillText !== undefined) return `Filled ${Array.from(fillText).length} chars`; + + const pointSuffix = buildPointSuffix(x, y); + const label = buildTouchTargetLabel(extra); + if (label) return buildTouchTargetMessage(label, extra ?? {}, pointSuffix); + if (!pointSuffix) return undefined; + + return buildPointTouchMessage(extra, pointSuffix); +} + +function readString(data: Record | undefined, key: string): string | undefined { + const value = data?.[key]; + return typeof value === 'string' ? value : undefined; +} + +function buildPointSuffix(x: number | undefined, y: number | undefined): string { + return x === undefined || y === undefined ? '' : ` (${x}, ${y})`; +} + +function buildTouchTargetLabel(extra: Record | undefined): string | undefined { + const ref = readString(extra, 'ref'); + return ref === undefined ? readString(extra, 'selector') : `@${ref}`; +} + +function buildPointTouchMessage( + extra: Record | undefined, + pointSuffix: string, +): string { + return extra?.gesture === 'longpress' ? `Long pressed${pointSuffix}` : `Tapped${pointSuffix}`; } function buildTouchTargetMessage(