diff --git a/CONTEXT.md b/CONTEXT.md index 585d62a7c..ded0aea4b 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -49,6 +49,7 @@ - Coverage manifest: `CONTRACT_COVERAGE` export beside each interaction contract test file claiming which matrix cells it proves; the coverage gate requires every enforced/delegated cell to be claimed and rejects overclaims of waived cells. - Delegation-on-error: a fast path falling back to the runtime path on semantic failure shapes. It closes failure-side guarantee cells only — never success-path parity. - Ref generation pin: optional `~s` suffix on an @ref carrying the snapshot generation it was minted from. Accepted as input everywhere, emitted by no tree output (snapshot token budget), auto-appended by the MCP layer, stripped and ignored by replay. +- Settled observation: opt-in (`--settle`) post-action payload on press/click/fill/longpress — the quiet-window stable loop re-captures until the UI settles, and the response carries the diff vs the pre-action tree (changed lines only, added lines with fresh refs, `refsGeneration` when the settled tree was stored). Best-effort: never fails the action; `settled: false` plus a hint on never-quiet content. - Snapshot capture plan: per-strategy ordered chain of iOS snapshot capture backends (recursive tree, query sweep, private AX) run by one plan runner under a shared wall-clock budget; recovery ordering is declared data, never a per-call-site branch. - Snapshot quality verdict: structured outcome (state, backend, reason code, effective depth, collapsed leaves) computed once by the plan runner and shipped with every planned snapshot payload; the daemon and CLI render it instead of re-deriving degradation from node shapes. - AX-unavailable target invalidation: iOS/macOS runner behavior where a root accessibility snapshot failure such as `kAXErrorIllegalArgument` marks the cached `XCUIApplication` target handle suspect. The runner fails closed for degraded interactive snapshots, clears the cached target, and lets the next command reacquire the app through normal activation. diff --git a/docs/adr/0008-command-descriptor-registry.md b/docs/adr/0008-command-descriptor-registry.md index ceb041ad1..ed664dd75 100644 --- a/docs/adr/0008-command-descriptor-registry.md +++ b/docs/adr/0008-command-descriptor-registry.md @@ -31,6 +31,9 @@ every consumer table is **derived** by pure, parity-tested projection: - The descriptor composes a `surface` facet (owned by `src/commands/**`: identity, CLI schema/reader, MCP), a `capability` facet (owned by `src/core/capabilities`), and a `daemon` facet (route + request-policy traits, **owned under `src/daemon/`** per ADR 0003), plus a typed result. +- Narrow command traits that affect multiple projections, such as interaction post-action observation + (`--settle` / `--verify`), live on the descriptor so CLI flags, command metadata, timeout policy, and + tests derive from one fact instead of repeating command-name lists. - The public catalog, capability matrix, daemon command registry, batch allowlist, MCP tool list, CLI schema, and the Node client surface become pure projections of the descriptor set. The `src/core/dispatch.ts` `switch` is replaced by a total map keyed on the command-name union, so a missing diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index e6c870b2b..299d78a7d 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -184,7 +184,9 @@ function summarizeProviderScenarioFlagCoverage(files) { ['batchMaxSteps', 'batch max-step guard', ['maxSteps']], ['findFirst', 'find first disambiguation'], ['findLast', 'find last disambiguation'], - ['verify', 'post-action evidence capture on press/click/fill'], + ['verify', 'descriptor post-action evidence capture'], + ['settle', 'descriptor post-action settled-diff observation'], + ['settleQuietMs', 'settle quiet-window tuning'], ]; const sources = files.map((file) => fs.readFileSync(file, 'utf8')).join('\n'); return flagTargets.map(([key, reason, aliases = []]) => { diff --git a/src/cli/parser/cli-flags.ts b/src/cli/parser/cli-flags.ts index 2645690fa..2bccc5a49 100644 --- a/src/cli/parser/cli-flags.ts +++ b/src/cli/parser/cli-flags.ts @@ -106,6 +106,8 @@ export type CliFlags = CloudProviderProfileFields & pixels?: number; doubleTap?: boolean; verify?: boolean; + settle?: boolean; + settleQuietMs?: number; clickButton?: ClickButton; backMode?: BackMode; pauseMs?: number; @@ -211,6 +213,11 @@ export const REPEATED_TOUCH_FLAGS = flagKeys( 'jitterPx', 'doubleTap', ); +// Interaction commands with the descriptor post-action observation trait use +// these flags for `--settle` (#1101). --timeout doubles as the settle deadline +// (flag-sourced budget on the interaction descriptors, mirroring wait's +// positional budget). +export const SETTLE_FLAGS = flagKeys('settle', 'settleQuietMs', 'timeoutMs'); export const REPLAY_FLAGS = flagKeys('replayUpdate', 'replayEnv'); const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ @@ -848,6 +855,22 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageDescription: 'Capture cheap post-action evidence (AX digest, node counts, changedFromBefore) instead of a follow-up snapshot', }, + { + key: 'settle', + names: ['--settle'], + type: 'boolean', + usageLabel: '--settle', + usageDescription: + 'After the action, wait for the UI to go quiet and return the settled diff vs the pre-action tree in the same response (best-effort; never fails the action)', + }, + { + key: 'settleQuietMs', + names: ['--settle-quiet'], + type: 'int', + min: 0, + usageLabel: '--settle-quiet ', + usageDescription: 'Settle: quiet window the UI must hold to count as settled (default 500ms)', + }, { key: 'clickButton', names: ['--button'], @@ -1051,7 +1074,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ min: 1, usageLabel: '--timeout ', usageDescription: - 'Prepare/Replay/Snapshot/Test: maximum wall-clock time for the command or attempt', + 'Prepare/Replay/Snapshot/Test: maximum wall-clock time for the command or attempt. With --settle: the settle-wait deadline (default 10s)', }, { key: 'retries', diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index 3a5a5f3da..fcf5a0dc0 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -55,18 +55,25 @@ const AGENT_START_LINES = [ 'Default to agent-device for installs, opens, snapshots, interactions, screenshots, logs, network/perf evidence, and verification.', 'Use raw adb, simctl, xcrun, or platform scripts only when this help calls out a tool gap or platform setup step.', 'Start with agent-device help workflow to understand the core loop and how to use the tool.', + // Benchmarked 2026-07-05 (#1101): agents that only read --help skipped the + // help-workflow pointer and fell into plain-snapshot loops; stating the loop + // here is what makes small models pick snapshot -i and --settle unprompted. + 'Core loop: open -> snapshot -i (interactive tree with @refs) -> press/click/fill --settle (returns the settled diff with fresh @refs) -> repeat. Verify with diff snapshot -i.', ] as const; const AGENT_QUICKSTART_LINES = [ 'Planning output contract: when asked to plan commands, output command lines only: no prose, numbering, Markdown fences, pipes, or shell helpers.', 'Default loop: devices/apps -> open -> snapshot -i -> press/fill/get/is/wait/find -> verify with diff snapshot -> close.', 'Verify a mutation with diff snapshot (or diff snapshot -i), not a full snapshot: it prints only the added/removed/changed lines since the last snapshot in this session, so confirming an action costs a few lines instead of the whole tree.', - 'Use selectors or refs as positional targets: id="submit", label="Allow", or @e12 from snapshot -i.', + 'Collapse act+observe into one call with --settle on press/click/fill/longpress: the response waits for the UI to go quiet and carries the settled diff with fresh refs (settled: false plus a hint on never-quiet content; the action itself never fails). Tune with --settle-quiet and --timeout .', + 'Treat the --settle diff as the post-action observation: if it already shows the next target or final evidence, act on that fresh ref or answer from it instead of taking another snapshot.', + 'Network-backed or debounced results may arrive after the --settle quiet window; follow the settled action with wait text "Expected result" or wait instead of polling full snapshots.', + 'Use selectors or refs as positional targets: id="submit", label="Allow", role=button label="Send", or @e12 from snapshot -i. Do not write role names as selector keys, such as button="Search".', 'Pin a ref to the snapshot that minted it with ~s (n = refsGeneration in the snapshot response): press @e12~s4. Pinned refs get exact staleness warnings instead of the coarse tree-changed one; plain refs stay valid input.', 'Plain snapshot reads state; snapshot -i refreshes current interactive refs only.', 'Default snapshot text is an agent-facing, token-efficient view for planning and targeting actions.', 'Read-only visible/state question: use snapshot/get/is/find; use snapshot -i only when refs are needed.', - 'Anti-pattern: snapshot -i followed by snapshot -i | grep ...; prior refs stay valid until app state changes, and --force-full is the explicit full re-read.', + 'Anti-pattern: snapshot -i followed by snapshot -i | grep ... or hiding command output with 2>/dev/null | jq ...; inspect raw agent-device output first, since hints and errors are part of the planning contract.', 'Truncated text/input preview: expand first with snapshot -s @e12, not get text.', 'React Native apps: read help react-native for Metro, DevTools routing, and RN-specific blockers; use react-native dismiss-overlay for LogBox/RedBox overlays.', 'React Native JS memory leaks: read help cdp; use heap usage samples for a quick signal, then snapshot diff/leak-triplet for retained object proof.', @@ -142,6 +149,7 @@ Fresh machine or first iOS run: Command shape: Plans should use agent-device commands, not raw platform tools, pseudo commands, package-manager aliases, or helper prose. If the user asks for a command plan, final output should be command lines only: no intro sentence, numbered list, Markdown fence, shell pipe, grep/head/tail helper, or explanatory bullets. + While exploring, do not pipe or redirect agent-device output through jq/grep/head/tail or 2>/dev/null; raw output carries refs, warnings, hints, and diagnostics needed for the next step. Put subcommand first, then positionals, then flags: agent-device open com.example.app --session checkout --platform android --relaunch agent-device record start ./checkout.mp4 --session checkout @@ -175,8 +183,9 @@ Snapshots and refs: @e14 [cell] label="Profiles" focused -> tvOS focus is currently on this row. [off-screen below] 4 items: "Privacy", "About" -> scroll down, then snapshot -i; those are hints, not refs. Re-snapshot after navigation, submit, typing/fill, modal/list/reload/dynamic changes when you need new refs. - Anti-pattern: snapshot -i followed by snapshot -i | grep ... + Anti-pattern: snapshot -i followed by snapshot -i | grep ..., or adding 2>/dev/null | jq ... before reading the raw command output. Refs from the first snapshot remain valid until you press, click, fill, type, scroll, go back, wait for async UI, or otherwise change app state. + A --settle response is the observation for that mutation. If its diff already shows the next target or final evidence, use that fresh ref or answer from the diff instead of taking a follow-up snapshot. After a mutation, prefer a known selector/label directly (for example press 'label="Send"') because interaction commands refresh interactive state internally. If you need to discover the new control, use snapshot -i, or snapshot -i -s "Composer" when a stable container label/id can scope the refresh. If typing/fill opened the keyboard or changed layout and the next target has no stable selector, run snapshot -i, use the fresh ref, then verify with wait/find or diff snapshot -i. For a targeted query, use find/get/is. If you truly need the full tree again, pass --force-full. @@ -187,6 +196,7 @@ Snapshots and refs: Selectors: Use selectors as positional targets: id="field-email" or label="Allow". + Selector terms are key=value filters such as id="submit", label="Search", text="Search", or role=button label="Search". Do not write role names as keys, such as button="Search"; use role=button label="Search" or the @ref from the latest snapshot/settle diff. Do not use CSS selectors, pseudo refs, --selector, --text, or raw x/y when refs/selectors exist. agent-device fill 'id="catalog-search"' "tart" --delay-ms 80 agent-device press 'id="submit-order"' @@ -223,6 +233,7 @@ Read-only and waits: agent-device wait 'label="Ready"' 3000 agent-device wait stable agent-device wait stable 500 10000 + For network-backed search/typeahead, --settle confirms the local UI quieted after fill/press; use wait text "Expected result" or wait for server-loaded content that can arrive later. agent-device find "Increment" press --json For async/list text presence, prefer wait text over is visible when no interaction is needed. wait stable [quietMs] [timeoutMs] (defaults 500/10000) replaces guessed sleeps between actions and snapshots in the core loop: it polls the interactive-only tree and resolves once two or more consecutive captures are unchanged for quietMs, or fails with the standard wait-timeout shape plus the last capture stats (captures, nodeCount) on timeout; use it after relaunch/navigation instead of a fixed wait before the next snapshot or action. diff --git a/src/client/client-normalizers.ts b/src/client/client-normalizers.ts index f47dc0dfa..f9000dbcb 100644 --- a/src/client/client-normalizers.ts +++ b/src/client/client-normalizers.ts @@ -343,6 +343,8 @@ export function buildFlags(options: InternalRequestOptions): CommandFlags { pixels: options.pixels, doubleTap: options.doubleTap, verify: options.verify, + settle: options.settle, + settleQuietMs: options.settleQuietMs, clickButton: options.clickButton, pauseMs: options.pauseMs, pattern: options.pattern, diff --git a/src/client/client-types.ts b/src/client/client-types.ts index c253a71ad..157564c72 100644 --- a/src/client/client-types.ts +++ b/src/client/client-types.ts @@ -629,10 +629,25 @@ export type CaptureDiffOptions = DeviceCommandBaseOptions & out?: string; }; +/** + * Opt-in (#1101): after the action, wait for the UI to go quiet and return the + * settled diff vs the pre-action tree (`settle` on the result) in the same + * response. Best-effort — never fails the action. `settleQuietMs` tunes the + * quiet window (default 500ms); `timeoutMs` bounds the settle wait (default + * 10s) when `settle` is true. A bare `timeoutMs` without `settle` is ignored + * for compatibility; `settleQuietMs` still requires `settle`. + */ +type SettleCommandOptions = { + settle?: boolean; + settleQuietMs?: number; + timeoutMs?: number; +}; + export type ClickOptions = DeviceCommandBaseOptions & SelectorSnapshotCommandOptions & InteractionTarget & - RepeatedPressOptions & { + RepeatedPressOptions & + SettleCommandOptions & { button?: ClickButton; /** * Opt-in (#1047): return cheap post-action evidence (AX digest, node counts, @@ -645,13 +660,15 @@ export type ClickOptions = DeviceCommandBaseOptions & export type PressOptions = DeviceCommandBaseOptions & SelectorSnapshotCommandOptions & InteractionTarget & - RepeatedPressOptions & { + RepeatedPressOptions & + SettleCommandOptions & { verify?: boolean; }; export type LongPressOptions = DeviceCommandBaseOptions & SelectorSnapshotCommandOptions & - InteractionTarget & { + InteractionTarget & + SettleCommandOptions & { durationMs?: number; }; @@ -697,7 +714,8 @@ export type TypeTextOptions = DeviceCommandBaseOptions & { export type FillOptions = DeviceCommandBaseOptions & SelectorSnapshotCommandOptions & - InteractionTarget & { + InteractionTarget & + SettleCommandOptions & { text: string; delayMs?: number; verify?: boolean; @@ -928,6 +946,8 @@ type CommandExecutionOptions = Partial & { pixels?: number; doubleTap?: boolean; verify?: boolean; + settle?: boolean; + settleQuietMs?: number; clickButton?: ClickButton; pauseMs?: number; pattern?: SwipePattern; diff --git a/src/commands/cli-grammar/common.ts b/src/commands/cli-grammar/common.ts index 16972dd21..eadccb283 100644 --- a/src/commands/cli-grammar/common.ts +++ b/src/commands/cli-grammar/common.ts @@ -88,6 +88,17 @@ export function selectorSnapshotOptionsFromFlags(flags: CliFlags): SelectorSnaps }; } +// Descriptor post-action observation commands use --settle (#1101). +// --timeout doubles as the settle deadline only when --settle is present; a +// bare --timeout stays compatible and is ignored by touch commands. +export function settleInputFromFlags(flags: CliFlags): Record { + return compactRecord({ + settle: flags.settle, + settleQuietMs: flags.settleQuietMs, + timeoutMs: flags.timeoutMs, + }); +} + export function repeatedInputFromFlags(flags: CliFlags): Record { return compactRecord({ count: flags.count, diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index 6ae015fb3..7846504a0 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -18,7 +18,16 @@ import type { TypeTextOptions, } from '../../client/client-types.ts'; import type { CommandSchemaOverride } from '../../utils/cli-command-schema-types.ts'; -import { REPEATED_TOUCH_FLAGS, SELECTOR_SNAPSHOT_FLAGS } from '../../cli/parser/cli-flags.ts'; +import { + REPEATED_TOUCH_FLAGS, + SELECTOR_SNAPSHOT_FLAGS, + SETTLE_FLAGS, + type FlagKey, +} from '../../cli/parser/cli-flags.ts'; +import { + commandSupportsSettleObservation, + commandSupportsVerifyEvidence, +} from '../../core/command-descriptor/registry.ts'; import { defineCommandFacet, defineCommandFamilyFromFacets } from '../family/types.ts'; import { defineExecutableCommand } from '../command-contract.ts'; import { @@ -70,7 +79,12 @@ const interactionCliSchemas = { usageOverride: 'click ', positionalArgs: ['target'], allowsExtraPositionals: true, - allowedFlags: [...REPEATED_TOUCH_FLAGS, 'clickButton', 'verify', ...SELECTOR_SNAPSHOT_FLAGS], + allowedFlags: [ + ...REPEATED_TOUCH_FLAGS, + 'clickButton', + ...postActionObservationCliFlags('click'), + ...SELECTOR_SNAPSHOT_FLAGS, + ], }, press: { usageOverride: 'press ', @@ -78,7 +92,11 @@ const interactionCliSchemas = { 'Short press a semantic UI target by ref, selector, or point. For native context menus or hold gestures, use longpress instead of press --hold-ms.', positionalArgs: ['targetOrX', 'y?'], allowsExtraPositionals: true, - allowedFlags: [...REPEATED_TOUCH_FLAGS, 'verify', ...SELECTOR_SNAPSHOT_FLAGS], + allowedFlags: [ + ...REPEATED_TOUCH_FLAGS, + ...postActionObservationCliFlags('press'), + ...SELECTOR_SNAPSHOT_FLAGS, + ], }, longpress: { usageOverride: 'longpress [durationMs]', @@ -86,7 +104,7 @@ const interactionCliSchemas = { 'Open native context menus or long-press targets by ref, selector, or point. Duration is positional, for example longpress @e12 800 or longpress 300 500 800.', positionalArgs: ['targetOrX', 'yOrDurationMs?', 'durationMs?'], allowsExtraPositionals: true, - allowedFlags: [...SELECTOR_SNAPSHOT_FLAGS], + allowedFlags: [...postActionObservationCliFlags('longpress'), ...SELECTOR_SNAPSHOT_FLAGS], }, swipe: { helpDescription: 'Swipe coordinates with optional repeat pattern', @@ -114,7 +132,7 @@ const interactionCliSchemas = { usageOverride: 'fill | fill <@ref|selector> ', positionalArgs: ['targetOrX', 'yOrText', 'text?'], allowsExtraPositionals: true, - allowedFlags: [...SELECTOR_SNAPSHOT_FLAGS, 'delayMs', 'verify'], + allowedFlags: [...SELECTOR_SNAPSHOT_FLAGS, 'delayMs', ...postActionObservationCliFlags('fill')], }, scroll: { usageOverride: 'scroll [amount] [--pixels ] [--duration-ms ]', @@ -130,6 +148,13 @@ type InteractionCommandName = InteractionCommandMetadata['name']; const { gesture: _gestureDaemonWriter, ...gestureProjectionAliasDaemonWriters } = gestureDaemonWriters; +function postActionObservationCliFlags(command: InteractionCommandName): readonly FlagKey[] { + const flags: FlagKey[] = []; + if (commandSupportsVerifyEvidence(command)) flags.push('verify'); + if (commandSupportsSettleObservation(command)) flags.push(...SETTLE_FLAGS); + return flags; +} + const clickCommandDefinition = defineExecutableCommand(metadata('click'), (client, input) => client.interactions.click(toClickOptions(input)), ); @@ -221,6 +246,7 @@ const fillCommandFacet = defineCommandFacet({ cliSchema: interactionCliSchemas.fill, cliReader: interactionCliReaders.fill, daemonWriter: interactionDaemonWriters.fill, + cliOutputFormatter: interactionCliOutputFormatters.fill, }); const longPressCommandFacet = defineCommandFacet({ @@ -230,6 +256,7 @@ const longPressCommandFacet = defineCommandFacet({ cliSchema: interactionCliSchemas.longpress, cliReader: interactionCliReaders.longpress, daemonWriter: interactionDaemonWriters.longpress, + cliOutputFormatter: interactionCliOutputFormatters.longpress, }); const swipeCommandFacet = defineCommandFacet({ @@ -343,6 +370,7 @@ function toClickOptions(input: ClickInput): ClickOptions { ...toRepeatedOptions(input), button: input.button, verify: input.verify, + ...toSettleOptions(input), }; } @@ -353,6 +381,7 @@ function toPressOptions(input: PressInput): PressOptions { ...toSelectorSnapshotOptions(input), ...toRepeatedOptions(input), verify: input.verify, + ...toSettleOptions(input), }; } @@ -364,6 +393,7 @@ function toFillOptions(input: FillInput): FillOptions { text: input.text, delayMs: input.delayMs, verify: input.verify, + ...toSettleOptions(input), }; } @@ -373,6 +403,19 @@ function toLongPressOptions(input: LongPressInput): LongPressOptions { ...toClientInteractionTarget(input.target), ...toSelectorSnapshotOptions(input), durationMs: input.durationMs, + ...toSettleOptions(input), + }; +} + +function toSettleOptions(input: { + settle?: boolean; + settleQuietMs?: number; + timeoutMs?: number; +}): Pick { + return { + settle: input.settle, + settleQuietMs: input.settleQuietMs, + timeoutMs: input.timeoutMs, }; } diff --git a/src/commands/interaction/interactions.ts b/src/commands/interaction/interactions.ts index f5a145339..26afc7b9b 100644 --- a/src/commands/interaction/interactions.ts +++ b/src/commands/interaction/interactions.ts @@ -26,6 +26,7 @@ import { requiredDaemonString, repeatedInputFromFlags, selectorSnapshotInputFromFlags, + settleInputFromFlags, targetInputFromClientTarget, } from '../cli-grammar/common.ts'; import type { CliReader, DaemonWriter, CommandInput } from '../cli-grammar/types.ts'; @@ -35,6 +36,7 @@ export const interactionCliReaders = { ...commonInputFromFlags(flags), ...selectorSnapshotInputFromFlags(flags), ...repeatedInputFromFlags(flags), + ...settleInputFromFlags(flags), target: targetInputFromClientTarget(readInteractionTargetFromPositionals(positionals)), button: flags.clickButton, verify: flags.verify, @@ -43,6 +45,7 @@ export const interactionCliReaders = { ...commonInputFromFlags(flags), ...selectorSnapshotInputFromFlags(flags), ...repeatedInputFromFlags(flags), + ...settleInputFromFlags(flags), target: targetInputFromClientTarget(readInteractionTargetFromPositionals(positionals)), verify: flags.verify, }), @@ -51,6 +54,7 @@ export const interactionCliReaders = { return { ...commonInputFromFlags(flags), ...selectorSnapshotInputFromFlags(flags), + ...settleInputFromFlags(flags), target: targetInputFromClientTarget(decoded), durationMs: decoded.durationMs, }; @@ -79,6 +83,7 @@ export const interactionCliReaders = { return { ...commonInputFromFlags(flags), ...selectorSnapshotInputFromFlags(flags), + ...settleInputFromFlags(flags), target: targetInputFromClientTarget(decoded.target), text: decoded.text, delayMs: flags.delayMs, diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index ad322f6af..757184d4c 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -37,6 +37,11 @@ import { } from '../../core/scroll-gesture.ts'; import { SCROLL_INPUT_DIRECTIONS } from './runtime/gestures.ts'; import { FIND_LOCATORS } from '../../utils/finders.ts'; +import { + commandSupportsSettleObservation, + commandSupportsVerifyEvidence, +} from '../../core/command-descriptor/registry.ts'; +import type { PostActionObservationSupportFor } from '../../core/command-descriptor/post-action-observation.ts'; const FIND_ACTION_VALUES = [ 'click', @@ -71,19 +76,45 @@ const verifyField = () => 'Capture cheap post-action evidence (AX digest, node counts, changedFromBefore) instead of a follow-up snapshot.', ); +const settleFields = () => ({ + settle: booleanField( + 'After the action, wait for the UI to go quiet and return the settled diff vs the pre-action tree in the same response. Best-effort; never fails the action.', + ), + settleQuietMs: integerField('Settle: quiet window in milliseconds (default 500).', { min: 0 }), + timeoutMs: integerField('Settle: wait deadline in milliseconds (default 10000).', { min: 1 }), +}); + +type VerifyFieldMap = { verify: ReturnType }; +type SettleFieldMap = ReturnType; +type PostActionObservationFields = + PostActionObservationSupportFor extends 'settle-and-verify' + ? VerifyFieldMap & SettleFieldMap + : PostActionObservationSupportFor extends 'settle' + ? SettleFieldMap + : {}; + +function postActionObservationFields( + command: TName, +): PostActionObservationFields { + return { + ...(commandSupportsVerifyEvidence(command) ? { verify: verifyField() } : {}), + ...(commandSupportsSettleObservation(command) ? settleFields() : {}), + } as PostActionObservationFields; +} + const clickFields = { target: requiredField(interactionTargetField()), button: enumField(CLICK_BUTTONS, 'Pointer button for platforms that support mouse buttons.'), ...selectorSnapshotFields(), ...repeatedFields(), - verify: verifyField(), + ...postActionObservationFields('click'), }; const pressFields = { target: requiredField(interactionTargetField()), ...selectorSnapshotFields(), ...repeatedFields(), - verify: verifyField(), + ...postActionObservationFields('press'), }; const fillFields = { @@ -91,13 +122,14 @@ const fillFields = { text: requiredField(stringField('Text to enter into the target.')), delayMs: integerField('Delay between typed characters.', { min: 0 }), ...selectorSnapshotFields(), - verify: verifyField(), + ...postActionObservationFields('fill'), }; const longPressFields = { target: requiredField(interactionTargetField()), durationMs: integerField('Long press duration in milliseconds.', { min: 0 }), ...selectorSnapshotFields(), + ...postActionObservationFields('longpress'), }; const swipeFields = { diff --git a/src/commands/interaction/output.test.ts b/src/commands/interaction/output.test.ts index f1c9c1b5e..6ed5652b1 100644 --- a/src/commands/interaction/output.test.ts +++ b/src/commands/interaction/output.test.ts @@ -4,6 +4,15 @@ import { interactionCliOutputFormatters } from './output.ts'; const formatFind = (result: Record) => interactionCliOutputFormatters.find({ input: {}, result }); +const formatPress = (result: Record) => + interactionCliOutputFormatters.press({ input: {}, result }); + +const formatFill = (result: Record) => + interactionCliOutputFormatters.fill({ input: {}, result }); + +const formatLongPress = (result: Record) => + interactionCliOutputFormatters.longpress({ input: {}, result }); + describe('find CLI output', () => { test('click prints the same success line as a direct press', () => { const output = formatFind({ @@ -42,3 +51,97 @@ describe('find CLI output', () => { expect(output.text).toBe('Found: true'); }); }); + +describe('press CLI output', () => { + test('appends settle verdict and diff lines for selector/coordinate tap messages', () => { + const output = formatPress({ + message: 'Tapped (278, 817)', + x: 278, + y: 817, + settle: { + settled: true, + waitedMs: 1200, + diff: { + summary: { additions: 1, removals: 1, unchanged: 8 }, + lines: [ + { kind: 'removed', text: '@e4 [button] "Search"' }, + { kind: 'added', text: '@e9 [text] "Notifications"' }, + ], + }, + }, + }); + + expect(output.text).toBe( + [ + 'Tapped (278, 817)', + 'settled after 1200ms: +1 -1 (~8 unchanged)', + '- @e4 [button] "Search"', + '+ @e9 [text] "Notifications"', + ].join('\n'), + ); + }); +}); + +describe('fill CLI output', () => { + test('prints the fill success message without settle details by default', () => { + const output = formatFill({ + text: 'alpenglow', + message: 'Filled 9 chars', + }); + + expect(output.text).toBe('Filled 9 chars'); + }); + + test('appends settle verdict and diff lines when present', () => { + const output = formatFill({ + text: 'alpenglow', + message: 'Filled 9 chars', + settle: { + settled: true, + waitedMs: 750, + diff: { + summary: { additions: 2, removals: 1, unchanged: 4 }, + lines: [ + { kind: 'removed', text: '@e4 [text-field] "Search"' }, + { kind: 'added', text: '@e23 [text-field] "alpenglow"' }, + { kind: 'added', text: '@e31 [static-text] "Alpenglow"' }, + ], + }, + }, + }); + + expect(output.text).toBe( + [ + 'Filled 9 chars', + 'settled after 750ms: +2 -1 (~4 unchanged)', + '- @e4 [text-field] "Search"', + '+ @e23 [text-field] "alpenglow"', + '+ @e31 [static-text] "Alpenglow"', + ].join('\n'), + ); + }); +}); + +describe('longpress CLI output', () => { + test('appends settle verdict and diff lines when present', () => { + const output = formatLongPress({ + message: 'Long pressed (60, 40)', + settle: { + settled: true, + waitedMs: 600, + diff: { + summary: { additions: 1, removals: 0, unchanged: 6 }, + lines: [{ kind: 'added', text: '@e12 [button] "Copy"' }], + }, + }, + }); + + expect(output.text).toBe( + [ + 'Long pressed (60, 40)', + 'settled after 600ms: +1 -0 (~6 unchanged)', + '+ @e12 [button] "Copy"', + ].join('\n'), + ); + }); +}); diff --git a/src/commands/interaction/output.ts b/src/commands/interaction/output.ts index 383775f3e..95d28945f 100644 --- a/src/commands/interaction/output.ts +++ b/src/commands/interaction/output.ts @@ -37,14 +37,68 @@ function tapCliOutput(result: CommandRequestResult): CliOutput { const x = data.x; const y = data.y; if (!ref || typeof x !== 'number' || typeof y !== 'number') { - return defaultCommandCliOutput(data); + const output = defaultCommandCliOutput(data); + return { data: output.data, text: appendSettleText(output.text, data.settle) }; } - return { data, text: `Tapped @${ref} (${x}, ${y})` }; + return { data, text: appendSettleText(`Tapped @${ref} (${x}, ${y})`, data.settle) }; +} + +function messageWithSettleCliOutput(result: CommandRequestResult): CliOutput { + const data = result as Record; + const output = defaultCommandCliOutput(data); + return { data: output.data, text: appendSettleText(output.text, data.settle) }; +} + +function appendSettleText(text: string | null | undefined, settle: unknown): string { + return `${text ?? ''}${formatSettleText(settle)}`; +} + +type SettleTextView = { + settled?: boolean; + waitedMs?: number; + hint?: string; + diff?: { + summary?: { additions?: number; removals?: number; unchanged?: number }; + lines?: Array<{ kind?: string; text?: string }>; + truncated?: boolean; + }; +}; + +/** + * Compact `--settle` (#1101) rendering appended to the tap line: the verdict, + * the changed-count summary, and the changed lines themselves (the payload the + * agent acts on). Empty for non-settle responses. + */ +function formatSettleText(settle: unknown): string { + if (!settle || typeof settle !== 'object') return ''; + const view = settle as SettleTextView; + const parts = [formatSettleVerdict(view), ...formatSettleDiffLines(view.diff)]; + if (view.hint) parts.push(`hint: ${view.hint}`); + return `\n${parts.join('\n')}`; +} + +function formatSettleDiffLines(diff: SettleTextView['diff']): string[] { + const lines = (diff?.lines ?? []).map( + (line) => `${line.kind === 'removed' ? '-' : '+'} ${line.text ?? ''}`, + ); + if (diff?.truncated) lines.push('… changed lines truncated'); + return lines; +} + +function formatSettleVerdict(view: SettleTextView): string { + const verdict = view.settled === true ? 'settled' : 'not settled'; + const summary = view.diff?.summary; + const counts = summary + ? ` +${summary.additions ?? 0} -${summary.removals ?? 0} (~${summary.unchanged ?? 0} unchanged)` + : ''; + return `${verdict} after ${view.waitedMs ?? 0}ms:${counts}`; } export const interactionCliOutputFormatters = { click: resultOutput(tapCliOutput), press: resultOutput(tapCliOutput), + fill: resultOutput(messageWithSettleCliOutput), + longpress: resultOutput(messageWithSettleCliOutput), get: ({ input, result }) => getCliOutput({ result: result as CommandRequestResult, diff --git a/src/commands/interaction/runtime/gestures.test.ts b/src/commands/interaction/runtime/gestures.test.ts index e6c957771..4fff1584d 100644 --- a/src/commands/interaction/runtime/gestures.test.ts +++ b/src/commands/interaction/runtime/gestures.test.ts @@ -38,6 +38,44 @@ test('runtime focus and longPress share selector/ref target resolution', async ( ]); }); +test('runtime longPress with settle drops the non-hittable hint when the diff proves a change', async () => { + let captureCount = 0; + const nonHittableSnapshot = selectorSnapshot(); + const nonHittableNode = nonHittableSnapshot.nodes[0]; + assert.ok(nonHittableNode); + nonHittableSnapshot.nodes[0] = { + ...nonHittableNode, + hittable: false, + }; + const changedSnapshot = selectorSnapshot(); + const changedNode = changedSnapshot.nodes[0]; + assert.ok(changedNode); + changedSnapshot.nodes[0] = { + ...changedNode, + label: 'Context menu', + value: undefined, + hittable: true, + }; + const device = createInteractionDevice(nonHittableSnapshot, { + captureSnapshot: async () => { + captureCount += 1; + return { snapshot: captureCount === 1 ? nonHittableSnapshot : changedSnapshot }; + }, + longPress: async () => ({ ok: true }), + }); + + const result = await device.interactions.longPress(selector('label=Continue'), { + session: 'default', + settle: { quietMs: 25, timeoutMs: 2_000 }, + }); + + assert.equal(result.kind, 'selector'); + if (result.kind !== 'selector') return; + assert.equal(result.targetHittable, false); + assert.deepEqual(result.settle?.diff?.summary, { additions: 1, removals: 1, unchanged: 0 }); + assert.equal('hint' in result, false); +}); + test('runtime scroll resolves selector targets before calling the backend primitive', async () => { const calls: unknown[] = []; const device = createInteractionDevice(selectorSnapshot(), { diff --git a/src/commands/interaction/runtime/gestures.ts b/src/commands/interaction/runtime/gestures.ts index 987dbecc2..87813c4b8 100644 --- a/src/commands/interaction/runtime/gestures.ts +++ b/src/commands/interaction/runtime/gestures.ts @@ -40,6 +40,11 @@ import { type ResolvedInteractionTarget, resolveInteractionTarget, } from './resolution.ts'; +import { + applyPostActionObservation, + planPostActionObservation, + type SettlePostActionObservationOptions, +} from './post-action-observation.ts'; export type FocusCommandOptions = CommandContext & { target: InteractionTarget; @@ -50,7 +55,7 @@ export type FocusCommandResult = ResolvedInteractionTarget & BackendResultEnvelo export type LongPressCommandOptions = CommandContext & { target: InteractionTarget; durationMs?: number; -}; +} & SettlePostActionObservationOptions; export type { LongPressCommandResult }; @@ -155,10 +160,12 @@ export const longPressCommand: RuntimeCommand< LongPressCommandOptions, LongPressCommandResult > = async (runtime, options): Promise => { + const observation = planPostActionObservation(options); const resolved = await resolveInteractionTarget(runtime, options, { action: 'longPress', requireInteractive: true, promoteToHittableAncestor: true, + captureEvidenceBaseline: observation.needsPreActionBaseline, }); if (!runtime.backend.longPress) { throw new AppError('UNSUPPORTED_OPERATION', 'longPress is not supported by this backend'); @@ -172,12 +179,18 @@ export const longPressCommand: RuntimeCommand< durationMs, }); const formattedBackendResult = toBackendResult(backendResult); - return { - ...resolved, - ...(durationMs !== undefined ? { durationMs } : {}), - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), - ...successText(`Long pressed (${point.x}, ${point.y})`), - }; + return await applyPostActionObservation( + runtime, + options, + resolved, + { + ...resolved, + ...(durationMs !== undefined ? { durationMs } : {}), + ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + ...successText(`Long pressed (${point.x}, ${point.y})`), + }, + observation, + ); }; export const scrollCommand: RuntimeCommand = async ( diff --git a/src/commands/interaction/runtime/interactions.test.ts b/src/commands/interaction/runtime/interactions.test.ts index 7f7e7e98d..4e35d61a7 100644 --- a/src/commands/interaction/runtime/interactions.test.ts +++ b/src/commands/interaction/runtime/interactions.test.ts @@ -338,6 +338,48 @@ test('runtime press with verify drops the non-hittable hint when evidence proves assert.equal('hint' in result, false); }); +test('runtime press with settle drops the non-hittable hint when the diff proves a change', async () => { + const nonHittableSnapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: false, + }, + ]); + const changedSnapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'StaticText', + label: 'Welcome', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: true, + }, + ]); + let captureCount = 0; + const device = createInteractionDevice(nonHittableSnapshot, { + captureSnapshot: async () => { + captureCount += 1; + return { snapshot: captureCount === 1 ? nonHittableSnapshot : changedSnapshot }; + }, + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + settle: { quietMs: 25, timeoutMs: 2_000 }, + }); + + assert.equal(result.kind, 'selector'); + if (result.kind !== 'selector') return; + assert.equal(result.targetHittable, false); + assert.deepEqual(result.settle?.diff?.summary, { additions: 1, removals: 1, unchanged: 0 }); + assert.equal('hint' in result, false); +}); + test('runtime press keeps the non-hittable hint when evidence shows no change', async () => { const nonHittableSnapshot = makeSnapshotState([ { diff --git a/src/commands/interaction/runtime/interactions.ts b/src/commands/interaction/runtime/interactions.ts index 4f3d87e55..349ffe049 100644 --- a/src/commands/interaction/runtime/interactions.ts +++ b/src/commands/interaction/runtime/interactions.ts @@ -2,16 +2,13 @@ import { AppError } from '../../../kernel/errors.ts'; import type { ClickButton } from '../../../core/click-button.ts'; import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; import { isFillableType } from '../../../snapshot/snapshot-processing.ts'; -import type { Point, SnapshotNode } from '../../../kernel/snapshot.ts'; +import type { Point } from '../../../kernel/snapshot.ts'; import { requireIntInRange } from '../../../utils/validation.ts'; import { successText } from '../../../utils/success-text.ts'; import { findMistargetedTypeRefToken } from '../../../utils/type-target-warning.ts'; -import { summarizeAxEvidence } from '../../../utils/ax-digest.ts'; import type { FillCommandResult, - InteractionEvidence, PressCommandResult, - ResolvedInteractionTarget, ResolvedTarget, } from '../../../contracts/interaction.ts'; import { toBackendContext } from '../../runtime-common.ts'; @@ -22,11 +19,15 @@ import { } from '../../runtime-types.ts'; import type { RepeatedInput } from '../../command-input.ts'; import { - captureInteractionSnapshot, type InteractionTarget, preflightNativeRefInteraction, resolveInteractionTarget, } from './resolution.ts'; +import { + applyPostActionObservation, + planPostActionObservation, + type PostActionObservationOptions, +} from './post-action-observation.ts'; export { focusCommand, @@ -56,13 +57,7 @@ export type PressCommandOptions = CommandContext & RepeatedInput & { target: InteractionTarget; button?: ClickButton; - /** - * Opt-in (#1047): take one post-action interactive-only capture, digest it, - * and return it as `evidence` instead of the caller having to spend a full - * follow-up snapshot round trip to confirm the action had an effect. - */ - verify?: boolean; - }; + } & PostActionObservationOptions; export type ClickCommandOptions = PressCommandOptions; @@ -72,8 +67,7 @@ export type FillCommandOptions = CommandContext & { target: InteractionTarget; text: string; delayMs?: number; - verify?: boolean; -}; +} & PostActionObservationOptions; export type TypeTextCommandOptions = CommandContext & { text: string; @@ -101,15 +95,17 @@ export const fillCommand: RuntimeCommand options, ): Promise => { if (!options.text) throw new AppError('INVALID_ARGS', 'fill requires text'); - const verify = options.verify === true; - const nativeRefFill = verify ? null : await maybeFillRefTarget(runtime, options); + const observation = planPostActionObservation(options); + const nativeRefFill = observation.needsPreActionBaseline + ? null + : await maybeFillRefTarget(runtime, options); if (nativeRefFill) return nativeRefFill; const resolved = await resolveInteractionTarget(runtime, options, { action: 'fill', requireInteractive: true, promoteToHittableAncestor: false, - captureEvidenceBaseline: verify, + captureEvidenceBaseline: observation.needsPreActionBaseline, }); if (!runtime.backend.fill) { throw new AppError('UNSUPPORTED_OPERATION', 'fill is not supported by this backend'); @@ -127,14 +123,18 @@ export const fillCommand: RuntimeCommand nodeType && !isFillableType(nodeType, runtime.backend.platform) ? `fill target ${formatTargetForWarning(resolved)} resolved to "${nodeType}", attempting fill anyway.` : undefined; - const evidence = verify ? await captureVerifyEvidence(runtime, options, resolved) : undefined; - return reconcileNonHittableHintWithEvidence({ - ...resolved, - text: options.text, - ...(warning ? { warning } : {}), - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), - ...(evidence ? { evidence } : {}), - }); + return await applyPostActionObservation( + runtime, + options, + resolved, + { + ...resolved, + text: options.text, + ...(warning ? { warning } : {}), + ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + }, + observation, + ); }; export const typeTextCommand: RuntimeCommand< @@ -175,15 +175,17 @@ async function tapCommand( options: PressCommandOptions, action: 'click' | 'press', ): Promise { - const verify = options.verify === true; - const nativeRefTap = verify ? null : await maybeTapRefTarget(runtime, options, action); + const observation = planPostActionObservation(options); + const nativeRefTap = observation.needsPreActionBaseline + ? null + : await maybeTapRefTarget(runtime, options, action); if (nativeRefTap) return nativeRefTap; const resolved = await resolveInteractionTarget(runtime, options, { action, requireInteractive: true, promoteToHittableAncestor: true, - captureEvidenceBaseline: verify, + captureEvidenceBaseline: observation.needsPreActionBaseline, }); if (!runtime.backend.tap) { throw new AppError('UNSUPPORTED_OPERATION', 'tap is not supported by this backend'); @@ -198,63 +200,16 @@ async function tapCommand( doubleTap: options.doubleTap, }); const formattedBackendResult = toBackendResult(backendResult); - const evidence = verify ? await captureVerifyEvidence(runtime, options, resolved) : undefined; - return reconcileNonHittableHintWithEvidence({ - ...resolved, - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), - ...(evidence ? { evidence } : {}), - }); -} - -/** - * Post-action side of `--verify` (#1047): one interactive-only capture through - * the same capture helper the resolution path already uses, digested and then - * discarded — the node tree itself is never attached to the result, only the - * cheap summary. Best-effort: a failed capture must not turn a successful - * action into a failure, so this returns `undefined` instead of throwing. - */ -async function captureVerifyEvidence( - runtime: AgentDeviceRuntime, - options: CommandContext, - resolved: ResolvedInteractionTarget, -): Promise { - const preActionNodes: SnapshotNode[] | undefined = - 'preActionNodes' in resolved ? resolved.preActionNodes : undefined; - try { - const capture = await captureInteractionSnapshot(runtime, options, true); - const after = summarizeAxEvidence(capture.snapshot.nodes); - // No pre-action baseline (for example the baseline capture itself failed) - // means we cannot claim a change happened — default to false rather than - // asserting a change we did not actually observe. - const changedFromBefore = - preActionNodes !== undefined && after.digest !== summarizeAxEvidence(preActionNodes).digest; - return { ...after, changedFromBefore }; - } catch { - return undefined; - } -} - -// The resolution-time non-hittable hint warns the action "may have had no -// visible effect". When --verify evidence proves the interactive tree changed, -// that warning is contradicted by data sitting next to it in the same response -// — drop it and let targetHittable + evidence speak for themselves. -function reconcileNonHittableHintWithEvidence(result: T): T { - // Widened view: point-target results carry none of these fields, which is - // exactly the no-op path. - const view = result as { - targetHittable?: boolean; - hint?: string; - evidence?: InteractionEvidence; - }; - if ( - view.targetHittable !== false || - view.evidence?.changedFromBefore !== true || - view.hint === undefined - ) { - return result; - } - const { hint: _hint, ...rest } = view; - return rest as T; + return await applyPostActionObservation( + runtime, + options, + resolved, + { + ...resolved, + ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + }, + observation, + ); } function requireResolvedPoint(result: { point?: Point }): Point { diff --git a/src/commands/interaction/runtime/post-action-observation.ts b/src/commands/interaction/runtime/post-action-observation.ts new file mode 100644 index 000000000..a070294a6 --- /dev/null +++ b/src/commands/interaction/runtime/post-action-observation.ts @@ -0,0 +1,155 @@ +import type { SnapshotNode } from '../../../kernel/snapshot.ts'; +import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; +import { summarizeAxEvidence } from '../../../utils/ax-digest.ts'; +import type { + InteractionEvidence, + ResolvedInteractionTarget, + SettleObservation, + SettleParams, +} from '../../../contracts/interaction.ts'; +import { captureInteractionSnapshot } from './resolution.ts'; +import { settleAfterInteraction, settleEvidence } from './settle.ts'; + +type ObservedResult = T & { + evidence?: InteractionEvidence; + settle?: SettleObservation; +}; + +export type PostActionObservationOptions = { + /** + * Opt-in (#1047): take one post-action interactive-only capture, digest it, + * and return it as `evidence` instead of forcing a follow-up snapshot. + */ + verify?: boolean; + /** + * Opt-in (#1101): after the action, wait for the UI to go quiet and return + * the settled diff vs the pre-action tree in the same response. + */ + settle?: SettleParams; +}; + +export type SettlePostActionObservationOptions = Pick; + +export type PostActionObservationPlan = { + verify: boolean; + /** + * Verify and settle compare against the pre-action tree, so callers must use + * the resolution path instead of native ref fast paths. + */ + needsPreActionBaseline: boolean; + settle?: SettleParams; +}; + +export function planPostActionObservation( + options: PostActionObservationOptions, +): PostActionObservationPlan { + const verify = options.verify === true; + const needsPreActionBaseline = verify || options.settle !== undefined; + return { + verify, + needsPreActionBaseline, + ...(options.settle !== undefined ? { settle: options.settle } : {}), + }; +} + +export async function applyPostActionObservation( + runtime: AgentDeviceRuntime, + options: CommandContext, + resolved: ResolvedInteractionTarget, + result: T, + params: PostActionObservationPlan, +): Promise> { + const observed = await observeAfterInteraction(runtime, options, resolved, params); + return reconcileNonHittableHintWithEvidence({ ...result, ...observed }); +} + +/** + * Post-action observation composition: `--settle` runs the quiet-window loop + * (settle.ts) and, when `--verify` rides along, its final capture doubles as + * the evidence source. Without settle, verify keeps its single dedicated + * capture. Best-effort: observation never turns a successful action into a + * failure. + */ +async function observeAfterInteraction( + runtime: AgentDeviceRuntime, + options: CommandContext, + resolved: ResolvedInteractionTarget, + params: PostActionObservationPlan, +): Promise<{ evidence?: InteractionEvidence; settle?: SettleObservation }> { + if (params.settle !== undefined) { + const outcome = await settleAfterInteraction(runtime, options, { + ...params.settle, + resolved, + }); + const evidence = params.verify + ? settleEvidence( + outcome.settledNodes, + 'preActionNodes' in resolved ? resolved.preActionNodes : undefined, + ) + : undefined; + return { settle: outcome.observation, ...(evidence ? { evidence } : {}) }; + } + if (!params.verify) return {}; + const evidence = await captureVerifyEvidence(runtime, options, resolved); + return evidence ? { evidence } : {}; +} + +/** + * Post-action side of `--verify` (#1047): one interactive-only capture through + * the same capture helper the resolution path already uses, digested and then + * discarded. The node tree itself is never attached to the result, only the + * cheap summary. + */ +async function captureVerifyEvidence( + runtime: AgentDeviceRuntime, + options: CommandContext, + resolved: ResolvedInteractionTarget, +): Promise { + const preActionNodes: SnapshotNode[] | undefined = + 'preActionNodes' in resolved ? resolved.preActionNodes : undefined; + try { + const capture = await captureInteractionSnapshot(runtime, options, true); + const after = summarizeAxEvidence(capture.snapshot.nodes); + // No pre-action baseline means we cannot claim a change happened; default + // to false rather than asserting a change we did not actually observe. + const changedFromBefore = + preActionNodes !== undefined && after.digest !== summarizeAxEvidence(preActionNodes).digest; + return { ...after, changedFromBefore }; + } catch { + return undefined; + } +} + +// The resolution-time non-hittable hint warns the action "may have had no +// visible effect". When --verify evidence proves the interactive tree changed, +// or --settle returns a material diff, that warning is contradicted by data +// sitting next to it in the same response; drop it and let targetHittable plus +// the observation speak for themselves. +function reconcileNonHittableHintWithEvidence(result: ObservedResult): T { + // Widened view: point-target results carry none of these fields, which is + // exactly the no-op path. + const view = result as { + targetHittable?: boolean; + hint?: string; + evidence?: InteractionEvidence; + settle?: SettleObservation; + }; + if ( + view.targetHittable !== false || + !hasMaterialPostActionChange(view) || + view.hint === undefined + ) { + return result; + } + const { hint: _hint, ...rest } = view; + return rest as T; +} + +function hasMaterialPostActionChange(view: { + evidence?: InteractionEvidence; + settle?: SettleObservation; +}): boolean { + if (view.evidence?.changedFromBefore === true) return true; + const summary = view.settle?.diff?.summary; + return !!summary && (summary.additions > 0 || summary.removals > 0); +} diff --git a/src/commands/interaction/runtime/selector-read.ts b/src/commands/interaction/runtime/selector-read.ts index 0ac338228..88dc204ac 100644 --- a/src/commands/interaction/runtime/selector-read.ts +++ b/src/commands/interaction/runtime/selector-read.ts @@ -42,6 +42,12 @@ import { resolveRefNode, } from './selector-read-shared.ts'; import { findNodeByLabel, resolveRefLabel, shouldScopeFind } from './selector-read-utils.ts'; +import { + DEFAULT_STABLE_QUIET_MS, + runStableCaptureLoop, + TINY_STABLE_TREE_HINT, + TINY_STABLE_TREE_NODE_COUNT, +} from './stable-capture.ts'; import { now, sleep, toBackendContext } from '../../runtime-common.ts'; export type { SelectorSnapshotOptions } from './selector-read-shared.ts'; @@ -153,10 +159,6 @@ export function ref(refInput: string, options: { fallbackLabel?: string } = {}): const DEFAULT_TIMEOUT_MS = 10_000; const POLL_INTERVAL_MS = 300; -const DEFAULT_QUIET_MS = 500; -// Below this node count a settled tree is suspicious: real app surfaces have -// more than a handful of accessibility nodes, splash/loading screens do not. -const TINY_STABLE_TREE_NODE_COUNT = 5; export const findCommand: RuntimeCommand = async ( runtime, @@ -521,6 +523,9 @@ async function snapshotContainsText( return Boolean(findNodeByLabel(capture.snapshot.nodes, text)); } +// The quiet-window loop itself lives in stable-capture.ts and is shared with +// the interaction `--settle` flag (#1101); this wrapper maps the loop outcome +// to wait's throwing semantics. async function waitForStable( runtime: AgentDeviceRuntime, options: WaitCommandOptions, @@ -528,110 +533,33 @@ async function waitForStable( timeoutMs: number | null | undefined, ): Promise> { const timeout = timeoutMs ?? DEFAULT_TIMEOUT_MS; - const quiet = quietMs ?? DEFAULT_QUIET_MS; - const start = now(runtime); - let captures = 0; - let lastDigest: string | undefined; - let lastNodeCount = 0; - let quietSinceMs = start; - while (now(runtime) - start < timeout) { - const capture = await captureStableSignalWithinDeadline( - runtime, - options, - timeout - (now(runtime) - start), - ); - if (!capture) { - throw new AppError('COMMAND_FAILED', 'wait timed out waiting for a stable UI', { - reason: 'wait_stable_timeout', - captureStalled: true, - quietMs: quiet, - timeoutMs: timeout, - captures, - nodeCount: lastNodeCount, - hint: 'A snapshot capture stalled past the wait timeout, so no settle verdict is available. The UI may still be readable: retry, or use screenshot to inspect the surface.', - }); - } - captures += 1; - const digest = digestSnapshotNodes(capture.snapshot.nodes); - const nowMs = now(runtime); - if (digest !== lastDigest) { - lastDigest = digest; - lastNodeCount = capture.snapshot.nodes.length; - quietSinceMs = nowMs; - } else if (captures >= 2 && nowMs - quietSinceMs >= quiet) { - return { - kind: 'stable', - waitedMs: nowMs - start, - captures, - nodeCount: lastNodeCount, - // A settled-but-tiny tree usually means a splash/loading surface, not - // real content: stability alone is a weak readiness signal there. - ...(lastNodeCount < TINY_STABLE_TREE_NODE_COUNT - ? { - hint: 'Settled on a nearly-empty tree — the app may still be loading. Wait for specific content (wait text ...) before interacting.', - } - : {}), - }; - } - await sleep(runtime, POLL_INTERVAL_MS); - } - throw new AppError('COMMAND_FAILED', 'wait timed out waiting for a stable UI', { - reason: 'wait_stable_timeout', + const quiet = quietMs ?? DEFAULT_STABLE_QUIET_MS; + const outcome = await runStableCaptureLoop(runtime, options, { quietMs: quiet, timeoutMs: timeout, - captures, - nodeCount: lastNodeCount, - }); -} - -// Intentionally does not update the session snapshot: the stable loop captures -// an interactive-only tree purely as a settle signal, and overwriting the -// session's richer cached snapshot with the filtered tree would degrade -// subsequent ref/get/find lookups against the same session. -// -// Resolves undefined when the capture does not return within remainingMs. A -// stalled backend capture (observed with macOS AX captures) must not push the -// stable wait past the user-supplied timeout into the daemon request timeout. -// The deadline uses a real timer even when runtime.clock is injected: test -// clocks advance synthetic time synchronously and cannot represent a hung -// backend call. -async function captureStableSignalWithinDeadline( - runtime: AgentDeviceRuntime, - options: WaitCommandOptions, - remainingMs: number, -): Promise { - const capture = captureSelectorSnapshot(runtime, options, { - updateSession: false, - interactiveOnly: true, }); - let timer: NodeJS.Timeout | undefined; - try { - const result = await Promise.race([ - capture, - new Promise((resolve) => { - timer = setTimeout(() => resolve(undefined), remainingMs); - }), - ]); - if (result === undefined) { - // The abandoned capture settles (or fails) on its own; swallow it so it - // cannot surface as an unhandled rejection after the wait already threw. - capture.catch(() => {}); - } - return result; - } finally { - if (timer !== undefined) clearTimeout(timer); + if (!outcome.settled) { + throw new AppError('COMMAND_FAILED', 'wait timed out waiting for a stable UI', { + reason: 'wait_stable_timeout', + ...(outcome.stalled ? { captureStalled: true } : {}), + quietMs: quiet, + timeoutMs: timeout, + captures: outcome.captures, + nodeCount: outcome.nodeCount, + ...(outcome.stalled + ? { + hint: 'A snapshot capture stalled past the wait timeout, so no settle verdict is available. The UI may still be readable: retry, or use screenshot to inspect the surface.', + } + : {}), + }); } -} - -function digestSnapshotNodes(nodes: SnapshotNode[]): string { - return nodes.map(digestSnapshotNode).join('|'); -} - -function digestSnapshotNode(node: SnapshotNode): string { - const rect = node.rect - ? `${Math.round(node.rect.x)},${Math.round(node.rect.y)},${Math.round(node.rect.width)},${Math.round(node.rect.height)}` - : ''; - return `${node.type ?? ''}#${node.label ?? ''}#${node.identifier ?? ''}#${rect}`; + return { + kind: 'stable', + waitedMs: outcome.waitedMs, + captures: outcome.captures, + nodeCount: outcome.nodeCount, + ...(outcome.nodeCount < TINY_STABLE_TREE_NODE_COUNT ? { hint: TINY_STABLE_TREE_HINT } : {}), + }; } async function resolveSelectorNode( diff --git a/src/commands/interaction/runtime/settle.test.ts b/src/commands/interaction/runtime/settle.test.ts new file mode 100644 index 000000000..3a8f7cb16 --- /dev/null +++ b/src/commands/interaction/runtime/settle.test.ts @@ -0,0 +1,379 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { AgentDeviceBackend, BackendSnapshotResult } from '../../../backend.ts'; +import type { SnapshotState } from '../../../kernel/snapshot.ts'; +import { createLocalArtifactAdapter } from '../../../io.ts'; +import { + createAgentDevice, + createMemorySessionStore, + localCommandPolicy, +} from '../../../runtime.ts'; +import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts'; +import { ref, selector } from './selector-read.ts'; +import { NEVER_SETTLED_HINT } from './settle.ts'; + +// #1101 --settle: quiet-window settle loop composition on the interaction +// commands. Budgets are injected (fake clock) — no real waiting. + +function createFakeClock(stepMs = 300): { + now: () => number; + sleep: (ms: number) => Promise; +} { + let elapsed = 0; + return { + now: () => elapsed, + sleep: async (ms: number) => { + elapsed += ms > 0 ? ms : stepMs; + }, + }; +} + +function buttonSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: true, + }, + ]); +} + +// Five nodes so a settled capture clears the tiny-tree readiness heuristic. +function welcomeSnapshot(): SnapshotState { + return makeSnapshotState( + ['Welcome!', 'Next', 'Back', 'Home', 'Menu'].map((label, index) => ({ + index, + depth: index === 0 ? 0 : 1, + ...(index === 0 ? {} : { parentIndex: 0 }), + type: index === 0 ? 'StaticText' : 'Button', + label, + rect: { x: 10, y: 20 + index * 60, width: 100, height: 40 }, + hittable: true, + })), + ); +} + +function createSettleDevice(params: { + stored: SnapshotState; + captureSnapshot: () => Promise | BackendSnapshotResult; + tap?: () => Promise>; +}): ReturnType { + return createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => await params.captureSnapshot(), + tap: async () => (params.tap ? await params.tap() : { ok: true }), + fill: async () => ({ ok: true }), + longPress: async () => ({ ok: true }), + typeText: async () => {}, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot: params.stored }]), + policy: localCommandPolicy(), + clock: createFakeClock(), + }); +} + +test('press --settle returns the settled diff and stores the settled tree', async () => { + const before = buttonSnapshot(); + const after = welcomeSnapshot(); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + // Capture 1 = selector resolution (baseline). Captures 2+ = settle loop. + return { snapshot: captures === 1 ? before : after }; + }, + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + settle: {}, + }); + + assert.equal(result.kind, 'selector'); + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, true); + assert.equal(settle.quietMs, 500); + assert.equal(settle.timeoutMs, 10_000); + assert.ok(settle.captures >= 2); + assert.equal(settle.hint, undefined); + assert.deepEqual(settle.diff?.summary, { additions: 5, removals: 1, unchanged: 0 }); + const addedRefs = settle.diff?.lines + .filter((line) => line.kind === 'added') + .map((line) => line.ref); + assert.deepEqual(addedRefs, ['e1', 'e2', 'e3', 'e4', 'e5']); + const removed = settle.diff?.lines.find((line) => line.kind === 'removed'); + assert.match(removed?.text ?? '', /Continue/); + assert.equal(removed?.ref, undefined); + + // The settled tree became the session snapshot: the diff's refs resolve. + const stored = (await device.sessions.get('default')) as { snapshot?: SnapshotState }; + assert.equal(stored.snapshot?.nodes[0]?.label, 'Welcome!'); +}); + +test('never-settling content returns the last diff with settled: false and a hint', async () => { + const before = buttonSnapshot(); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + if (captures === 1) return { snapshot: before }; + // Ticker: every capture differs. + return { + snapshot: makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'StaticText', + label: `Tick ${captures}`, + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: true, + }, + ]), + }; + }, + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + settle: { quietMs: 500, timeoutMs: 2_000 }, + }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, false); + assert.equal(settle.hint, NEVER_SETTLED_HINT); + // The LAST capture's diff still ships — best-effort observation. + assert.ok(settle.diff); + assert.equal(settle.diff.summary.additions, 1); + assert.match(settle.diff.lines.find((line) => line.kind === 'added')?.text ?? '', /Tick/); +}); + +test('a broken settle capture never fails the action', async () => { + const before = buttonSnapshot(); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + if (captures === 1) return { snapshot: before }; + throw new Error('AX bridge crashed'); + }, + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + settle: {}, + }); + + // The press itself succeeded; the observation reports its own failure. + assert.equal(result.kind, 'selector'); + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, false); + assert.equal(settle.diff, undefined); + assert.match(settle.hint ?? '', /Settle observation unavailable \(AX bridge crashed\)/); +}); + +test('--settle --verify shares the settle captures for evidence (zero extra captures)', async () => { + const before = buttonSnapshot(); + const after = welcomeSnapshot(); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : after }; + }, + }); + + const result = await device.interactions.click(selector('label=Continue'), { + session: 'default', + verify: true, + settle: {}, + }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, true); + assert.ok(result.evidence); + assert.equal(result.evidence?.changedFromBefore, true); + assert.equal(result.evidence?.nodeCount, 5); + // 1 resolution capture + the settle loop's captures — verify added none. + assert.equal(captures, 1 + settle.captures); +}); + +test('longpress --settle rides the same observation path', async () => { + const before = buttonSnapshot(); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : welcomeSnapshot() }; + }, + }); + + const result = await device.interactions.longPress(ref('@e1'), { + session: 'default', + durationMs: 400, + settle: { quietMs: 100, timeoutMs: 1_000 }, + }); + + assert.equal(result.kind, 'ref'); + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, true); + assert.equal(settle.quietMs, 100); + assert.equal(settle.timeoutMs, 1_000); + assert.ok(settle.diff); +}); + +test('the settled diff line list is bounded with a truncation marker', async () => { + const before = buttonSnapshot(); + const bigTree = makeSnapshotState( + Array.from({ length: 120 }, (_, index) => ({ + index, + depth: 0, + type: 'StaticText', + label: `Row ${index}`, + rect: { x: 0, y: index * 20, width: 100, height: 20 }, + hittable: true, + })), + ); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : bigTree }; + }, + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + settle: {}, + }); + + const diff = result.settle?.diff; + assert.ok(diff); + assert.equal(diff.summary.additions, 120); + assert.equal(diff.lines.length, 80); + assert.equal(diff.truncated, true); +}); + +test('keyboard Key nodes never spend the settled diff budget', async () => { + const before = buttonSnapshot(); + // A fill-style settled tree: a summoned keyboard (container + keys) plus the + // content change the agent actually cares about. + const keyboardTree = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'StaticText', + label: 'Results for alpenglow', + rect: { x: 0, y: 0, width: 200, height: 20 }, + hittable: true, + }, + { + index: 1, + depth: 0, + type: 'Keyboard', + label: 'keyboard', + rect: { x: 0, y: 500, width: 400, height: 300 }, + hittable: true, + }, + ...Array.from({ length: 26 }, (_, key) => ({ + index: key + 2, + depth: 1, + parentIndex: 1, + type: 'Key', + label: String.fromCharCode(97 + key), + rect: { x: key * 10, y: 520, width: 10, height: 40 }, + hittable: true, + })), + ...['Next', 'Back', 'Home'].map((label, extra) => ({ + index: extra + 28, + depth: 0, + type: 'Button', + label, + rect: { x: 0, y: 40 + extra * 40, width: 100, height: 40 }, + hittable: true, + })), + ]); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : keyboardTree }; + }, + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + settle: {}, + }); + + const diff = result.settle?.diff; + assert.ok(diff); + const texts = diff.lines.map((line) => line.text).join('\n'); + assert.match(texts, /Results for alpenglow/); + assert.match(texts, /keyboard/); + // The container line survives as the keyboard signal; individual keys do not. + assert.ok(!diff.lines.some((line) => /\[key\]/.test(line.text))); + assert.equal(diff.summary.additions, 5); +}); + +test('added lines win diff-budget slots over removals under truncation', async () => { + // 120 removals precede the additions positionally; the fresh-ref additions + // must still survive the 80-line cap. + const bigBefore = makeSnapshotState( + Array.from({ length: 120 }, (_, index) => ({ + index, + depth: 0, + type: 'StaticText', + label: `Old row ${index}`, + rect: { x: 0, y: index * 20, width: 100, height: 20 }, + hittable: true, + })), + ); + const afterTree = makeSnapshotState( + Array.from({ length: 10 }, (_, index) => ({ + index, + depth: 0, + type: 'Button', + label: `New action ${index}`, + rect: { x: 0, y: index * 40, width: 100, height: 40 }, + hittable: true, + })), + ); + let captures = 0; + const device = createSettleDevice({ + stored: bigBefore, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? bigBefore : afterTree }; + }, + }); + + const result = await device.interactions.press(selector('label="Old row 0"'), { + session: 'default', + settle: {}, + }); + + const diff = result.settle?.diff; + assert.ok(diff); + assert.equal(diff.truncated, true); + assert.equal(diff.lines.length, 80); + const added = diff.lines.filter((line) => line.kind === 'added'); + assert.equal(added.length, 10); + assert.ok(added.every((line) => line.ref !== undefined)); +}); diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts new file mode 100644 index 000000000..c0222ff66 --- /dev/null +++ b/src/commands/interaction/runtime/settle.ts @@ -0,0 +1,212 @@ +import type { SnapshotNode } from '../../../kernel/snapshot.ts'; +import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; +import { isSparseSnapshotQualityVerdict } from '../../../snapshot/snapshot-quality.ts'; +import { buildSnapshotDiff } from '../../../snapshot/snapshot-diff.ts'; +import { summarizeAxEvidence } from '../../../utils/ax-digest.ts'; +import type { + InteractionEvidence, + ResolvedInteractionTarget, + SettleObservation, + SettleParams, +} from '../../../contracts/interaction.ts'; +import type { CapturedSnapshot } from './selector-read-shared.ts'; +import { + DEFAULT_STABLE_QUIET_MS, + DEFAULT_STABLE_TIMEOUT_MS, + runStableCaptureLoop, + TINY_STABLE_TREE_HINT, + TINY_STABLE_TREE_NODE_COUNT, +} from './stable-capture.ts'; + +/** + * `--settle` (#1101): after a mutating interaction, wait for the UI to go + * quiet (wait stable's loop, shared via stable-capture.ts) and return the + * settled DIFF against the pre-action tree in the same response — one round + * trip instead of the interact → observe pair. + * + * Best-effort by contract: this module never throws. The action already + * succeeded when it runs; observation quality is advisory (same principle as + * `--verify` evidence). + */ + +export type SettleOutcome = { + observation: SettleObservation; + /** Nodes of the final capture; doubles as the `--verify` evidence source. */ + settledNodes?: SnapshotNode[]; +}; + +// Changed-lines bound: the settled diff is the response payload, and unbounded +// added/removed lists on a full screen transition would crowd out everything +// else. The summary always carries the true counts. +const MAX_SETTLE_DIFF_LINES = 80; + +export const NEVER_SETTLED_HINT = + 'The UI kept changing for the whole settle budget (animation, carousel, or ticker?). The diff reflects the last capture and may already be outdated. Raise --timeout, or verify the target content with wait text.'; + +const SETTLE_CAPTURE_STALLED_HINT = + 'A snapshot capture stalled past the settle budget, so no settle verdict is available. The action itself succeeded; retry the observation with snapshot or wait stable.'; + +export async function settleAfterInteraction( + runtime: AgentDeviceRuntime, + options: CommandContext, + params: SettleParams & { resolved: ResolvedInteractionTarget }, +): Promise { + const quietMs = params.quietMs ?? DEFAULT_STABLE_QUIET_MS; + const timeoutMs = params.timeoutMs ?? DEFAULT_STABLE_TIMEOUT_MS; + const base: SettleObservation = { settled: false, waitedMs: 0, captures: 0, quietMs, timeoutMs }; + try { + const outcome = await runStableCaptureLoop(runtime, options, { quietMs, timeoutMs }); + const observation: SettleObservation = { + ...base, + settled: outcome.settled, + waitedMs: outcome.waitedMs, + captures: outcome.captures, + }; + if (!outcome.lastCapture) { + return { + observation: { + ...observation, + hint: outcome.stalled ? SETTLE_CAPTURE_STALLED_HINT : NEVER_SETTLED_HINT, + }, + }; + } + const stored = await storeSettledSnapshot(runtime, options, outcome.lastCapture); + const settledNodes = outcome.lastCapture.snapshot.nodes; + return { + observation: { + ...observation, + // The diff (with its added-line refs) is only attached when the settled + // tree actually became the stored session snapshot: those refs must be + // valid against the tree the next @ref command resolves on. The daemon + // treats `diff` presence as "this response issues refs". + ...(stored + ? { diff: buildSettleDiff(resolveBaselineNodes(params.resolved), settledNodes) } + : {}), + ...resolveSettleHint(outcome, stored, settledNodes.length), + }, + settledNodes, + }; + } catch (error) { + // Never fail the action over the observation: report that settling itself + // broke and let the caller fall back to an explicit snapshot. + return { + observation: { + ...base, + hint: `Settle observation unavailable (${error instanceof Error ? error.message : String(error)}). The action itself succeeded; take a snapshot to observe the result.`, + }, + }; + } +} + +/** + * `--settle --verify` composition: the settle loop's final capture doubles as + * the verify evidence source, so the pair costs zero extra captures. Without a + * final capture there is no evidence — best-effort, like verify itself. + */ +export function settleEvidence( + settledNodes: SnapshotNode[] | undefined, + preActionNodes: SnapshotNode[] | undefined, +): InteractionEvidence | undefined { + if (!settledNodes) return undefined; + const after = summarizeAxEvidence(settledNodes); + const changedFromBefore = + preActionNodes !== undefined && after.digest !== summarizeAxEvidence(preActionNodes).digest; + return { ...after, changedFromBefore }; +} + +function resolveBaselineNodes(resolved: ResolvedInteractionTarget): SnapshotNode[] { + return 'preActionNodes' in resolved && resolved.preActionNodes ? resolved.preActionNodes : []; +} + +function buildSettleDiff( + baselineNodes: SnapshotNode[], + settledNodes: SnapshotNode[], +): NonNullable { + // Flattened compare, like `diff -i`: both sides are interactive-flavored + // captures and depth jitter across captures should not read as change. When + // the baseline came from a richer stored tree (ref targets reuse the session + // snapshot), extra baseline-only lines surface as removals — advisory noise, + // the same baseline caveat --verify's changedFromBefore already accepts. + const diff = buildSnapshotDiff( + withoutKeyboardKeys(baselineNodes), + withoutKeyboardKeys(settledNodes), + { flatten: true, withRefs: true }, + ); + const changed = diff.lines.filter((line) => line.kind !== 'unchanged'); + const lines = capSettleDiffLines(changed).map((line) => ({ + kind: line.kind as 'added' | 'removed', + text: line.text, + ...(line.ref ? { ref: line.ref } : {}), + })); + return { + summary: diff.summary, + lines, + ...(changed.length > lines.length ? { truncated: true as const } : {}), + }; +} + +// The iOS QWERTY keyboard is ~50 Key nodes; a fill that summons it would spend +// most of the capped line budget spelling out the keyboard instead of the +// content change the agent actually asked to observe. The Keyboard container +// node stays, so "keyboard appeared/left" remains one visible diff line. +function withoutKeyboardKeys(nodes: SnapshotNode[]): SnapshotNode[] { + return nodes.filter((node) => node.type !== 'Key'); +} + +/** + * Truncation policy: added lines win. They carry the settled tree's fresh + * refs — the actionable half of the diff — while removals only describe what + * left the screen (the summary still counts them). Relative order within each + * kind is preserved; removals fill whatever budget the additions leave. + */ +function capSettleDiffLines(changed: T[]): T[] { + if (changed.length <= MAX_SETTLE_DIFF_LINES) return changed; + const added = changed.filter((line) => line.kind === 'added'); + const keptAdded = new Set(added.slice(0, MAX_SETTLE_DIFF_LINES)); + let removedBudget = MAX_SETTLE_DIFF_LINES - keptAdded.size; + const kept: T[] = []; + for (const line of changed) { + if (keptAdded.has(line)) { + kept.push(line); + } else if (line.kind === 'removed' && removedBudget > 0) { + kept.push(line); + removedBudget -= 1; + } + } + return kept; +} + +function resolveSettleHint( + outcome: { settled: boolean; stalled: boolean }, + stored: boolean, + settledNodeCount: number, +): { hint?: string } { + if (outcome.stalled) return { hint: SETTLE_CAPTURE_STALLED_HINT }; + if (!outcome.settled) return { hint: NEVER_SETTLED_HINT }; + if (!stored) { + return { + hint: 'Settled on a sparse, unreadable tree — the diff is omitted. Use screenshot as visual truth before interacting further.', + }; + } + // Same weak-readiness signal wait stable reports: a settled-but-tiny tree + // usually means a splash/loading surface, not real content. + if (settledNodeCount < TINY_STABLE_TREE_NODE_COUNT) return { hint: TINY_STABLE_TREE_HINT }; + return {}; +} + +// The settle loop itself captures with updateSession: false (a capture that +// later stalls must not race a session write past the response). The FINAL +// settled tree IS stored — its refs ride the diff payload, so they must be +// resolvable by the next @ref command. Sparse-quality captures are not stored +// (mirroring captureSelectorSnapshot) and therefore issue no refs. +async function storeSettledSnapshot( + runtime: AgentDeviceRuntime, + options: CommandContext, + capture: CapturedSnapshot, +): Promise { + if (isSparseSnapshotQualityVerdict(capture.snapshot.snapshotQuality)) return false; + const session = await runtime.sessions.get(options.session ?? 'default'); + if (!session) return false; + await runtime.sessions.set({ ...session, snapshot: capture.snapshot }); + return true; +} diff --git a/src/commands/interaction/runtime/stable-capture.ts b/src/commands/interaction/runtime/stable-capture.ts new file mode 100644 index 000000000..c137d5143 --- /dev/null +++ b/src/commands/interaction/runtime/stable-capture.ts @@ -0,0 +1,154 @@ +import type { SnapshotNode } from '../../../kernel/snapshot.ts'; +import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; +import { now, sleep } from '../../runtime-common.ts'; +import { + captureSelectorSnapshot, + type CapturedSnapshot, + type SelectorSnapshotOptions, +} from './selector-read-shared.ts'; + +/** + * The quiet-window stable-capture loop shared by `wait stable` and the + * interaction `--settle` flag (#1101): capture the interactive-only tree, + * digest it, and declare the UI stable once at least two captures in a row + * stay identical for `quietMs`. The loop itself never throws and never + * updates the session snapshot — callers map the outcome to their own + * semantics (`wait stable` throws on timeout; `--settle` is best-effort and + * reports `settled: false`). + */ + +const STABLE_POLL_INTERVAL_MS = 300; +export const DEFAULT_STABLE_QUIET_MS = 500; +export const DEFAULT_STABLE_TIMEOUT_MS = 10_000; +// Below this node count a settled tree is suspicious: real app surfaces have +// more than a handful of accessibility nodes, splash/loading screens do not. +export const TINY_STABLE_TREE_NODE_COUNT = 5; +// A settled-but-tiny tree usually means a splash/loading surface, not real +// content: stability alone is a weak readiness signal there. +export const TINY_STABLE_TREE_HINT = + 'Settled on a nearly-empty tree — the app may still be loading. Wait for specific content (wait text ...) before interacting.'; + +export type StableCaptureLoopResult = { + /** Two identical captures held the quiet window before the deadline. */ + settled: boolean; + /** A capture stalled past the remaining deadline (no verdict available). */ + stalled: boolean; + waitedMs: number; + captures: number; + nodeCount: number; + /** The most recent completed capture, settled or not. */ + lastCapture?: CapturedSnapshot; +}; + +export async function runStableCaptureLoop( + runtime: AgentDeviceRuntime, + options: CommandContext & SelectorSnapshotOptions, + params: { quietMs: number; timeoutMs: number }, +): Promise { + const { quietMs, timeoutMs } = params; + const start = now(runtime); + // Cadence derives from the quiet window (never slower than the default + // poll): a caller asking for a 50ms quiet window should not be forced onto a + // 300ms grid — and tests inject the budget instead of waiting real time. + const pollMs = Math.min(STABLE_POLL_INTERVAL_MS, Math.max(25, quietMs)); + let captures = 0; + let lastDigest: string | undefined; + let lastNodeCount = 0; + let lastCapture: CapturedSnapshot | undefined; + let quietSinceMs = start; + while (now(runtime) - start < timeoutMs) { + const capture = await captureStableSignalWithinDeadline( + runtime, + options, + timeoutMs - (now(runtime) - start), + ); + if (!capture) { + return { + settled: false, + stalled: true, + waitedMs: now(runtime) - start, + captures, + nodeCount: lastNodeCount, + lastCapture, + }; + } + captures += 1; + lastCapture = capture; + const digest = digestSnapshotNodes(capture.snapshot.nodes); + const nowMs = now(runtime); + if (digest !== lastDigest) { + lastDigest = digest; + lastNodeCount = capture.snapshot.nodes.length; + quietSinceMs = nowMs; + } else if (captures >= 2 && nowMs - quietSinceMs >= quietMs) { + return { + settled: true, + stalled: false, + waitedMs: nowMs - start, + captures, + nodeCount: lastNodeCount, + lastCapture, + }; + } + await sleep(runtime, pollMs); + } + return { + settled: false, + stalled: false, + waitedMs: now(runtime) - start, + captures, + nodeCount: lastNodeCount, + lastCapture, + }; +} + +// Intentionally does not update the session snapshot: the stable loop captures +// an interactive-only tree purely as a settle signal, and overwriting the +// session's richer cached snapshot with the filtered tree would degrade +// subsequent ref/get/find lookups against the same session. (`--settle` DOES +// store its final capture, but explicitly, in one place, after the loop.) +// +// Resolves undefined when the capture does not return within remainingMs. A +// stalled backend capture (observed with macOS AX captures) must not push the +// stable wait past the user-supplied timeout into the daemon request timeout. +// The deadline uses a real timer even when runtime.clock is injected: test +// clocks advance synthetic time synchronously and cannot represent a hung +// backend call. +async function captureStableSignalWithinDeadline( + runtime: AgentDeviceRuntime, + options: CommandContext & SelectorSnapshotOptions, + remainingMs: number, +): Promise { + const capture = captureSelectorSnapshot(runtime, options, { + updateSession: false, + interactiveOnly: true, + }); + let timer: NodeJS.Timeout | undefined; + try { + const result = await Promise.race([ + capture, + new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), remainingMs); + }), + ]); + if (result === undefined) { + // The abandoned capture settles (or fails) on its own; swallow it so it + // cannot surface as an unhandled rejection after the wait already threw. + capture.catch(() => {}); + } + return result; + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +function digestSnapshotNodes(nodes: SnapshotNode[]): string { + return nodes.map(digestSnapshotNode).join('|'); +} + +function digestSnapshotNode(node: SnapshotNode): string { + const rect = node.rect + ? `${Math.round(node.rect.x)},${Math.round(node.rect.y)},${Math.round(node.rect.width)},${Math.round(node.rect.height)}` + : ''; + return `${node.type ?? ''}#${node.label ?? ''}#${node.identifier ?? ''}#${rect}`; +} diff --git a/src/contracts/interaction-guarantees.ts b/src/contracts/interaction-guarantees.ts index 4cce8f3c2..7efb60bcd 100644 --- a/src/contracts/interaction-guarantees.ts +++ b/src/contracts/interaction-guarantees.ts @@ -50,6 +50,11 @@ export const INTERACTION_GUARANTEES = [ 'responseIdentity', // --verify captures a pre-action baseline and post-action digest. 'verifyEvidence', + // --settle (#1101) waits for the UI to go quiet after the action and returns + // the settled diff vs the pre-action tree (plus refsGeneration when the + // settled tree was stored) in the same response. Best-effort: never fails + // the action. + 'settleObservation', // Failures use the shared codes/messages/hints (no-match diagnostics, // ambiguous shape, offscreen reasons). NOTE: expected to split into // errorCodes (stable codes / fallback classification) vs errorDiagnostics @@ -130,37 +135,48 @@ const SHARED_RESPONSE_CONSTRUCTION: GuaranteeEnforcement = { via: 'src/daemon/handlers/interaction-touch-response.ts#buildInteractionResponseData', }; +// The two runtime tree paths (selector and ref resolution) run the SAME shared +// guard/observation implementations; only how the target is found +// (disambiguation) and how failures are described (errorTaxonomy) differ. +const RUNTIME_TREE_SHARED_GUARANTEES = { + occlusion: { + kind: 'runtime', + via: 'src/snapshot/snapshot-occlusion.ts#isSnapshotNodeInteractionBlocked', + }, + offscreen: { + kind: 'runtime', + via: 'src/snapshot/mobile-snapshot-semantics.ts#isNodeVisibleOnScreen', + }, + nonHittable: { + kind: 'runtime', + via: 'src/core/interaction-targeting.ts#resolveActionableTouchResolution', + }, + responseConstruction: SHARED_RESPONSE_CONSTRUCTION, + responseIdentity: { + kind: 'runtime', + via: 'src/daemon/handlers/interaction-touch-targets.ts#interactionResultExtra', + }, + verifyEvidence: { + kind: 'runtime', + via: 'src/commands/interaction/runtime/interactions.ts#pressCommand', + appliesTo: ['press', 'click', 'fill'], + }, + settleObservation: { + kind: 'runtime', + via: 'src/commands/interaction/runtime/settle.ts#settleAfterInteraction', + }, +} satisfies Partial>; + export const INTERACTION_DISPATCH_PATHS: Record = { 'runtime-selector': { description: 'Daemon tree capture, selector chain resolution, guarded coordinate tap.', commands: ['press', 'click', 'fill', 'longpress'], guarantees: { + ...RUNTIME_TREE_SHARED_GUARANTEES, disambiguation: { kind: 'runtime', via: 'src/daemon/selectors-resolve.ts#resolveSelectorChain', }, - occlusion: { - kind: 'runtime', - via: 'src/snapshot/snapshot-occlusion.ts#isSnapshotNodeInteractionBlocked', - }, - offscreen: { - kind: 'runtime', - via: 'src/snapshot/mobile-snapshot-semantics.ts#isNodeVisibleOnScreen', - }, - nonHittable: { - kind: 'runtime', - via: 'src/core/interaction-targeting.ts#resolveActionableTouchResolution', - }, - responseConstruction: SHARED_RESPONSE_CONSTRUCTION, - responseIdentity: { - kind: 'runtime', - via: 'src/daemon/handlers/interaction-touch-targets.ts#interactionResultExtra', - }, - verifyEvidence: { - kind: 'runtime', - via: 'src/commands/interaction/runtime/interactions.ts#pressCommand', - appliesTo: ['press', 'click', 'fill'], - }, errorTaxonomy: { kind: 'runtime', via: 'src/daemon/selectors-resolve.ts#formatSelectorFailure', @@ -171,32 +187,11 @@ export const INTERACTION_DISPATCH_PATHS: Record; + diff?: { + summary: { additions: number; removals: number; unchanged: number }; + lines: SettleDiffLine[]; + /** Present (true) when lines were capped to the response bound. */ + truncated?: boolean; + }; + hint?: string; +}; + export type PressCommandResult = ResolvedInteractionTarget & { backendResult?: Record; message?: string; warning?: string; evidence?: InteractionEvidence; + settle?: SettleObservation; }; export type FillCommandResult = ResolvedInteractionTarget & { @@ -90,6 +153,7 @@ export type FillCommandResult = ResolvedInteractionTarget & { backendResult?: Record; message?: string; evidence?: InteractionEvidence; + settle?: SettleObservation; }; export type LongPressCommandResult = ResolvedInteractionTarget & { @@ -97,4 +161,5 @@ export type LongPressCommandResult = ResolvedInteractionTarget & { backendResult?: Record; message?: string; warning?: string; + settle?: SettleObservation; }; diff --git a/src/core/command-descriptor/__tests__/post-action-observation.test.ts b/src/core/command-descriptor/__tests__/post-action-observation.test.ts new file mode 100644 index 000000000..c1293b5d9 --- /dev/null +++ b/src/core/command-descriptor/__tests__/post-action-observation.test.ts @@ -0,0 +1,67 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { PUBLIC_COMMANDS } from '../../../command-catalog.ts'; +import { findCommandMetadata } from '../../../commands/command-metadata.ts'; +import { getCliCommandSchema } from '../../../utils/command-schema.ts'; +import { + commandDescriptors, + commandSupportsSettleObservation, + commandSupportsVerifyEvidence, + resolveCommandPostActionObservationSupport, +} from '../registry.ts'; + +const SETTLE_OBSERVATION_COMMANDS = [ + PUBLIC_COMMANDS.click, + PUBLIC_COMMANDS.fill, + PUBLIC_COMMANDS.longPress, + PUBLIC_COMMANDS.press, +] as const; + +test('post-action observation descriptor traits are the source for settle command support', () => { + const descriptorCommands = commandDescriptors + .filter( + (descriptor) => resolveCommandPostActionObservationSupport(descriptor.name) !== undefined, + ) + .map((descriptor) => descriptor.name) + .sort(); + assert.deepEqual(descriptorCommands, [...SETTLE_OBSERVATION_COMMANDS].sort()); + + assert.equal(resolveCommandPostActionObservationSupport('click'), 'settle-and-verify'); + assert.equal(resolveCommandPostActionObservationSupport('press'), 'settle-and-verify'); + assert.equal(resolveCommandPostActionObservationSupport('fill'), 'settle-and-verify'); + assert.equal(resolveCommandPostActionObservationSupport('longpress'), 'settle'); + assert.equal(commandSupportsVerifyEvidence('longpress'), false); +}); + +test('post-action observation CLI flags follow descriptor traits', () => { + for (const command of SETTLE_OBSERVATION_COMMANDS) { + const schema = getCliCommandSchema(command); + const allowedFlags = new Set(schema.allowedFlags ?? []); + assert.equal(allowedFlags.has('settle'), true, `${command}: missing --settle`); + assert.equal(allowedFlags.has('settleQuietMs'), true, `${command}: missing --settle-quiet`); + assert.equal(allowedFlags.has('timeoutMs'), true, `${command}: missing settle --timeout`); + assert.equal( + allowedFlags.has('verify'), + commandSupportsVerifyEvidence(command), + `${command}: verify flag must match descriptor trait`, + ); + } +}); + +test('post-action observation metadata fields follow descriptor traits', () => { + for (const descriptor of commandDescriptors) { + const metadata = findCommandMetadata(descriptor.name); + if (!metadata) continue; + const properties = metadata.inputSchema.properties ?? {}; + assert.equal( + Object.hasOwn(properties, 'settle'), + commandSupportsSettleObservation(descriptor.name), + `${descriptor.name}: settle field must match descriptor trait`, + ); + assert.equal( + Object.hasOwn(properties, 'verify'), + commandSupportsVerifyEvidence(descriptor.name), + `${descriptor.name}: verify field must match descriptor trait`, + ); + } +}); diff --git a/src/core/command-descriptor/__tests__/timeout-policy.test.ts b/src/core/command-descriptor/__tests__/timeout-policy.test.ts index b331d3cda..f0bddb799 100644 --- a/src/core/command-descriptor/__tests__/timeout-policy.test.ts +++ b/src/core/command-descriptor/__tests__/timeout-policy.test.ts @@ -1,10 +1,15 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import { PUBLIC_COMMANDS } from '../../../command-catalog.ts'; -import { commandDescriptors, resolveCommandTimeoutPolicy } from '../registry.ts'; +import { + commandDescriptors, + resolveCommandPostActionObservationSupport, + resolveCommandTimeoutPolicy, +} from '../registry.ts'; import { DEFAULT_TIMEOUT_POLICY } from '../timeout-policy.ts'; +import { DEFAULT_STABLE_TIMEOUT_MS } from '../../../commands/interaction/runtime/stable-capture.ts'; -// ADR 0011 completeness gate for the descriptor timeout policy (the layer that +// ADR 0008 completeness gate for the descriptor timeout policy (the layer that // replaced the two hand-maintained client lists `isExplicitTimeoutCommand` and // `DAEMON_PRESERVING_TIMEOUT_COMMANDS`): every public command must carry a // declared policy, and the sets of commands that deviate from the shared @@ -14,6 +19,15 @@ import { DEFAULT_TIMEOUT_POLICY } from '../timeout-policy.ts'; // src/utils/__tests__/daemon-client.test.ts, which survived this migration // unchanged. +function settleObservationCommandNames(): string[] { + return commandDescriptors + .filter( + (descriptor) => resolveCommandPostActionObservationSupport(descriptor.name) !== undefined, + ) + .map((descriptor) => descriptor.name) + .sort(); +} + test('every public command declares a timeout policy on its descriptor', () => { const byName = new Map(commandDescriptors.map((descriptor) => [descriptor.name, descriptor])); for (const command of Object.values(PUBLIC_COMMANDS)) { @@ -72,20 +86,41 @@ test('daemon-preserving timeout commands are a bounded, reviewed set', () => { }); test('budget sources deviating from the default are bounded, reviewed sets', () => { - const flagBudget: string[] = []; + const flagBoundBudget: string[] = []; + const flagWidenBudget: string[] = []; const positionalBudget: string[] = []; for (const descriptor of commandDescriptors) { - if (descriptor.timeoutPolicy.budget.source === 'flag') flagBudget.push(descriptor.name); - if (descriptor.timeoutPolicy.budget.source === 'positional-parser') { + const budget = descriptor.timeoutPolicy.budget; + if (budget.source === 'flag') { + const widen = 'envelope' in budget && budget.envelope === 'widen'; + (widen ? flagWidenBudget : flagBoundBudget).push(descriptor.name); + } + if (budget.source === 'positional-parser') { positionalBudget.push(descriptor.name); } } // --timeout bounds the request envelope for these commands only. - assert.deepEqual(flagBudget.sort(), ['prepare', 'replay', 'snapshot']); + assert.deepEqual(flagBoundBudget.sort(), ['prepare', 'replay', 'snapshot']); + // --timeout bounds the --settle wait on these commands (#1101); like wait's + // positional budget it only ever widens the envelope, never shrinks it. + assert.deepEqual(flagWidenBudget.sort(), settleObservationCommandNames()); // wait's budget travels as a positional and must widen the envelope. assert.deepEqual(positionalBudget, ['wait']); }); +test('settle timeout policy default matches the runtime settle loop default', () => { + for (const command of settleObservationCommandNames()) { + const budget = resolveCommandTimeoutPolicy(command).budget; + assert.equal(budget.source, 'flag', `${command}: expected flag budget`); + assert.equal(budget.envelope, 'widen', `${command}: expected widening budget`); + assert.equal( + budget.defaultBudgetMs, + DEFAULT_STABLE_TIMEOUT_MS, + `${command}: default settle budget must match runtime default`, + ); + } +}); + test('request envelopes deviating from the default are bounded, reviewed sets', () => { const EXPECTED_ENVELOPES: Record = { prepare: 240_000, diff --git a/src/core/command-descriptor/post-action-observation.ts b/src/core/command-descriptor/post-action-observation.ts new file mode 100644 index 000000000..ce5715941 --- /dev/null +++ b/src/core/command-descriptor/post-action-observation.ts @@ -0,0 +1,24 @@ +import { PUBLIC_COMMANDS, type PublicCommandName } from '../../command-catalog.ts'; + +export type PostActionObservationSupport = 'settle' | 'settle-and-verify'; + +const POST_ACTION_OBSERVATION_BY_COMMAND = { + [PUBLIC_COMMANDS.click]: 'settle-and-verify', + [PUBLIC_COMMANDS.press]: 'settle-and-verify', + [PUBLIC_COMMANDS.fill]: 'settle-and-verify', + [PUBLIC_COMMANDS.longPress]: 'settle', +} as const satisfies Partial>; + +export type PostActionObservationCommandName = keyof typeof POST_ACTION_OBSERVATION_BY_COMMAND; + +export type PostActionObservationSupportFor = + TName extends PostActionObservationCommandName + ? (typeof POST_ACTION_OBSERVATION_BY_COMMAND)[TName] + : undefined; + +export function resolvePostActionObservationSupport( + command: string | undefined, +): PostActionObservationSupport | undefined { + if (command === undefined) return undefined; + return POST_ACTION_OBSERVATION_BY_COMMAND[command as PostActionObservationCommandName]; +} diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index b666c0990..19b8ef709 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -11,6 +11,8 @@ import { INSTALL_REQUEST_TIMEOUT_MS, PREPARE_REQUEST_TIMEOUT_MS, } from './timeout-policy.ts'; +import { resolvePostActionObservationSupport } from './post-action-observation.ts'; +import type { PostActionObservationSupport } from './post-action-observation.ts'; import type { CommandDescriptor, CommandTimeoutPolicy } from './types.ts'; // --------------------------------------------------------------------------- @@ -74,9 +76,9 @@ const APP_INSTALL_CAPABILITY = { } satisfies CommandCapability; // --------------------------------------------------------------------------- -// Timeout policies (ADR-0011) — the request-envelope budget source and the -// on-timeout daemon policy, derived VERBATIM from the two deleted client hand -// lists (`isExplicitTimeoutCommand` in daemon-client.ts and +// Timeout policies — descriptor-owned request-envelope budget source and +// on-timeout daemon policy (ADR 0008). This replaces the two deleted client +// hand lists (`isExplicitTimeoutCommand` in daemon-client.ts and // `DAEMON_PRESERVING_TIMEOUT_COMMANDS` in daemon-client-timeout.ts) plus the // per-command envelope branches of `resolveDaemonRequestTimeoutMs`. // --------------------------------------------------------------------------- @@ -99,6 +101,39 @@ const INSTALL_TIMEOUT_POLICY: CommandTimeoutPolicy = { envelopeMs: INSTALL_REQUEST_TIMEOUT_MS, }; +const DEFAULT_SETTLE_TIMEOUT_MS = 10_000; + +// Settle-capable interaction commands also resolve their target through the +// same platform accessibility capture as snapshot/find (#1105): a hung capture +// is their dominant timeout mode, so on top of the --settle flag-sourced +// widening envelope above, keep the daemon (and sessions) alive on timeout too. +const SETTLE_FLAG_PRESERVE_DAEMON_TIMEOUT_POLICY: CommandTimeoutPolicy = { + ...DEFAULT_TIMEOUT_POLICY, + // --settle (#1101) makes --timeout bound the SETTLE wait, not the whole + // request. Widen the envelope by the settle budget so selector/action + // overhead still has room before the post-action wait. + budget: { + source: 'flag', + envelope: 'widen', + defaultBudgetMs: DEFAULT_SETTLE_TIMEOUT_MS, + }, + onTimeout: 'preserve-daemon', +}; + +function interactionTimeoutPolicy(command: string): CommandTimeoutPolicy { + return resolvePostActionObservationSupport(command) !== undefined + ? SETTLE_FLAG_PRESERVE_DAEMON_TIMEOUT_POLICY + : PRESERVE_DAEMON_TIMEOUT_POLICY; +} + +function postActionObservation(command: string): PostActionObservationSupport { + const support = resolvePostActionObservationSupport(command); + if (support === undefined) { + throw new Error(`Missing post-action observation descriptor support for ${command}`); + } + return support; +} + // --------------------------------------------------------------------------- // The additive single source. Each entry carries the daemon route/traits + // capability + batchable flag copied VERBATIM from today's hand tables. @@ -461,49 +496,53 @@ const RAW_COMMAND_DESCRIPTORS = [ name: PUBLIC_COMMANDS.click, daemon: { route: 'interaction', replayScopedAction: true, androidBlockingDialogGuard: true }, capability: { apple: APPLE_SIM_AND_DEVICE, android: ANDROID_ALL, linux: LINUX_DEVICE }, - timeoutPolicy: PRESERVE_DAEMON_TIMEOUT_POLICY, + timeoutPolicy: interactionTimeoutPolicy(PUBLIC_COMMANDS.click), + postActionObservation: postActionObservation(PUBLIC_COMMANDS.click), batchable: true, }, { name: PUBLIC_COMMANDS.fill, daemon: { route: 'interaction', replayScopedAction: true, androidBlockingDialogGuard: true }, capability: { apple: APPLE_SIM_AND_DEVICE, android: ANDROID_ALL, linux: LINUX_DEVICE }, - timeoutPolicy: PRESERVE_DAEMON_TIMEOUT_POLICY, + timeoutPolicy: interactionTimeoutPolicy(PUBLIC_COMMANDS.fill), + postActionObservation: postActionObservation(PUBLIC_COMMANDS.fill), batchable: true, }, { name: PUBLIC_COMMANDS.longPress, daemon: { route: 'interaction', replayScopedAction: true, androidBlockingDialogGuard: true }, capability: { apple: APPLE_SIM_AND_DEVICE, android: ANDROID_ALL, linux: LINUX_DEVICE }, - timeoutPolicy: PRESERVE_DAEMON_TIMEOUT_POLICY, + timeoutPolicy: interactionTimeoutPolicy(PUBLIC_COMMANDS.longPress), + postActionObservation: postActionObservation(PUBLIC_COMMANDS.longPress), batchable: true, }, { name: PUBLIC_COMMANDS.press, daemon: { route: 'interaction', replayScopedAction: true, androidBlockingDialogGuard: true }, capability: { apple: APPLE_SIM_AND_DEVICE, android: ANDROID_ALL, linux: LINUX_DEVICE }, - timeoutPolicy: PRESERVE_DAEMON_TIMEOUT_POLICY, + timeoutPolicy: interactionTimeoutPolicy(PUBLIC_COMMANDS.press), + postActionObservation: postActionObservation(PUBLIC_COMMANDS.press), batchable: true, }, { name: PUBLIC_COMMANDS.type, daemon: { route: 'interaction', replayScopedAction: true, androidBlockingDialogGuard: true }, capability: ALL_DEVICE_COMMAND_CAPABILITY, - timeoutPolicy: PRESERVE_DAEMON_TIMEOUT_POLICY, + timeoutPolicy: interactionTimeoutPolicy(PUBLIC_COMMANDS.type), batchable: true, }, { name: PUBLIC_COMMANDS.get, daemon: { route: 'interaction', replayScopedAction: true }, capability: ALL_DEVICE_COMMAND_CAPABILITY, - timeoutPolicy: PRESERVE_DAEMON_TIMEOUT_POLICY, + timeoutPolicy: interactionTimeoutPolicy(PUBLIC_COMMANDS.get), batchable: true, }, { name: PUBLIC_COMMANDS.is, daemon: { route: 'interaction', replayScopedAction: true }, capability: ALL_DEVICE_COMMAND_CAPABILITY, - timeoutPolicy: PRESERVE_DAEMON_TIMEOUT_POLICY, + timeoutPolicy: interactionTimeoutPolicy(PUBLIC_COMMANDS.is), batchable: true, }, @@ -663,12 +702,31 @@ export const commandDescriptors = RAW_COMMAND_DESCRIPTORS.map((descriptor) => ({ /** The literal union of every registered command name. */ export type Command = (typeof commandDescriptors)[number]['name']; +const COMMAND_DESCRIPTOR_BY_NAME: ReadonlyMap = new Map( + commandDescriptors.map((descriptor) => [descriptor.name, descriptor]), +); + const TIMEOUT_POLICY_BY_COMMAND: ReadonlyMap = new Map( commandDescriptors.map((descriptor) => [descriptor.name, descriptor.timeoutPolicy]), ); +export function resolveCommandPostActionObservationSupport( + command: string | undefined, +): PostActionObservationSupport | undefined { + if (command === undefined) return undefined; + return COMMAND_DESCRIPTOR_BY_NAME.get(command)?.postActionObservation; +} + +export function commandSupportsSettleObservation(command: string | undefined): boolean { + return resolveCommandPostActionObservationSupport(command) !== undefined; +} + +export function commandSupportsVerifyEvidence(command: string | undefined): boolean { + return resolveCommandPostActionObservationSupport(command) === 'settle-and-verify'; +} + /** - * The declared timeout policy for a command (ADR-0011). Command names outside + * The declared timeout policy for a command (ADR 0008). Command names outside * the registry (internal probes, unknown commands) fall back to * {@link DEFAULT_TIMEOUT_POLICY} — standard envelope, reset-daemon — exactly as * the deleted hand lists treated unlisted commands. diff --git a/src/core/command-descriptor/timeout-policy.ts b/src/core/command-descriptor/timeout-policy.ts index 3669308a6..ed9b96984 100644 --- a/src/core/command-descriptor/timeout-policy.ts +++ b/src/core/command-descriptor/timeout-policy.ts @@ -1,7 +1,7 @@ import type { CommandTimeoutPolicy } from './types.ts'; // Request-envelope constants, relocated from src/daemon/request-timeouts.ts when -// timeout policy joined the descriptor registry (ADR-0011): the envelopes are now +// timeout policy joined the descriptor registry (ADR 0008): the envelopes are now // declared per command on the descriptors, so their values live beside them. const DAEMON_REQUEST_TIMEOUT_MS = 90_000; @@ -15,7 +15,7 @@ export const INSTALL_REQUEST_TIMEOUT_MS = 180_000; * The timeout policy most commands share: standard envelope, no user-supplied * budget, and a daemon reset on timeout (a hung request usually means daemon * state is suspect). Referenced explicitly by each descriptor — required, not - * inherited — so adding a command forces a decision (ADR-0011). Also the + * inherited — so adding a command forces a decision (ADR 0008). Also the * fallback for command names outside the registry (internal probes, unknown * commands), which matches the old hand lists: not listed meant default * envelope + reset-daemon. diff --git a/src/core/command-descriptor/types.ts b/src/core/command-descriptor/types.ts index f5548f433..1a741917e 100644 --- a/src/core/command-descriptor/types.ts +++ b/src/core/command-descriptor/types.ts @@ -1,5 +1,6 @@ import type { CommandCapability } from '../capabilities.ts'; import type { DaemonCommandDescriptor } from '../../daemon/daemon-command-registry.ts'; +import type { PostActionObservationSupport } from './post-action-observation.ts'; /** * The daemon route + request-policy traits for a command, minus the `command` @@ -11,13 +12,21 @@ import type { DaemonCommandDescriptor } from '../../daemon/daemon-command-regist export type DaemonCommandTraits = Omit; /** - * Where a command's user-facing time budget comes from (ADR-0011, "Timeout - * policy joins the descriptor registry"). + * Where a command's user-facing time budget comes from. The policy lives on + * command descriptors (ADR 0008); ADR 0011 moved the former timeout hand lists + * onto that descriptor surface. * * - `'none'` — the command has no user-supplied budget; the request * envelope is exactly `envelopeMs`. - * - `'flag'` — the `--timeout` flag (`flags.timeoutMs`) overrides the - * envelope when present. + * - `'flag'` — the `--timeout` flag (`flags.timeoutMs`). By default it + * REPLACES the envelope (replay semantics: --timeout + * bounds the request). With `envelope: 'widen'` it only + * ever EXTENDS the envelope to envelopeMs + budget + + * margin (interaction --settle semantics, #1101: the + * flag bounds a post-action wait, so the request must + * also cover selector/action overhead). `defaultBudgetMs` + * is used when the feature flag is present but the + * numeric timeout flag is omitted. * - `'positional-parser'`— the budget travels inside the positionals; `parser` * extracts it (or returns null when none was given). * The client widens the envelope to @@ -25,7 +34,7 @@ export type DaemonCommandTraits = Omit; */ export type CommandTimeoutBudget = | { source: 'none' } - | { source: 'flag' } + | { source: 'flag'; envelope?: 'bound' | 'widen'; defaultBudgetMs?: number } | { source: 'positional-parser'; parser: (positionals: string[]) => number | null }; /** @@ -65,9 +74,13 @@ export type CommandTimeoutPolicy = { * (from STRUCTURED_BATCH_COMMAND_NAMES). * - `mcpExposed` — whether the command is surfaced over MCP. * - `timeoutPolicy` — the request-envelope budget source + on-timeout daemon - * policy (ADR-0011). REQUIRED on every entry — most commands - * share the explicit `DEFAULT_TIMEOUT_POLICY` constant, but a - * new command must say so rather than inherit silently. + * policy. REQUIRED on every entry — most commands share the + * explicit `DEFAULT_TIMEOUT_POLICY` constant, but a new + * command must say so rather than inherit silently. + * - `postActionObservation` — optional interaction observation trait for + * commands that support `--settle`/`--verify`; consumed by + * command surfaces and timeout policy instead of repeated + * command-name lists. * * The registry started dormant (proven byte-equal to the hand tables by the * parity tests) and is now the live source: the daemon registry, capability @@ -81,9 +94,5 @@ export type CommandDescriptor = { batchable: boolean; mcpExposed: boolean; timeoutPolicy: CommandTimeoutPolicy; + postActionObservation?: PostActionObservationSupport; }; - -/** Identity helper that pins each entry to the {@link CommandDescriptor} shape. */ -export function defineCommandDescriptor(descriptor: CommandDescriptor): CommandDescriptor { - return descriptor; -} diff --git a/src/core/wait-positionals.ts b/src/core/wait-positionals.ts index ca6452064..612d9579d 100644 --- a/src/core/wait-positionals.ts +++ b/src/core/wait-positionals.ts @@ -38,7 +38,7 @@ export function parseWaitPositionals(args: string[]): WaitParsed | null { * The user-supplied budget of a `wait` invocation, or null when none was given. * The budget travels as a positional, not a flag, so it is parsed the same way * the daemon will. Referenced by the `wait` descriptor's timeout policy - * (ADR-0011) so the client's request envelope can extend past it. + * (ADR 0008) so the client's request envelope can extend past it. */ export function resolveWaitBudgetMs(positionals: string[]): number | null { const parsed = parseWaitPositionals(positionals); diff --git a/src/daemon/__tests__/response-views.test.ts b/src/daemon/__tests__/response-views.test.ts index cadcc64e5..837df2483 100644 --- a/src/daemon/__tests__/response-views.test.ts +++ b/src/daemon/__tests__/response-views.test.ts @@ -252,3 +252,96 @@ test('snapshot digest preserves refsGeneration — the pinning signal for the re const digest = RESPONSE_VIEWS.snapshot!({ ...SNAPSHOT_DATA, refsGeneration: 7 }, 'digest'); expect(digest.refsGeneration).toBe(7); }); + +// --- #1101 --settle: interaction settle digest view --- + +const SETTLE_DATA: DaemonResponseData = { + ref: 'e2', + x: 200, + y: 322, + message: 'Tapped @e2 (200, 322)', + settle: { + settled: true, + waitedMs: 60, + captures: 2, + quietMs: 25, + timeoutMs: 2000, + refsGeneration: 8, + diff: { + summary: { additions: 1, removals: 1, unchanged: 4 }, + lines: [ + { kind: 'removed', text: '@e2 [button] "Continue"' }, + { kind: 'added', text: '@e4 [text] "Welcome!"', ref: 'e4' }, + ], + }, + }, +}; + +test('interaction settle views are registered for all four touch commands', () => { + expect(typeof RESPONSE_VIEWS.press).toBe('function'); + expect(RESPONSE_VIEWS.press).toBe(RESPONSE_VIEWS.click); + expect(RESPONSE_VIEWS.press).toBe(RESPONSE_VIEWS.fill); + expect(RESPONSE_VIEWS.press).toBe(RESPONSE_VIEWS.longpress); +}); + +test('settle digest keeps the verdict, summary, refsGeneration, and capped added refs; drops the line texts', () => { + const digest = RESPONSE_VIEWS.press!(SETTLE_DATA, 'digest'); + expect(digest.settle).toEqual({ + settled: true, + waitedMs: 60, + captures: 2, + quietMs: 25, + timeoutMs: 2000, + refsGeneration: 8, + refs: [{ ref: 'e4' }], + diff: { summary: { additions: 1, removals: 1, unchanged: 4 } }, + }); + // Every other (cheap) field is preserved verbatim. + expect(digest.ref).toBe('e2'); + expect(digest.message).toBe('Tapped @e2 (200, 322)'); +}); + +test('settle digest caps added refs at the snapshot digest ref limit', () => { + const lines = Array.from({ length: 20 }, (_, index) => ({ + kind: 'added' as const, + text: `@e${index + 1} [button] "Item ${index + 1}"`, + ref: `e${index + 1}`, + })); + const digest = RESPONSE_VIEWS.press!( + { + settle: { + settled: false, + waitedMs: 2000, + captures: 7, + quietMs: 25, + timeoutMs: 2000, + refsGeneration: 9, + diff: { + summary: { additions: 20, removals: 0, unchanged: 0 }, + lines, + truncated: true, + }, + }, + }, + 'digest', + ); + + expect((digest.settle as { refs?: unknown[] }).refs).toHaveLength(12); + expect((digest.settle as { refs?: unknown[] }).refs?.at(0)).toEqual({ ref: 'e1' }); + expect((digest.settle as { refs?: unknown[] }).refs?.at(11)).toEqual({ ref: 'e12' }); +}); + +test('plain interaction responses pass through UNCHANGED at every level', () => { + const plain: DaemonResponseData = { ref: 'e2', x: 200, y: 322, message: 'Tapped @e2' }; + expect(RESPONSE_VIEWS.press!(plain, 'digest')).toBe(plain); + expect(RESPONSE_VIEWS.press!(plain, 'default')).toBe(plain); + expect(RESPONSE_VIEWS.press!(plain, 'full')).toBe(plain); + // A diff-less settle payload (stalled/unstored observation) is already cheap. + const noDiff: DaemonResponseData = { ref: 'e2', settle: { settled: false, hint: 'stalled' } }; + expect(RESPONSE_VIEWS.press!(noDiff, 'digest')).toBe(noDiff); +}); + +test('settle default and full return today’s shape unchanged (same reference)', () => { + expect(RESPONSE_VIEWS.press!(SETTLE_DATA, 'default')).toBe(SETTLE_DATA); + expect(RESPONSE_VIEWS.press!(SETTLE_DATA, 'full')).toBe(SETTLE_DATA); +}); diff --git a/src/daemon/client/daemon-client-timeout.ts b/src/daemon/client/daemon-client-timeout.ts index dd1909697..1244794bd 100644 --- a/src/daemon/client/daemon-client-timeout.ts +++ b/src/daemon/client/daemon-client-timeout.ts @@ -4,6 +4,11 @@ import { emitDiagnostic } from '../../utils/diagnostics.ts'; import { isAgentDeviceDaemonProcess } from '../../utils/process-identity.ts'; import { PUBLIC_COMMANDS } from '../../command-catalog.ts'; import { resolveCommandTimeoutPolicy } from '../../core/command-descriptor/registry.ts'; +import type { + CommandTimeoutBudget, + CommandTimeoutPolicy, +} from '../../core/command-descriptor/types.ts'; +import type { DaemonRequest } from '../types.ts'; import type { DaemonPaths } from '../config.ts'; import { removeDaemonInfo, @@ -18,6 +23,74 @@ const IOS_RUNNER_XCODEBUILD_KILL_PATTERNS = [ 'xcodebuild build-for-testing .*apple-runner/AgentDeviceRunner/AgentDeviceRunner\\.xcodeproj', ]; +type BoundedTimeoutPolicy = CommandTimeoutPolicy & { envelopeMs: number }; +type FlagTimeoutBudget = Extract; +type RequestFlags = Omit['flags']; + +// Derives the request envelope from the command's declared timeout policy +// (ADR 0008) instead of the former per-command-name special cases. +export function resolveDaemonRequestTimeoutMs( + req: Omit, +): number | undefined { + const policy = resolveCommandTimeoutPolicy(req.command); + if (policy.envelopeMs === 'unbounded') return undefined; + const boundedPolicy: BoundedTimeoutPolicy = { ...policy, envelopeMs: policy.envelopeMs }; + return ( + resolvePositionalBudgetTimeoutMs(boundedPolicy, req.positionals ?? []) ?? + resolveFlagBudgetTimeoutMs(boundedPolicy, req.flags) ?? + boundedPolicy.envelopeMs + ); +} + +function resolvePositionalBudgetTimeoutMs( + policy: BoundedTimeoutPolicy, + positionals: string[], +): number | undefined { + if (policy.budget.source !== 'positional-parser') return undefined; + // The user budget travels inside the positionals (e.g. `wait ... 180000`). + // Without extending the envelope past it, the request dies at the default + // timeout with the runner/daemon torn down as collateral (#1075). + const budgetMs = policy.budget.parser(positionals); + return budgetMs === null ? undefined : widenToUserBudget(policy, budgetMs); +} + +function resolveFlagBudgetTimeoutMs( + policy: BoundedTimeoutPolicy, + flags: RequestFlags, +): number | undefined { + if (policy.budget.source !== 'flag') return undefined; + // 'widen' budgets (interaction --settle, #1101) bound an internal wait the + // request must outlive after selector resolution/action overhead. They are + // settle-gated for touch-command back-compat: a bare timeoutMs without + // --settle was historically ignored. Plain 'bound' budgets (replay, + // prepare, snapshot) replace the envelope verbatim. + if (policy.budget.envelope === 'widen') { + return resolveWideningFlagBudget(policy, policy.budget, flags); + } + return typeof flags?.timeoutMs === 'number' ? flags.timeoutMs : policy.envelopeMs; +} + +function resolveWideningFlagBudget( + policy: BoundedTimeoutPolicy, + budget: FlagTimeoutBudget, + flags: RequestFlags, +): number { + if (flags?.settle !== true) return policy.envelopeMs; + const budgetMs = typeof flags.timeoutMs === 'number' ? flags.timeoutMs : budget.defaultBudgetMs; + return typeof budgetMs === 'number' ? widenPastBaseEnvelope(policy, budgetMs) : policy.envelopeMs; +} + +function widenToUserBudget(policy: BoundedTimeoutPolicy, budgetMs: number): number { + return Math.max(policy.envelopeMs, budgetMs + REQUEST_TIMEOUT_BUDGET_MARGIN_MS); +} + +function widenPastBaseEnvelope(policy: BoundedTimeoutPolicy, budgetMs: number): number { + return Math.max( + policy.envelopeMs, + policy.envelopeMs + budgetMs + REQUEST_TIMEOUT_BUDGET_MARGIN_MS, + ); +} + export function handleRequestTimeout( info: DaemonInfo, statePaths: DaemonPaths, @@ -54,7 +127,7 @@ export function handleRequestTimeout( } // Whether a timed-out request tears down the local daemon is declared on the -// command's descriptor (ADR-0011, `timeoutPolicy.onTimeout`): read-only +// command's descriptor (ADR 0008, `timeoutPolicy.onTimeout`): read-only // capture/polling commands preserve the daemon so sessions survive and evidence // commands still work; everything else resets it. Unknown/undefined commands // fall back to the default reset-daemon policy. @@ -112,3 +185,8 @@ function resetDaemonAfterTimeout(info: DaemonInfo, paths: DaemonPaths): { forced } return { forcedKill }; } + +// Margin over a user-supplied budget so the daemon-side timeout result (with +// diagnostics) wins the race against the client envelope. Never shrinks the +// envelope below the command's declared base. +const REQUEST_TIMEOUT_BUDGET_MARGIN_MS = 30_000; diff --git a/src/daemon/client/daemon-client.ts b/src/daemon/client/daemon-client.ts index 2124cc1c1..86640f3a7 100644 --- a/src/daemon/client/daemon-client.ts +++ b/src/daemon/client/daemon-client.ts @@ -5,7 +5,6 @@ import type { import type { RequestProgressSink } from '../request-progress.ts'; import { createRequestId, emitDiagnostic, withDiagnosticTimer } from '../../utils/diagnostics.ts'; import { INTERNAL_COMMANDS, PUBLIC_COMMANDS } from '../../command-catalog.ts'; -import { resolveCommandTimeoutPolicy } from '../../core/command-descriptor/registry.ts'; import { prepareRemoteRequestArtifacts } from '../../remote/daemon-artifacts.ts'; import { cleanupDaemonAfterRequest, @@ -13,6 +12,7 @@ import { resolveClientSettings, } from './daemon-client-lifecycle.ts'; import { sendRequest } from './daemon-client-transport.ts'; +import { resolveDaemonRequestTimeoutMs } from './daemon-client-timeout.ts'; export { computeDaemonCodeSignature } from '../code-signature.ts'; export { downloadRemoteArtifact } from '../../remote/daemon-artifacts.ts'; @@ -21,7 +21,10 @@ export { resolveDaemonStartupHint, } from './daemon-client-metadata.ts'; export { canConnectSocket } from './daemon-client-transport.ts'; -export { shouldResetDaemonAfterRequestTimeout } from './daemon-client-timeout.ts'; +export { + resolveDaemonRequestTimeoutMs, + shouldResetDaemonAfterRequestTimeout, +} from './daemon-client-timeout.ts'; export type DaemonRequest = SharedDaemonRequest; export type DaemonResponse = SharedDaemonResponse; type DaemonTransportOptions = { @@ -116,30 +119,3 @@ function isInstallLikeCommand(command: string | undefined): boolean { command === INTERNAL_COMMANDS.installSource ); } - -// Derives the request envelope from the command's declared timeout policy -// (ADR-0011) instead of the former per-command-name special cases. -export function resolveDaemonRequestTimeoutMs( - req: Omit, -): number | undefined { - const policy = resolveCommandTimeoutPolicy(req.command); - if (policy.envelopeMs === 'unbounded') return undefined; - if (policy.budget.source === 'positional-parser') { - // The user budget travels inside the positionals (e.g. `wait ... 180000`). - // Without extending the envelope past it, the request dies at the default - // timeout with the runner/daemon torn down as collateral (#1075). - const budgetMs = policy.budget.parser(req.positionals ?? []); - if (budgetMs !== null) { - return Math.max(policy.envelopeMs, budgetMs + REQUEST_TIMEOUT_BUDGET_MARGIN_MS); - } - } - if (policy.budget.source === 'flag' && typeof req.flags?.timeoutMs === 'number') { - return req.flags.timeoutMs; - } - return policy.envelopeMs; -} - -// Margin over a user-supplied positional budget so the daemon-side timeout -// result (with its stable/wait diagnostics) wins the race against the client -// envelope. Never shrinks the envelope below the command's declared base. -const REQUEST_TIMEOUT_BUDGET_MARGIN_MS = 30_000; diff --git a/src/daemon/handlers/__tests__/interaction-settle.test.ts b/src/daemon/handlers/__tests__/interaction-settle.test.ts new file mode 100644 index 000000000..b7f22afd2 --- /dev/null +++ b/src/daemon/handlers/__tests__/interaction-settle.test.ts @@ -0,0 +1,329 @@ +import { test, expect, vi, beforeEach } from 'vitest'; +import { handleInteractionCommands } from '../interaction.ts'; +import type { SessionStore } from '../../session-store.ts'; +import type { SessionState } from '../../types.ts'; +import type { CommandFlags } from '../../../core/dispatch.ts'; +import type { SnapshotBackend } from '../../../kernel/snapshot.ts'; +import { buildSnapshotState } from '../snapshot-capture.ts'; +import { setSessionSnapshot } from '../../session-snapshot.ts'; +import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; +import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts'; + +// #1101 --settle daemon response shape: the settle payload (diff + settled + +// refsGeneration) rides the wire response through the shared builder, and a +// diff-carrying settle response is ref-issuing (clears snapshotRefsStale). +// Quiet windows are tuned down (--settle-quiet 25) so no test waits real time +// beyond a few poll ticks. + +vi.mock('../../../core/dispatch.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + dispatchCommand: vi.fn(async () => ({})), + }; +}); + +vi.mock('../interaction-snapshot.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + captureSnapshotForSession: vi.fn(async () => ({ + nodes: [], + createdAt: 0, + backend: 'xctest' as const, + })), + }; +}); + +import { dispatchCommand } from '../../../core/dispatch.ts'; +import { captureSnapshotForSession } from '../interaction-snapshot.ts'; +const mockDispatch = vi.mocked(dispatchCommand); +const mockCaptureSnapshotForSession = vi.mocked(captureSnapshotForSession); + +const BEFORE_NODES = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + rect: { x: 10, y: 20, width: 120, height: 44 }, + hittable: true, + }, +]; + +const AFTER_NODES = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'StaticText', + label: 'Welcome!', + rect: { x: 10, y: 20, width: 120, height: 44 }, + hittable: true, + }, +]; + +async function emulateCaptureSnapshotForSession( + session: SessionState, + flags: CommandFlags | undefined, + sessionStore: SessionStore, + contextFromFlags: ( + flags: CommandFlags | undefined, + appBundleId?: string, + traceLogPath?: string, + ) => Record, + options: { interactiveOnly: boolean }, +) { + const effectiveFlags = { ...(flags ?? {}), snapshotInteractiveOnly: options.interactiveOnly }; + const snapshotData = (await mockDispatch( + session.device, + 'snapshot', + [], + effectiveFlags.out, + contextFromFlags(effectiveFlags, session.appBundleId, session.trace?.outPath), + )) as { nodes?: never[]; truncated?: boolean; backend?: SnapshotBackend }; + const snapshot = buildSnapshotState(snapshotData ?? {}, effectiveFlags); + setSessionSnapshot(session, snapshot); + sessionStore.set(session.name, session); + return snapshot; +} + +function seedSession(sessionName: string, sessionStore: ReturnType) { + const session = makeIosSession(sessionName); + setSessionSnapshot(session, buildSnapshotState({ nodes: BEFORE_NODES, backend: 'xctest' }, {})); + // The seed emulates a snapshot response that issued these refs. + session.snapshotRefsStale = false; + sessionStore.set(sessionName, session); + return session; +} + +function mockCommandDispatch(params: { snapshots: Array }) { + let snapshotCalls = 0; + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'snapshot') { + const nodes = params.snapshots[Math.min(snapshotCalls, params.snapshots.length - 1)]; + snapshotCalls += 1; + return { nodes, backend: 'xctest' }; + } + return {}; + }); +} + +const contextFromFlags = () => ({}); + +beforeEach(() => { + mockDispatch.mockReset(); + mockDispatch.mockResolvedValue({}); + mockCaptureSnapshotForSession.mockReset(); + mockCaptureSnapshotForSession.mockImplementation(emulateCaptureSnapshotForSession); +}); + +const SETTLE_FLAGS = { settle: true, settleQuietMs: 25, timeoutMs: 2_000 }; + +type SettlePayload = { + settled: boolean; + captures: number; + quietMs: number; + timeoutMs: number; + refsGeneration?: number; + diff?: { + summary: { additions: number; removals: number; unchanged: number }; + lines: Array<{ kind: string; text: string; ref?: string }>; + }; + hint?: string; +}; + +function expectOkData( + response: Awaited>, +): Record { + expect(response?.ok).toBe(true); + if (!response || response.ok !== true) throw new Error('expected an ok daemon response'); + return (response.data ?? {}) as Record; +} + +function expectInvalidArgs( + response: Awaited>, +): Record { + expect(response?.ok).toBe(false); + if (!response || response.ok !== false) throw new Error('expected an invalid daemon response'); + expect(response.error?.code).toBe('INVALID_ARGS'); + return (response.error ?? {}) as Record; +} + +test('press --settle responds with the settled diff, refsGeneration, and clears the stale marker', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'settle-press'; + seedSession(sessionName, sessionStore); + // Resolution capture sees the pre-action tree; settle captures see the + // settled post-action tree. + mockCommandDispatch({ snapshots: [BEFORE_NODES, AFTER_NODES, AFTER_NODES, AFTER_NODES] }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'press', + positionals: ['label=Continue'], + flags: { ...SETTLE_FLAGS }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + const data = expectOkData(response); + const settle = data.settle as SettlePayload; + expect(settle).toBeTruthy(); + expect(settle.settled).toBe(true); + expect(settle.quietMs).toBe(25); + expect(settle.timeoutMs).toBe(2_000); + const diff = settle.diff as NonNullable; + expect(diff.summary).toEqual({ additions: 1, removals: 1, unchanged: 1 }); + const added = diff.lines.find((line) => line.kind === 'added'); + expect(added).toEqual({ kind: 'added', text: expect.stringContaining('Welcome!'), ref: 'e2' }); + + const session = sessionStore.get(sessionName) as SessionState; + // The settle response handed the settled tree's refs to the client: the + // coarse marker clears and the payload carries the stored generation. + expect(session.snapshotRefsStale).toBe(false); + expect(settle.refsGeneration).toBe(session.snapshotGeneration); + // The settled tree became the stored session snapshot. + expect(session.snapshot?.nodes.some((node) => node.label === 'Welcome!')).toBe(true); +}); + +test('press --settle keeps the stale-refs input warning while re-issuing fresh refs', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'settle-stale-ref'; + const session = seedSession(sessionName, sessionStore); + session.snapshotRefsStale = true; + sessionStore.set(sessionName, session); + mockCommandDispatch({ snapshots: [AFTER_NODES, AFTER_NODES] }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'press', + positionals: ['@e2'], + flags: { ...SETTLE_FLAGS }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + const data = expectOkData(response); + // The @ref was consumed while stale — the input warning stands… + expect(String(data.warning)).toMatch(/refs were issued/); + // …and the settled diff re-issues refs, so the NEXT @ref command is clean. + expect(sessionStore.get(sessionName)?.snapshotRefsStale).toBe(false); +}); + +test('a settle observation without a diff leaves ref staleness untouched', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'settle-stalled'; + seedSession(sessionName, sessionStore); + let snapshotCalls = 0; + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'snapshot') { + snapshotCalls += 1; + if (snapshotCalls === 1) return { nodes: BEFORE_NODES, backend: 'xctest' }; + throw new Error('AX bridge crashed'); + } + return {}; + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'press', + positionals: ['label=Continue'], + flags: { ...SETTLE_FLAGS }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + // The press still succeeds; the observation reports its own failure. + const data = expectOkData(response); + const settle = data.settle as SettlePayload; + expect(settle.settled).toBe(false); + expect(settle.diff).toBeUndefined(); + expect(settle.hint).toMatch(/Settle observation unavailable/); + // No refs were issued: the resolution capture left the marker stale. + expect(sessionStore.get(sessionName)?.snapshotRefsStale).toBe(true); +}); + +test('bare timeout without --settle stays compatible', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'settle-guard'; + seedSession(sessionName, sessionStore); + mockCommandDispatch({ snapshots: [BEFORE_NODES] }); + + const compatible = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'press', + positionals: ['label=Continue'], + flags: { timeoutMs: 2_000 }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + const data = expectOkData(compatible); + expect(data.settle).toBeUndefined(); +}); + +test('settle-specific tuning flags without --settle are rejected', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'settle-guard'; + seedSession(sessionName, sessionStore); + mockCommandDispatch({ snapshots: [BEFORE_NODES] }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'press', + positionals: ['label=Continue'], + flags: { settleQuietMs: 25 }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + const error = expectInvalidArgs(response); + expect(error.message).toMatch(/--settle-quiet requires --settle/); +}); + +test('fill @ref --settle carries the settle payload on the ref wire shape', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'settle-fill'; + seedSession(sessionName, sessionStore); + mockCommandDispatch({ snapshots: [AFTER_NODES, AFTER_NODES] }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'fill', + positionals: ['@e2', 'hello'], + flags: { ...SETTLE_FLAGS }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + const data = expectOkData(response); + const settle = data.settle as SettlePayload; + expect(settle.settled).toBe(true); + expect(typeof settle.refsGeneration).toBe('number'); +}); diff --git a/src/daemon/handlers/interaction-flags.ts b/src/daemon/handlers/interaction-flags.ts index 5a8d840e1..ed2d0cfee 100644 --- a/src/daemon/handlers/interaction-flags.ts +++ b/src/daemon/handlers/interaction-flags.ts @@ -1,4 +1,6 @@ import type { CommandFlags } from '../../core/dispatch.ts'; +import type { PostActionObservationCommandName } from '../../core/command-descriptor/post-action-observation.ts'; +import type { SettleParams } from '../../contracts/interaction.ts'; import type { DaemonResponse } from '../types.ts'; import { errorResponse } from './response.ts'; @@ -30,3 +32,36 @@ export function unsupportedRefSnapshotFlags(flags: CommandFlags | undefined): st } return unsupported; } + +/** + * `--settle` (#1101) flag grammar for commands carrying the descriptor + * post-action observation trait: `--settle` opts in, `--settle-quiet ` + * overrides the quiet window, and `--timeout ` bounds the settle wait (the + * same budget the descriptor's flag-sourced timeout policy widens the request + * envelope past, mirroring wait's positional budget). Preserve compatibility + * for a bare `--timeout` without `--settle`: older touch commands silently + * ignored it. Only `--settle-quiet` is settle-specific enough to reject when + * orphaned. + */ +export function settleFlagGuardResponse( + command: PostActionObservationCommandName, + flags: CommandFlags | undefined, +): DaemonResponse | null { + if (!flags || flags.settle === true) return null; + const orphaned: string[] = []; + if (flags.settleQuietMs !== undefined) orphaned.push('--settle-quiet'); + if (orphaned.length === 0) return null; + return errorResponse( + 'INVALID_ARGS', + `${command}: ${orphaned.join(', ')} require${orphaned.length === 1 ? 's' : ''} --settle.`, + ); +} + +/** The runtime settle request for a command's flags, or undefined without --settle. */ +export function readSettleRequest(flags: CommandFlags | undefined): SettleParams | undefined { + if (flags?.settle !== true) return undefined; + return { + ...(flags.settleQuietMs !== undefined ? { quietMs: flags.settleQuietMs } : {}), + ...(flags.timeoutMs !== undefined ? { timeoutMs: flags.timeoutMs } : {}), + }; +} diff --git a/src/daemon/handlers/interaction-touch-response.ts b/src/daemon/handlers/interaction-touch-response.ts index e6290b256..e86d6232f 100644 --- a/src/daemon/handlers/interaction-touch-response.ts +++ b/src/daemon/handlers/interaction-touch-response.ts @@ -3,6 +3,7 @@ import type { FillCommandResult, LongPressCommandResult, PressCommandResult, + SettleObservation, } from '../../contracts/interaction.ts'; import { successText } from '../../utils/success-text.ts'; import { interactionResultExtra, stripAtPrefix } from './interaction-touch-targets.ts'; @@ -64,6 +65,15 @@ export function buildInteractionResponseData(params: { * warning. */ staleRefsWarning?: string; + /** + * `--settle` (#1101): the session's `snapshotGeneration` AFTER the settled + * tree became the stored snapshot. Rides INSIDE the settle payload as + * `settle.refsGeneration` — a settle response with a diff hands the client + * fresh refs (added lines carry them), making it ref-issuing like snapshot/ + * find; the generation is what MCP auto-pinning merges per-ref (#1076). + * Only attached when the settle observation actually carries a diff. + */ + settleRefsGeneration?: number; }): InteractionResponsePayloads { const { source, referenceFrame, extra } = params; if (source.kind === 'runner-payload') { @@ -85,6 +95,7 @@ export function buildInteractionResponseData(params: { referenceFrame, extra: { ...interactionResultExtra(result), + ...settleExtra(result.settle, params.settleRefsGeneration), ...(extra ?? {}), }, }); @@ -96,6 +107,7 @@ export function buildInteractionResponseData(params: { ...(result.point ? { x: result.point.x, y: result.point.y } : {}), }), ...interactionResultExtra(result), + ...settleExtra(result.settle, params.settleRefsGeneration), } : visualization; const warning = composeResponseWarning( @@ -109,6 +121,17 @@ export function buildInteractionResponseData(params: { return { result: visualization, responseData }; } +// Attaches refsGeneration inside the settle payload when the response is +// ref-issuing (diff present). Overrides the raw `settle` from +// interactionResultExtra by key order in the extras spread. +function settleExtra( + settle: SettleObservation | undefined, + refsGeneration: number | undefined, +): Record { + if (!settle?.diff || refsGeneration === undefined) return {}; + return { settle: { ...settle, refsGeneration } }; +} + function composeResponseWarning( resultWarning: string | undefined, staleRefsWarning: string | undefined, diff --git a/src/daemon/handlers/interaction-touch-targets.ts b/src/daemon/handlers/interaction-touch-targets.ts index 76eac41d4..b83917ce0 100644 --- a/src/daemon/handlers/interaction-touch-targets.ts +++ b/src/daemon/handlers/interaction-touch-targets.ts @@ -175,8 +175,10 @@ export function interactionResultExtra( ): Record { // `evidence` (#1047, opt-in via --verify) is additive on press/fill only — // LongPressCommandResult has no evidence field, so it reads as undefined - // (and gets dropped by the response layer) for longpress. + // (and gets dropped by the response layer) for longpress. `settle` (#1101, + // opt-in via --settle) is additive on all four touch commands. const evidence = 'evidence' in result ? result.evidence : undefined; + const settle = result.settle; if (result.kind === 'ref') { return { ref: stripAtPrefix(result.target?.kind === 'ref' ? result.target.ref : undefined), @@ -185,6 +187,7 @@ export function interactionResultExtra( targetHittable: result.targetHittable, hint: result.hint, evidence, + settle, }; } if (result.kind === 'selector') { @@ -195,9 +198,10 @@ export function interactionResultExtra( targetHittable: result.targetHittable, hint: result.hint, evidence, + settle, }; } - return { evidence }; + return { evidence, settle }; } export function formatTouchTargetLabel( diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 8973aeadc..d1e4e3af1 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -14,13 +14,17 @@ import type { import { asAppError, normalizeError } from '../../kernel/errors.ts'; import type { DaemonResponse, SessionState } from '../types.ts'; import { finalizeTouchInteraction, type InteractionHandlerParams } from './interaction-common.ts'; -import { resolveRefStalenessWarning } from '../session-snapshot.ts'; +import { markSessionSnapshotRefsIssued, resolveRefStalenessWarning } from '../session-snapshot.ts'; import { buildInteractionResponseData, type InteractionResponsePayloads, } from './interaction-touch-response.ts'; import type { CaptureSnapshotForSession } from './interaction-snapshot.ts'; -import type { RefSnapshotFlagGuardResponse } from './interaction-flags.ts'; +import { + readSettleRequest, + settleFlagGuardResponse, + type RefSnapshotFlagGuardResponse, +} from './interaction-flags.ts'; import { readSnapshotNodesReferenceFrame, resolveDirectTouchReferenceFrameSafely, @@ -41,6 +45,10 @@ import { import { getActiveAndroidSnapshotFreshness } from '../android-snapshot-freshness.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts'; +import { + commandSupportsSettleObservation, + commandSupportsVerifyEvidence, +} from '../../core/command-descriptor/registry.ts'; import { MAESTRO_NON_HITTABLE_FALLBACK_MESSAGE } from '../../core/interactor-types.ts'; import { isDirectIosSelectorFallbackError, @@ -93,6 +101,8 @@ async function dispatchTargetedTouchViaRuntime( if (unsupportedSurfaceResponse) return unsupportedSurfaceResponse; const unsupported = requireCommandSupported(capabilityCommand, session.device); if (unsupported) return unsupported; + const invalidSettleFlags = settleFlagGuardResponse(command, req.flags); + if (invalidSettleFlags) return invalidSettleFlags; const clickButton = resolveClickButton(req.flags); const resultButtonTag = buttonTag(clickButton); @@ -204,11 +214,13 @@ async function runTargetedTouchInteraction(params: { durationMs?: number; }): Promise { const { runtime, command, target, sessionName, requestId, flags } = params; + const settle = readSettleRequest(flags); if (command === 'longpress') { return await runtime.interactions.longPress(target, { session: sessionName, requestId, durationMs: params.durationMs, + settle, }); } @@ -222,6 +234,7 @@ async function runTargetedTouchInteraction(params: { jitterPx: flags?.jitterPx, doubleTap: flags?.doubleTap, verify: flags?.verify, + settle, }; return command === 'click' ? await runtime.interactions.click(target, options) @@ -253,9 +266,29 @@ async function buildTargetedTouchResponsePayloads(params: { referenceFrame, extra, staleRefsWarning: params.staleRefsWarning, + settleRefsGeneration: settleRefsGenerationIssue(session, result), }); } +/** + * #1101 `--settle`: a settle observation carrying a diff hands the client refs + * minted from the freshly stored settled tree (added lines carry them), which + * makes the response ref-issuing like snapshot/find (#1076): the coarse + * `snapshotRefsStale` marker clears (the same accepted coarse blessing as + * find's single re-issued ref) and the stored tree's generation rides inside + * the settle payload for MCP per-ref pinning. Without a diff — never captured, + * or sparse-quality capture that was not stored — nothing was issued and the + * staleness machinery is left untouched. + */ +function settleRefsGenerationIssue( + session: SessionState, + result: PressCommandResult | FillCommandResult | LongPressCommandResult, +): number | undefined { + if (!result.settle?.diff) return undefined; + markSessionSnapshotRefsIssued(session); + return session.snapshotGeneration; +} + function readLongPressResultDuration(result: TargetedTouchResult): number | undefined { return 'durationMs' in result ? result.durationMs : undefined; } @@ -270,7 +303,8 @@ function readDirectIosSelectorTapTarget(params: { if (commandLabel !== 'click') return null; if (target.kind !== 'selector') return null; if (hasNonDefaultClickOptions(flags)) return null; - if (flags?.verify === true) return null; + if (commandSupportsVerifyEvidence(commandLabel) && flags?.verify === true) return null; + if (commandSupportsSettleObservation(commandLabel) && flags?.settle === true) return null; return readDirectSelectorWithMaestroFallback(session, target.selector, flags); } @@ -438,37 +472,26 @@ async function dispatchFillViaRuntime( if (unsupported) return unsupported; } if (!session) return noActiveSessionError(); + const invalidSettleFlags = settleFlagGuardResponse('fill', req.flags); + if (invalidSettleFlags) return invalidSettleFlags; const parsedTarget = parseFillTarget(req.positionals ?? []); if (!parsedTarget.ok) return parsedTarget.response; - // Read before the Android freshness refresh recaptures — see the press path. - const staleRefsWarning = - parsedTarget.target.kind === 'ref' - ? resolveRefStalenessWarning({ - session, - ref: parsedTarget.target.ref, - mintedGeneration: parsedTarget.refGeneration, - }) - : undefined; - if (parsedTarget.target.kind === 'ref') { - const invalidRefFlagsResponse = params.refSnapshotFlagGuardResponse('fill', req.flags); - if (invalidRefFlagsResponse) return invalidRefFlagsResponse; - await refreshAndroidRefSnapshotIfFreshnessActive(params, session); - } - const directSelector = readDirectIosSelectorFillTarget({ + const refPreamble = await prepareFillRefTarget( + params, session, - target: parsedTarget.target, - flags: req.flags, - }); - if (directSelector) { - const directResponse = await dispatchDirectIosSelectorFill( - params, - session, - directSelector, - parsedTarget.text, - ); - if (directResponse) return directResponse; - } + parsedTarget.target, + parsedTarget.refGeneration, + ); + if (refPreamble.response) return refPreamble.response; + const { staleRefsWarning } = refPreamble; + const directResponse = await maybeDispatchDirectIosSelectorFill( + params, + session, + parsedTarget.target, + parsedTarget.text, + ); + if (directResponse) return directResponse; return await dispatchRuntimeInteraction(params, { run: async (runtime) => @@ -477,23 +500,73 @@ async function dispatchFillViaRuntime( requestId: req.meta?.requestId, delayMs: req.flags?.delayMs, verify: req.flags?.verify, + settle: readSettleRequest(req.flags), }), - buildPayloads: (result) => { - const referenceFrame = - result.kind === 'point' - ? undefined - : readSnapshotNodesReferenceFrame(session.snapshot?.nodes ?? []); - return buildInteractionResponseData({ - // refBackendWireShape keeps the historical fill @ref wire response - // (backendResult + identity extras) while the shared builder owns the - // extras — the hand-rolled version of this branch dropped evidence - // (PR #1064 review). - source: { kind: 'runtime', result, refBackendWireShape: true }, - referenceFrame, - extra: { text: parsedTarget.text }, - staleRefsWarning, - }); - }, + buildPayloads: (result) => + buildFillResponsePayloads({ session, result, text: parsedTarget.text, staleRefsWarning }), + }); +} + +// The fill @ref preamble shared with the press path's shape: read staleness +// relative to what the client knew BEFORE any internal recapture, validate +// @ref-incompatible flags, and run the Android freshness refresh. +async function prepareFillRefTarget( + params: InteractionHandlerParams & { + captureSnapshotForSession: CaptureSnapshotForSession; + refSnapshotFlagGuardResponse: RefSnapshotFlagGuardResponse; + }, + session: SessionState, + target: InteractionTarget, + refGeneration: number | undefined, +): Promise<{ response?: DaemonResponse; staleRefsWarning?: string }> { + if (target.kind !== 'ref') return {}; + const staleRefsWarning = resolveRefStalenessWarning({ + session, + ref: target.ref, + mintedGeneration: refGeneration, + }); + const invalidRefFlagsResponse = params.refSnapshotFlagGuardResponse('fill', params.req.flags); + if (invalidRefFlagsResponse) return { response: invalidRefFlagsResponse, staleRefsWarning }; + await refreshAndroidRefSnapshotIfFreshnessActive(params, session); + return { staleRefsWarning }; +} + +async function maybeDispatchDirectIosSelectorFill( + params: InteractionHandlerParams & { captureSnapshotForSession: CaptureSnapshotForSession }, + session: SessionState, + target: InteractionTarget, + text: string, +): Promise { + const directSelector = readDirectIosSelectorFillTarget({ + session, + target, + flags: params.req.flags, + }); + if (!directSelector) return null; + return await dispatchDirectIosSelectorFill(params, session, directSelector, text); +} + +function buildFillResponsePayloads(params: { + session: SessionState; + result: FillCommandResult; + text: string; + staleRefsWarning: string | undefined; +}): InteractionResponsePayloads { + const { session, result } = params; + const referenceFrame = + result.kind === 'point' + ? undefined + : readSnapshotNodesReferenceFrame(session.snapshot?.nodes ?? []); + return buildInteractionResponseData({ + // refBackendWireShape keeps the historical fill @ref wire response + // (backendResult + identity extras) while the shared builder owns the + // extras — the hand-rolled version of this branch dropped evidence + // (PR #1064 review). + source: { kind: 'runtime', result, refBackendWireShape: true }, + referenceFrame, + extra: { text: params.text }, + staleRefsWarning: params.staleRefsWarning, + settleRefsGeneration: settleRefsGenerationIssue(session, result), }); } @@ -504,7 +577,8 @@ function readDirectIosSelectorFillTarget(params: { }): DirectIosSelectorTarget | null { const { session, target, flags } = params; if (target.kind !== 'selector') return null; - if (flags?.verify === true) return null; + if (commandSupportsVerifyEvidence('fill') && flags?.verify === true) return null; + if (commandSupportsSettleObservation('fill') && flags?.settle === true) return null; return readDirectSelectorWithMaestroFallback(session, target.selector, flags); } diff --git a/src/daemon/response-views.ts b/src/daemon/response-views.ts index 8b6660ecf..4f851d104 100644 --- a/src/daemon/response-views.ts +++ b/src/daemon/response-views.ts @@ -123,9 +123,65 @@ function selectorReadView(data: DaemonResponseData, level: ResponseLevel): Daemo return { ...data, node: compactSelectorNode(node as SnapshotNode) }; } +/** + * Token-cheap settle digest for interaction commands (#1101). CONSERVATIVE: + * only acts on a result that carries a `settle.diff` payload (the `--settle` + * opt-in) and otherwise returns the data UNCHANGED, so plain interaction + * responses stay byte-identical at every level. The digest keeps the verdict + * fields and the changed-line COUNTS (`diff.summary`) plus `refsGeneration`, + * and drops the diff line texts — the changed-count summary is the digest + * answer; the lines are the default-level payload. `full` returns today's + * shape unchanged (nothing richer is computed yet). + */ +function interactionSettleView(data: DaemonResponseData, level: ResponseLevel): DaemonResponseData { + if (level !== 'digest') return data; + const settle = data.settle; + if (!settle || typeof settle !== 'object' || Array.isArray(settle)) return data; + const { diff, ...rest } = settle as Record; + if (!diff || typeof diff !== 'object' || Array.isArray(diff)) return data; + const diffRecord = diff as Record; + const summary = diffRecord.summary; + const refs = readSettleDigestRefs(diffRecord.lines); + return { + ...data, + settle: { + ...rest, + ...(refs.length > 0 ? { refs } : {}), + diff: { summary }, + }, + }; +} + +type DigestRef = { ref: string }; + +function readSettleDigestRefs(lines: unknown): DigestRef[] { + if (!Array.isArray(lines)) return []; + return lines.flatMap(readSettleDigestRef).slice(0, DIGEST_REF_LIMIT); +} + +function readSettleDigestRef(line: unknown): DigestRef[] { + const record = readObjectRecord(line); + if (record?.kind !== 'added') return []; + return readDigestRef(record.ref); +} + +function readDigestRef(ref: unknown): DigestRef[] { + return typeof ref === 'string' && ref.length > 0 ? [{ ref }] : []; +} + +function readObjectRecord(value: unknown): Record | undefined { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined; +} + export const RESPONSE_VIEWS: Record = { snapshot: snapshotView, screenshot: screenshotView, find: selectorReadView, get: selectorReadView, + press: interactionSettleView, + click: interactionSettleView, + fill: interactionSettleView, + longpress: interactionSettleView, }; diff --git a/src/daemon/session-snapshot.ts b/src/daemon/session-snapshot.ts index 25ad0b86d..ab6f37c67 100644 --- a/src/daemon/session-snapshot.ts +++ b/src/daemon/session-snapshot.ts @@ -32,9 +32,13 @@ export const STALE_SNAPSHOT_REFS_WARNING = * issuing the full ref set and stays conservative-stale) * * Cleared (set false) only where the client demonstrably receives the new - * refs: the snapshot command response (buildNextSnapshotSession) and find + * refs: the snapshot command response (buildNextSnapshotSession), find * responses that return a ref minted from the freshly stored tree - * (handlers/find.ts, dispatchFindReadOnlyViaRuntime in selector-runtime.ts). + * (handlers/find.ts, dispatchFindReadOnlyViaRuntime in selector-runtime.ts), + * and interaction --settle responses whose settled diff carries refs minted + * from the freshly stored settled tree (settleRefsGenerationIssue in + * handlers/interaction-touch.ts — the same accepted coarse blessing as find's + * single re-issued ref; per-ref precision is the MCP pin layer's job). */ export function setSessionSnapshot(session: SessionState, snapshot: SnapshotState): void { if (session.snapshot !== snapshot) { diff --git a/src/mcp/__tests__/command-tools.test.ts b/src/mcp/__tests__/command-tools.test.ts index f9489f381..5b302b83c 100644 --- a/src/mcp/__tests__/command-tools.test.ts +++ b/src/mcp/__tests__/command-tools.test.ts @@ -416,6 +416,129 @@ test('MCP merges digest-level snapshot refs too', async () => { assert.deepEqual(runCalls[1]?.input, { target: { kind: 'ref', ref: '@e9~s41' } }); }); +// --- #1101 --settle: interaction responses re-pin from the settled diff --- + +test('MCP merges per-ref pins from a settle response diff (merge-only)', async () => { + const runCalls: Array<{ name: string; input: unknown }> = []; + const executor = createCommandToolExecutor({ + createClient: () => ({}) as AgentDeviceClient, + runCommand: async (_client, name, input) => { + runCalls.push({ name, input }); + if (name === 'snapshot') { + return { nodes: [{ ref: 'e2' }, { ref: 'e37' }], truncated: false, refsGeneration: 7 }; + } + if (name === 'press') { + // press @e2 --settle: the settled diff issues e4 at generation 8. + return { + ref: 'e2', + settle: { + settled: true, + waitedMs: 60, + captures: 2, + quietMs: 25, + timeoutMs: 2000, + refsGeneration: 8, + diff: { + summary: { additions: 1, removals: 1, unchanged: 1 }, + lines: [ + { kind: 'removed', text: '@e2 [button] "Continue"' }, + { kind: 'added', text: '@e4 [text] "Welcome!"', ref: 'e4' }, + ], + }, + }, + }; + } + return {}; + }, + }); + + await executor.execute('snapshot', { session: 'demo' }); + await executor.execute('press', { session: 'demo', target: { kind: 'ref', ref: '@e2' } }); + await executor.execute('press', { session: 'demo', target: { kind: 'ref', ref: '@e4' } }); + await executor.execute('get', { + session: 'demo', + format: 'text', + target: { kind: 'ref', ref: '@e37' }, + }); + + // The first press consumed the snapshot pin… + assert.deepEqual(runCalls[1]?.input, { session: 'demo', target: { kind: 'ref', ref: '@e2~s7' } }); + // …its settle diff issued e4 at the settle generation… + assert.deepEqual(runCalls[2]?.input, { session: 'demo', target: { kind: 'ref', ref: '@e4~s8' } }); + // …and refs absent from the diff keep their older pins (merge-only), so the + // daemon can warn precisely about the replaced tree. + assert.deepEqual(runCalls[3]?.input, { + session: 'demo', + format: 'text', + target: { kind: 'ref', ref: '@e37~s7' }, + }); +}); + +test('MCP merges digest-level settle refs too, including never-settled diffs', async () => { + const runCalls: Array<{ name: string; input: unknown }> = []; + const executor = createCommandToolExecutor({ + createClient: () => ({}) as AgentDeviceClient, + runCommand: async (_client, name, input) => { + runCalls.push({ name, input }); + if (name === 'press' && runCalls.length === 1) { + return { + ref: 'e2', + settle: { + settled: false, + waitedMs: 2000, + captures: 7, + quietMs: 25, + timeoutMs: 2000, + refsGeneration: 9, + refs: [{ ref: 'e4' }], + diff: { + summary: { additions: 1, removals: 1, unchanged: 1 }, + }, + hint: 'The UI kept changing for the whole settle budget.', + }, + }; + } + return {}; + }, + }); + + await executor.execute('press', { + session: 'demo', + responseLevel: 'digest', + target: { kind: 'ref', ref: '@e2' }, + settle: true, + }); + await executor.execute('press', { session: 'demo', target: { kind: 'ref', ref: '@e4' } }); + + assert.deepEqual(runCalls[1]?.input, { + session: 'demo', + target: { kind: 'ref', ref: '@e4~s9' }, + }); +}); + +test('MCP leaves pins untouched for plain (non-settle) interaction responses', async () => { + const runCalls: Array<{ name: string; input: unknown }> = []; + const executor = createCommandToolExecutor({ + createClient: () => ({}) as AgentDeviceClient, + runCommand: async (_client, name, input) => { + runCalls.push({ name, input }); + if (name === 'snapshot') { + return { nodes: [{ ref: 'e2' }], truncated: false, refsGeneration: 7 }; + } + // A plain press response has no settle payload and no refsGeneration — + // it must NOT clear the scope (only snapshot/find responses without a + // generation do that). + return { ref: 'e2', x: 10, y: 20 }; + }, + }); + + await executor.execute('snapshot', { session: 'demo' }); + await executor.execute('press', { session: 'demo', target: { kind: 'ref', ref: '@e2' } }); + await executor.execute('press', { session: 'demo', target: { kind: 'ref', ref: '@e2' } }); + + assert.deepEqual(runCalls[2]?.input, { session: 'demo', target: { kind: 'ref', ref: '@e2~s7' } }); +}); + test('MCP passes never-issued refs through unpinned (coarse floor, never guess)', async () => { const runCalls: Array<{ name: string; input: unknown }> = []; const executor = createPinningExecutor(runCalls); diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index 0df701371..4c81dab7b 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -180,6 +180,60 @@ const interactionEvidenceSchema: JsonSchema = objectSchema( ['nodeCount', 'interactiveNodeCount', 'digest', 'changedFromBefore'], ); +// SettleObservation (src/contracts/interaction.ts) — opt-in `--settle` settled +// diff observation (#1101). +const settleObservationSchema: JsonSchema = objectSchema( + { + settled: booleanSchema( + 'Whether the UI held the quiet window before the deadline. false is advisory, not failure.', + ), + waitedMs: numberSchema(), + captures: numberSchema(), + quietMs: numberSchema(), + timeoutMs: numberSchema(), + refsGeneration: numberSchema( + 'Snapshot generation of the stored settled tree; refs on added diff lines were minted from it.', + ), + refs: { + type: 'array', + items: objectSchema( + { + ref: stringSchema('Plain ref body (e12) minted from the stored settled tree.'), + }, + ['ref'], + ), + }, + diff: objectSchema( + { + summary: objectSchema( + { + additions: numberSchema(), + removals: numberSchema(), + unchanged: numberSchema(), + }, + ['additions', 'removals', 'unchanged'], + ), + lines: { + type: 'array', + items: objectSchema( + { + kind: enumSchema(['added', 'removed']), + text: stringSchema(), + ref: stringSchema('Plain ref body (e12) for added lines.'), + }, + ['kind', 'text'], + ), + }, + truncated: booleanSchema('Lines were capped to the response bound.'), + }, + ['summary', 'lines'], + 'Settled diff vs the pre-action tree (changed lines only).', + ), + hint: stringSchema(), + }, + ['settled', 'waitedMs', 'captures', 'quietMs', 'timeoutMs'], +); + // boot / shutdown share the resolved-device header (src/contracts/device.ts). const deviceHeaderProperties: Record = { platform: enumSchema(PLATFORMS), @@ -210,6 +264,7 @@ export const COMMAND_OUTPUT_SCHEMAS = { message: stringSchema(), warning: stringSchema(), evidence: interactionEvidenceSchema, + settle: settleObservationSchema, }, }), fill: interactionResultSchema({ @@ -219,6 +274,7 @@ export const COMMAND_OUTPUT_SCHEMAS = { backendResult: backendResultSchema, message: stringSchema(), evidence: interactionEvidenceSchema, + settle: settleObservationSchema, }, required: ['text'], }), @@ -228,6 +284,7 @@ export const COMMAND_OUTPUT_SCHEMAS = { backendResult: backendResultSchema, message: stringSchema(), warning: stringSchema(), + settle: settleObservationSchema, }, }), diff --git a/src/mcp/command-tools.ts b/src/mcp/command-tools.ts index 5e493a323..e8e571f5e 100644 --- a/src/mcp/command-tools.ts +++ b/src/mcp/command-tools.ts @@ -113,6 +113,22 @@ export function createCommandToolExecutor(deps: CommandToolExecutorDeps = {}): C */ const REF_ISSUING_TOOLS: ReadonlySet = new Set(['snapshot', 'find'] as CommandName[]); +/** + * `--settle` (#1101) makes an interaction response CONDITIONALLY ref-issuing: + * when it carries `settle.diff` + `settle.refsGeneration`, the diff's added + * lines hand out refs minted from the freshly stored settled tree. These tools + * are NOT in REF_ISSUING_TOOLS on purpose — a plain (non-settle) press carries + * no generation, and treating that as "issuing response without a generation" + * would clear the scope's pins on every ordinary tap. Absent or diff-less + * settle payloads leave pins untouched. + */ +const SETTLE_REF_ISSUING_TOOLS: ReadonlySet = new Set([ + 'press', + 'click', + 'fill', + 'longpress', +] as CommandName[]); + const TARGET_REF_TOOLS: ReadonlySet = new Set([ 'press', 'click', @@ -155,6 +171,10 @@ function mergeIssuedRefPins( name: CommandName, result: unknown, ): void { + if (SETTLE_REF_ISSUING_TOOLS.has(name)) { + mergeSettleIssuedRefPins(refPinsByScope, scopeKey, result); + return; + } if (!REF_ISSUING_TOOLS.has(name)) return; const record = asOptionalRecord(result); const refsGeneration = record?.refsGeneration; @@ -169,6 +189,33 @@ function mergeIssuedRefPins( recordIssuedPins(pins, issuedRefs, refsGeneration); } +/** + * MERGE-ONLY, like the snapshot/find rule: refs on the settled diff's added + * lines move to the settle generation; every other pin stays put (the settle + * capture replaced the tree, so an old pin on an unchanged-looking element is + * exactly what makes the daemon warn precisely). No settle payload, no diff, + * no digest refs, or no generation → not an issuing response; pins are left + * untouched. + */ +function mergeSettleIssuedRefPins( + refPinsByScope: Map>, + scopeKey: string, + result: unknown, +): void { + const settle = asOptionalRecord(asOptionalRecord(result)?.settle); + if (!settle) return; + const refsGeneration = settle?.refsGeneration; + if (typeof refsGeneration !== 'number') return; + const lines = asOptionalRecord(settle?.diff)?.lines; + const issuedRefs: string[] = []; + collectRefBodies(lines, issuedRefs); + collectRefBodies(settle.refs, issuedRefs); + if (issuedRefs.length === 0) return; + const pins = refPinsByScope.get(scopeKey) ?? new Map(); + refPinsByScope.set(scopeKey, pins); + recordIssuedPins(pins, issuedRefs, refsGeneration); +} + function recordIssuedPins( pins: Map, issuedRefs: string[], diff --git a/src/snapshot/snapshot-diff.ts b/src/snapshot/snapshot-diff.ts index 8febe3d87..991e106f0 100644 --- a/src/snapshot/snapshot-diff.ts +++ b/src/snapshot/snapshot-diff.ts @@ -9,6 +9,13 @@ import { export type SnapshotDiffLine = { kind: 'added' | 'removed' | 'unchanged'; text: string; + /** + * Plain ref body (`e12`) of the CURRENT-tree node behind an added line. + * Only populated with `withRefs` (interaction `--settle`, #1101) so the + * `diff` command's wire shape stays byte-identical. Removed/unchanged lines + * never carry it: a removed line's ref names a node of the replaced tree. + */ + ref?: string; }; export type SnapshotDiffSummary = { @@ -24,11 +31,14 @@ export type SnapshotDiffResult = { type SnapshotDiffOptions = { flatten?: boolean; + /** Attach the current-tree node ref to added lines (see SnapshotDiffLine.ref). */ + withRefs?: boolean; }; type SnapshotComparableLine = { text: string; comparable: string; + ref?: string; }; function snapshotNodeToComparableLine(node: SnapshotNode, depthOverride?: number): string { @@ -73,11 +83,13 @@ function snapshotNodesToLines( return nodes.map((node) => ({ text: formatSnapshotLine(node, 0, false), comparable: snapshotNodeToComparableLine(node, 0), + ...(options.withRefs && node.ref ? { ref: node.ref } : {}), })); } return buildSnapshotDisplayLines(nodes).map((line) => ({ text: line.text, comparable: snapshotNodeToComparableLine(line.node, line.depth), + ...(options.withRefs && line.node.ref ? { ref: line.node.ref } : {}), })); } @@ -96,7 +108,7 @@ function diffComparableLinesMyers( for (let d = 0; d <= max; d += 1) { trace.push(new Map(v)); for (let k = -d; k <= d; k += 2) { - const goDown = k === -d || (k !== d && getV(v, k - 1) < getV(v, k + 1)); + const goDown = shouldGoDown(v, k, d); let x = goDown ? getV(v, k + 1) : getV(v, k - 1) + 1; let y = x - k; while (x < n && y < m && previous[x]!.comparable === current[y]!.comparable) { @@ -127,7 +139,7 @@ function backtrackMyers( for (let d = trace.length - 1; d >= 0; d -= 1) { const v = trace[d]!; const k = x - y; - const goDown = k === -d || (k !== d && getV(v, k - 1) < getV(v, k + 1)); + const goDown = shouldGoDown(v, k, d); const prevK = goDown ? k + 1 : k - 1; const prevX = getV(v, prevK); const prevY = prevX - prevK; @@ -141,7 +153,8 @@ function backtrackMyers( if (d === 0) break; if (x === prevX) { - lines.push({ kind: 'added', text: current[prevY]!.text }); + const added = current[prevY]!; + lines.push({ kind: 'added', text: added.text, ...(added.ref ? { ref: added.ref } : {}) }); y = prevY; } else { lines.push({ kind: 'removed', text: previous[prevX]!.text }); @@ -153,6 +166,10 @@ function backtrackMyers( return lines; } +function shouldGoDown(v: Map, k: number, d: number): boolean { + return k === -d || (k !== d && getV(v, k - 1) < getV(v, k + 1)); +} + function getV(v: Map, k: number): number { return v.get(k) ?? 0; } diff --git a/src/utils/__tests__/daemon-client.test.ts b/src/utils/__tests__/daemon-client.test.ts index eadf01bc7..29a47722b 100644 --- a/src/utils/__tests__/daemon-client.test.ts +++ b/src/utils/__tests__/daemon-client.test.ts @@ -265,6 +265,57 @@ test('wait request timeout extends past the user-supplied wait budget', () => { assert.equal(resolveDaemonRequestTimeoutMs({ ...base, command: 'wait' }), 90_000); }); +test('interaction --settle budgets add post-action settle time on top of the normal envelope', () => { + const base = { + session: 'default', + positionals: ['@e2'], + meta: {}, + }; + + // --timeout bounds the SETTLE wait after selector resolution and the action, + // so the envelope keeps the normal touch-command overhead and then adds the + // settle budget plus the same safety margin used by wait. + assert.equal( + resolveDaemonRequestTimeoutMs({ + ...base, + command: 'press', + flags: { settle: true, timeoutMs: 120_000 }, + }), + 240_000, + ); + // A small settle deadline still needs the normal touch-command envelope plus + // room for post-action observation. + assert.equal( + resolveDaemonRequestTimeoutMs({ + ...base, + command: 'fill', + flags: { settle: true, timeoutMs: 5_000 }, + }), + 125_000, + ); + // Bare --settle uses the settle loop's default budget, so a slow pre-action + // capture still leaves room for the post-action observation to report. + assert.equal( + resolveDaemonRequestTimeoutMs({ + ...base, + command: 'longpress', + flags: { settle: true }, + }), + 130_000, + ); + // Bare timeoutMs without --settle remains wire-compatible with older touch + // command clients: it is ignored instead of opting into settle semantics. + assert.equal( + resolveDaemonRequestTimeoutMs({ + ...base, + command: 'press', + flags: { timeoutMs: 120_000 }, + }), + 90_000, + ); + assert.equal(resolveDaemonRequestTimeoutMs({ ...base, command: 'press', flags: {} }), 90_000); +}); + test('snapshot uses the standard daemon request timeout with an explicit override', () => { const base = { session: 'default', diff --git a/test/integration/interaction-contract/coordinate.contract.test.ts b/test/integration/interaction-contract/coordinate.contract.test.ts index 1ebea0178..6d60098ee 100644 --- a/test/integration/interaction-contract/coordinate.contract.test.ts +++ b/test/integration/interaction-contract/coordinate.contract.test.ts @@ -5,7 +5,7 @@ import { makeSnapshotState } from '../../../src/__tests__/test-utils/index.ts'; import { assertRpcError, assertRpcOk } from '../provider-scenarios/assertions.ts'; import { scenarioName } from './coverage-manifest.ts'; import { COORDINATE_COVERAGE } from './coordinate.coverage.ts'; -import { viewportOnlySnapshot } from './fixtures.ts'; +import { settledWelcomeSnapshot, viewportOnlySnapshot } from './fixtures.ts'; import { createContractDevice } from './runtime-harness.ts'; import { runnerTapEntry, runnerTapErrorEntry, withIosContractDaemon } from './daemon-harness.ts'; @@ -52,6 +52,31 @@ test(scenario('verifyEvidence'), async () => { assert.equal(result.evidence?.changedFromBefore, true); }); +test(scenario('settleObservation'), async () => { + let captureCount = 0; + const device = createContractDevice(viewportOnlySnapshot(), { + // Point targets normally skip captures entirely; --settle opts into the + // evidence-baseline capture (first call) and the settle loop's captures. + captureSnapshot: async () => { + captureCount += 1; + if (captureCount === 1) return { snapshot: viewportOnlySnapshot() }; + return { snapshot: settledWelcomeSnapshot() }; + }, + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press( + { kind: 'point', x: 10, y: 20 }, + { session: 'default', settle: { quietMs: 25, timeoutMs: 2_000 } }, + ); + + assert.equal(result.kind, 'point'); + const settle = result.settle; + assert.ok(settle, 'point press --settle must return a settle observation'); + assert.equal(settle.settled, true); + assert.deepEqual(settle.diff?.summary, { additions: 1, removals: 1, unchanged: 0 }); +}); + test(scenario('errorTaxonomy'), async () => { await withIosContractDaemon( [runnerTapErrorEntry(new Error('runner tap crashed'))], diff --git a/test/integration/interaction-contract/coordinate.coverage.ts b/test/integration/interaction-contract/coordinate.coverage.ts index e5ea5de53..efdce51ba 100644 --- a/test/integration/interaction-contract/coordinate.coverage.ts +++ b/test/integration/interaction-contract/coordinate.coverage.ts @@ -6,5 +6,7 @@ export const COORDINATE_COVERAGE = definePathCoverage('coordinate', { 'coordinate responseConstruction: daemon press x y response carries the canonical point field set', verifyEvidence: 'coordinate verifyEvidence: point click --verify returns a digest with change detection', + settleObservation: + 'coordinate settleObservation: point press --settle captures a baseline and returns the settled diff', errorTaxonomy: 'coordinate errorTaxonomy: backend failure surfaces as a normalized daemon error', }); diff --git a/test/integration/interaction-contract/direct-ios-selector.contract.test.ts b/test/integration/interaction-contract/direct-ios-selector.contract.test.ts index e682e6771..21a26cce4 100644 --- a/test/integration/interaction-contract/direct-ios-selector.contract.test.ts +++ b/test/integration/interaction-contract/direct-ios-selector.contract.test.ts @@ -102,6 +102,45 @@ test(scenario('verifyEvidence'), async () => { ); }); +test(scenario('settleObservation'), async () => { + await withIosContractDaemon( + [ + // --settle disables the direct path: runtime tree capture, coordinate + // tap, then the settle loop's two stable captures of the changed tree. + runnerSnapshotEntry(RUNNER_CONTINUE_NODES), + runnerTapEntry({ x: 200, y: 322 }), + runnerSnapshotEntry(RUNNER_CHANGED_NODES), + runnerSnapshotEntry(RUNNER_CHANGED_NODES), + ], + async (daemon, transcript) => { + const click = await daemon.callCommand('click', ['label=Continue'], { + settle: true, + settleQuietMs: 25, + timeoutMs: 2_000, + }); + const data = assertRpcOk(click); + + assert.equal(transcript.calls[0]?.command, 'ios.runner.snapshot'); + const tapRequest = transcript.calls.find((call) => call.command === 'ios.runner.tap') + ?.request as Record | undefined; + assert.equal(tapRequest?.selectorKey, undefined); + assert.equal(tapRequest?.x, 200); + + const settle = data.settle as Record | undefined; + assert.ok(settle, 'click --settle must return a settle observation'); + assert.equal(settle.settled, true); + assert.equal(typeof settle.refsGeneration, 'number'); + const diff = settle.diff as + | { summary: Record; lines: Array> } + | undefined; + assert.ok(diff, 'settle observation must carry the diff'); + assert.deepEqual(diff.summary, { additions: 1, removals: 1, unchanged: 1 }); + // The settled tree itself is never serialized into the response. + assert.equal(data.nodes, undefined); + }, + ); +}); + test(scenario('errorTaxonomy'), async () => { await withIosContractDaemon( [ diff --git a/test/integration/interaction-contract/direct-ios-selector.coverage.ts b/test/integration/interaction-contract/direct-ios-selector.coverage.ts index 0df1e1f80..c9fcb015a 100644 --- a/test/integration/interaction-contract/direct-ios-selector.coverage.ts +++ b/test/integration/interaction-contract/direct-ios-selector.coverage.ts @@ -13,4 +13,6 @@ export const DIRECT_IOS_SELECTOR_COVERAGE = definePathCoverage('direct-ios-selec 'direct-ios-selector responseConstruction: runner payload response carries the canonical selector field set', verifyEvidence: 'direct-ios-selector verifyEvidence: --verify disables the direct path and returns runtime evidence', + settleObservation: + 'direct-ios-selector settleObservation: --settle disables the direct path and returns the runtime settled diff', }); diff --git a/test/integration/interaction-contract/fixtures.ts b/test/integration/interaction-contract/fixtures.ts index ab4438370..232dd73d6 100644 --- a/test/integration/interaction-contract/fixtures.ts +++ b/test/integration/interaction-contract/fixtures.ts @@ -180,6 +180,22 @@ export function nonHittableButtonSnapshot(): SnapshotState { ]); } +// Post-action settled tree for --settle scenarios: vs continueButtonSnapshot +// the Continue button is replaced by a Welcome text — exactly one addition and +// one removal in the settled diff, with the added line carrying its ref. +export function settledWelcomeSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'StaticText', + label: 'Welcome!', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: true, + }, + ]); +} + // Viewport-only tree for coordinate scenarios. export function viewportOnlySnapshot(): SnapshotState { return makeSnapshotState([ diff --git a/test/integration/interaction-contract/native-ref.contract.test.ts b/test/integration/interaction-contract/native-ref.contract.test.ts index c3eaf62af..a167b1637 100644 --- a/test/integration/interaction-contract/native-ref.contract.test.ts +++ b/test/integration/interaction-contract/native-ref.contract.test.ts @@ -8,6 +8,7 @@ import { NATIVE_REF_COVERAGE } from './native-ref.coverage.ts'; import { closedDrawerSnapshot, continueButtonSnapshot, + settledWelcomeSnapshot, coveredButtonSnapshot, nonHittableCellSnapshot, } from './fixtures.ts'; @@ -116,6 +117,33 @@ test(scenario('verifyEvidence'), async () => { assert.ok(result.evidence?.digest.startsWith('ax1:')); }); +test(scenario('settleObservation'), async () => { + const calls: string[] = []; + const device = createContractDevice(continueButtonSnapshot(), { + platform: 'web', + captureSnapshot: async () => ({ snapshot: settledWelcomeSnapshot() }), + tap: async () => ({ ok: true }), + tapTarget: async (_context, target) => { + calls.push(target.ref); + return {}; + }, + }); + + const result = await device.interactions.click(ref('@e1'), { + session: 'default', + settle: { quietMs: 25, timeoutMs: 2_000 }, + }); + + // --settle delegates to the runtime-ref path: the fast path is skipped so + // the baseline and the settle captures exist. + assert.deepEqual(calls, []); + assert.equal(result.kind, 'ref'); + const settle = result.settle; + assert.ok(settle, 'click @ref --settle must return a settle observation'); + assert.equal(settle.settled, true); + assert.deepEqual(settle.diff?.summary, { additions: 1, removals: 1, unchanged: 0 }); +}); + test(scenario('responseIdentity'), async () => { const calls: string[] = []; const device = createNativeRefDevice(continueButtonSnapshot(), calls); diff --git a/test/integration/interaction-contract/native-ref.coverage.ts b/test/integration/interaction-contract/native-ref.coverage.ts index 5cf6c04ba..c9ed53d15 100644 --- a/test/integration/interaction-contract/native-ref.coverage.ts +++ b/test/integration/interaction-contract/native-ref.coverage.ts @@ -10,6 +10,8 @@ export const NATIVE_REF_COVERAGE = definePathCoverage('native-ref', { responseIdentity: 'native-ref responseIdentity: fast-path result echoes the ref target and backend result', verifyEvidence: 'native-ref verifyEvidence: --verify skips the fast path and returns evidence', + settleObservation: + 'native-ref settleObservation: --settle skips the fast path and returns the settled diff', // The preflight raises the runtime path's exact offscreen_ref shape (code, // reason, hint), which is the shared taxonomy on this path. errorTaxonomy: diff --git a/test/integration/interaction-contract/runtime-ref.contract.test.ts b/test/integration/interaction-contract/runtime-ref.contract.test.ts index ee642b066..850e4de48 100644 --- a/test/integration/interaction-contract/runtime-ref.contract.test.ts +++ b/test/integration/interaction-contract/runtime-ref.contract.test.ts @@ -12,6 +12,7 @@ import { coveredButtonSnapshot, nonHittableCellSnapshot, RUNNER_CONTINUE_NODES, + settledWelcomeSnapshot, } from './fixtures.ts'; import { createContractDevice } from './runtime-harness.ts'; import { runnerSnapshotEntry, runnerTapEntry, withIosContractDaemon } from './daemon-harness.ts'; @@ -92,6 +93,28 @@ test(scenario('verifyEvidence'), async () => { assert.ok(result.evidence?.digest.startsWith('ax1:')); }); +test(scenario('settleObservation'), async () => { + // The @ref resolves against the STORED session tree (no resolution capture); + // the settle loop's captures see the post-action tree. + const device = createContractDevice(continueButtonSnapshot(), { + captureSnapshot: async () => ({ snapshot: settledWelcomeSnapshot() }), + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press(ref('@e1'), { + session: 'default', + settle: { quietMs: 25, timeoutMs: 2_000 }, + }); + + assert.equal(result.kind, 'ref'); + const settle = result.settle; + assert.ok(settle, 'press @ref --settle must return a settle observation'); + assert.equal(settle.settled, true); + // Baseline is the stored pre-action tree the ref was resolved on. + assert.deepEqual(settle.diff?.summary, { additions: 1, removals: 1, unchanged: 0 }); + assert.equal(settle.diff?.lines.find((line) => line.kind === 'added')?.ref, 'e1'); +}); + test(scenario('errorTaxonomy'), async () => { const device = createContractDevice(continueButtonSnapshot(), { tap: async () => ({ ok: true }), diff --git a/test/integration/interaction-contract/runtime-ref.coverage.ts b/test/integration/interaction-contract/runtime-ref.coverage.ts index 323696084..eb9e30c2f 100644 --- a/test/integration/interaction-contract/runtime-ref.coverage.ts +++ b/test/integration/interaction-contract/runtime-ref.coverage.ts @@ -9,5 +9,7 @@ export const RUNTIME_REF_COVERAGE = definePathCoverage('runtime-ref', { responseIdentity: 'runtime-ref responseIdentity: result echoes the ref target and resolved node', verifyEvidence: 'runtime-ref verifyEvidence: click @ref --verify returns a digest with change detection', + settleObservation: + 'runtime-ref settleObservation: press @ref --settle diffs the settled tree against the stored baseline', errorTaxonomy: 'runtime-ref errorTaxonomy: unknown ref fails with the stale-ref hint', }); diff --git a/test/integration/interaction-contract/runtime-selector.contract.test.ts b/test/integration/interaction-contract/runtime-selector.contract.test.ts index 37c1cd8c0..be08df132 100644 --- a/test/integration/interaction-contract/runtime-selector.contract.test.ts +++ b/test/integration/interaction-contract/runtime-selector.contract.test.ts @@ -14,6 +14,7 @@ import { edgeGrazingDrawerSnapshot, nonHittableButtonSnapshot, RUNNER_CONTINUE_NODES, + settledWelcomeSnapshot, } from './fixtures.ts'; import { createContractDevice } from './runtime-harness.ts'; import { runnerSnapshotEntry, runnerTapEntry, withIosContractDaemon } from './daemon-harness.ts'; @@ -133,6 +134,35 @@ test(scenario('verifyEvidence'), async () => { assert.ok(result.evidence?.digest.startsWith('ax1:')); }); +test(scenario('settleObservation'), async () => { + const before = continueButtonSnapshot(); + const after = settledWelcomeSnapshot(); + let captures = 0; + const device = createContractDevice(before, { + // Resolution capture sees the pre-action tree; every settle capture sees + // the (already stable) post-action tree. + captureSnapshot: async () => ({ snapshot: captures++ === 0 ? before : after }), + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + settle: { quietMs: 25, timeoutMs: 2_000 }, + }); + + assert.equal(result.kind, 'selector'); + const settle = result.settle; + assert.ok(settle, 'press --settle must return a settle observation'); + assert.equal(settle.settled, true); + assert.ok(settle.captures >= 2); + assert.deepEqual(settle.diff?.summary, { additions: 1, removals: 1, unchanged: 0 }); + const added = settle.diff?.lines.find((line) => line.kind === 'added'); + assert.match(added?.text ?? '', /Welcome!/); + // Fresh refs ride the diff: the added line's ref resolves on the stored + // settled tree. + assert.equal(added?.ref, 'e1'); +}); + test(scenario('errorTaxonomy'), async () => { const device = createContractDevice(continueButtonSnapshot(), { tap: async () => ({ ok: true }), diff --git a/test/integration/interaction-contract/runtime-selector.coverage.ts b/test/integration/interaction-contract/runtime-selector.coverage.ts index 27e1db9d1..530f7cda3 100644 --- a/test/integration/interaction-contract/runtime-selector.coverage.ts +++ b/test/integration/interaction-contract/runtime-selector.coverage.ts @@ -14,6 +14,8 @@ export const RUNTIME_SELECTOR_COVERAGE = definePathCoverage('runtime-selector', 'runtime-selector responseIdentity: result echoes selectorChain and the resolved node', verifyEvidence: 'runtime-selector verifyEvidence: press --verify returns a digest with change detection', + settleObservation: + 'runtime-selector settleObservation: press --settle returns the settled diff with fresh refs', errorTaxonomy: 'runtime-selector errorTaxonomy: no-match failure carries the shared code and hint', }); diff --git a/test/integration/provider-scenarios/doctor.test.ts b/test/integration/provider-scenarios/doctor.test.ts index 476cc6cca..9b283eacb 100644 --- a/test/integration/provider-scenarios/doctor.test.ts +++ b/test/integration/provider-scenarios/doctor.test.ts @@ -62,7 +62,7 @@ test('Provider-backed integration doctor infers Android RN/Metro readiness throu } finally { await server.close(); } -}); +}, 15_000); test('Provider-backed integration doctor runs predictably for supported platform selectors', async () => { const devices = [ @@ -94,7 +94,7 @@ test('Provider-backed integration doctor runs predictably for supported platform } }, ); -}); +}, 15_000); test('Provider-backed integration doctor --app verifies an installed app without opening a session', async () => { const adbCalls: string[][] = []; diff --git a/test/integration/provider-scenarios/settle-observation.test.ts b/test/integration/provider-scenarios/settle-observation.test.ts new file mode 100644 index 000000000..dfd0b9304 --- /dev/null +++ b/test/integration/provider-scenarios/settle-observation.test.ts @@ -0,0 +1,246 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { assertRpcOk } from './assertions.ts'; +import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; +import { + createAppleRunnerProviderFromTranscript, + createRecordingAppleToolProvider, + simctlListDevicesHandler, +} from './providers.ts'; +import { createProviderTranscript, type ProviderScenarioProviderEntry } from './transcript.ts'; + +const APP = 'com.example.app'; +const DEVICE_ID = PROVIDER_SCENARIO_IOS_SIMULATOR.id; + +// #1101 --settle end-to-end: press executes the tap, the settle loop captures +// until the tree goes quiet, and the SAME response carries the settled diff +// (with fresh refs + refsGeneration) — the follow-up observation round trip +// and its stale-ref hazard both disappear. + +const BEFORE_NODES = [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + hittable: true, + rect: { x: 100, y: 300, width: 200, height: 44 }, + }, + { + index: 2, + parentIndex: 0, + type: 'Button', + label: 'Cancel', + hittable: true, + rect: { x: 100, y: 400, width: 200, height: 44 }, + }, +]; + +const SETTLED_NODES = [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Done', + hittable: true, + rect: { x: 100, y: 500, width: 200, height: 44 }, + }, +]; + +function snapshotEntry(nodes: readonly unknown[]): ProviderScenarioProviderEntry { + return { + command: 'ios.runner.snapshot', + deviceId: DEVICE_ID, + platform: 'apple', + result: { nodes, truncated: false }, + }; +} + +function tapEntry(x: number, y: number): ProviderScenarioProviderEntry { + return { + command: 'ios.runner.tap', + deviceId: DEVICE_ID, + platform: 'apple', + result: { x, y }, + }; +} + +test('Provider-backed integration press --settle returns the settled diff and fresh refs', async () => { + const runnerTranscript = createProviderTranscript([ + // snapshot -i: issues refs + snapshotEntry(BEFORE_NODES), + // press label=Continue --settle: resolution capture, tap, settle captures + snapshotEntry(BEFORE_NODES), + tapEntry(200, 322), + snapshotEntry(SETTLED_NODES), + snapshotEntry(SETTLED_NODES), + // press @e2 (the Done ref from the settled diff): tap on the stored tree + tapEntry(200, 522), + ]); + const appleRunnerProvider = createAppleRunnerProviderFromTranscript( + runnerTranscript, + 'ios.runner', + ); + const appleTool = createRecordingAppleToolProvider({ + simctl: simctlListDevicesHandler('com.apple.CoreSimulator.SimRuntime.iOS-18-0', [ + { name: PROVIDER_SCENARIO_IOS_SIMULATOR.name, udid: DEVICE_ID }, + ]), + }); + + await withProviderScenarioResource( + async () => + await createProviderScenarioHarness({ + appleRunnerProvider: () => appleRunnerProvider, + appleToolProvider: () => appleTool.provider, + deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], + }), + async (daemon) => { + const open = await daemon.callCommand('open', [APP], { + platform: 'ios', + udid: DEVICE_ID, + }); + assertRpcOk(open); + + const snapshot = await daemon.callCommand('snapshot', [], { + snapshotInteractiveOnly: true, + }); + assertRpcOk(snapshot); + + const press = await daemon.callCommand('press', ['label=Continue'], { + settle: true, + settleQuietMs: 25, + timeoutMs: 2_000, + }); + const pressData = assertRpcOk(press); + const settle = pressData.settle as { + settled: boolean; + captures: number; + refsGeneration?: number; + diff?: { + summary: { additions: number; removals: number; unchanged: number }; + lines: Array<{ kind: string; text: string; ref?: string }>; + }; + hint?: string; + }; + assert.ok(settle, 'press --settle must return a settle observation'); + assert.equal(settle.settled, true); + assert.equal(settle.captures, 2); + assert.equal(typeof settle.refsGeneration, 'number'); + assert.deepEqual(settle.diff?.summary, { additions: 1, removals: 2, unchanged: 1 }); + const added = settle.diff?.lines.find((line) => line.kind === 'added'); + assert.match(added?.text ?? '', /Done/); + assert.equal(added?.ref, 'e2'); + // The settled tree is never serialized into the response. + assert.equal(pressData.nodes, undefined); + + // The diff's ref acts directly on the stored settled tree — no fresh + // snapshot round trip and no stale-refs warning. + const followUp = await daemon.callCommand('press', ['@e2'], {}); + const followUpData = assertRpcOk(followUp); + assert.equal(followUpData.warning, undefined); + assert.equal(followUpData.x, 200); + assert.equal(followUpData.y, 522); + + runnerTranscript.assertComplete(); + }, + ); +}); + +test('Provider-backed integration never-settled press --settle keeps added refs actionable', async () => { + const loadingNodes = (label: string) => [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + parentIndex: 0, + type: 'StaticText', + label, + hittable: true, + rect: { x: 100, y: 300, width: 200, height: 44 }, + }, + ]; + const runnerTranscript = createProviderTranscript([ + // press label=Continue --settle: resolution capture, tap, then a changing + // settle stream. The loop times out and stores the last capture. + snapshotEntry(BEFORE_NODES), + tapEntry(200, 322), + snapshotEntry(loadingNodes('Loading 1')), + snapshotEntry(loadingNodes('Loading 2')), + snapshotEntry(SETTLED_NODES), + // press @e2 (the Done ref from the never-settled final diff): tap on the + // stored last tree, with no fresh snapshot round trip. + tapEntry(200, 522), + ]); + const appleRunnerProvider = createAppleRunnerProviderFromTranscript( + runnerTranscript, + 'ios.runner', + ); + const appleTool = createRecordingAppleToolProvider({ + simctl: simctlListDevicesHandler('com.apple.CoreSimulator.SimRuntime.iOS-18-0', [ + { name: PROVIDER_SCENARIO_IOS_SIMULATOR.name, udid: DEVICE_ID }, + ]), + }); + + await withProviderScenarioResource( + async () => + await createProviderScenarioHarness({ + appleRunnerProvider: () => appleRunnerProvider, + appleToolProvider: () => appleTool.provider, + deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], + }), + async (daemon) => { + const open = await daemon.callCommand('open', [APP], { + platform: 'ios', + udid: DEVICE_ID, + }); + assertRpcOk(open); + + const press = await daemon.callCommand('press', ['label=Continue'], { + settle: true, + settleQuietMs: 25, + timeoutMs: 60, + }); + const pressData = assertRpcOk(press); + const settle = pressData.settle as { + settled: boolean; + refsGeneration?: number; + diff?: { + summary: { additions: number; removals: number; unchanged: number }; + lines: Array<{ kind: string; text: string; ref?: string }>; + }; + hint?: string; + }; + assert.equal(settle.settled, false); + assert.equal(typeof settle.refsGeneration, 'number'); + assert.match(settle.hint ?? '', /kept changing/); + const added = settle.diff?.lines.find((line) => line.kind === 'added'); + assert.match(added?.text ?? '', /Done/); + assert.equal(added?.ref, 'e2'); + + const followUp = await daemon.callCommand('press', ['@e2'], {}); + const followUpData = assertRpcOk(followUp); + assert.equal(followUpData.warning, undefined); + assert.equal(followUpData.x, 200); + assert.equal(followUpData.y, 522); + + runnerTranscript.assertComplete(); + }, + ); +}); diff --git a/test/skillgym/suites/agent-device-smoke-suite.ts b/test/skillgym/suites/agent-device-smoke-suite.ts index 8dfc0fba8..c33855f8a 100644 --- a/test/skillgym/suites/agent-device-smoke-suite.ts +++ b/test/skillgym/suites/agent-device-smoke-suite.ts @@ -423,6 +423,8 @@ const RAW_COORDINATE_TARGET = /(?:^|\n)(?:agent-device\s+)?(?:click|fill|press)\s+-?\d+(?:\.\d+)?\s+-?\d+(?:\.\d+)?/i; const PSEUDO_ASSERTION_COMMAND = /(?:^|\n)\s*(?:assert|assertVisible|waitFor|waitForText)\b/i; const RAW_RECT_SNAPSHOT = /snapshot\b(?=[^\n]*-i\b)(?=[^\n]*(?:--json|--raw))/i; +const SHELL_OUTPUT_PROJECTION = /(?:2>\s*\/dev\/null|\|\s*(?:jq|grep|head|tail)\b)/i; +const ROLE_NAME_SELECTOR_KEY = /(?:^|[\s'"])button\s*=/i; const BOUNDED_PROFILE_SLOW = /react-devtools\s+profile\s+slow\b[^\n]*--limit\s+(?:5|10)\b/i; const BOUNDED_PROFILE_RERENDERS = /react-devtools\s+profile\s+rerenders\b[^\n]*--limit\s+(?:5|10)\b/i; @@ -993,6 +995,94 @@ const SKILL_GUIDANCE_CASES: Case[] = [ /--text\b/i, ], }), + makeCase({ + id: 'network-search-settle-then-wait', + contract: [ + 'App name: Agent Device Tester', + 'Current screen: Catalog search tab', + 'Search field selector: id="catalog-search"', + 'Search results are network-backed and may arrive after the local UI has gone quiet', + 'Expected result text: Alpenglow Trail Mug', + ], + task: 'Plan commands to search for "alpenglow" with the durable field selector, use --settle for the fill action, then wait for the network-backed result text. Do not verify this by taking repeated snapshots.', + outputs: [ + plannedCommand('fill'), + /id=(?:["']catalog-search["']|catalog-search)/i, + /alpenglow/i, + /--settle\b/i, + plannedCommand('wait text'), + /Alpenglow Trail Mug/i, + ], + forbiddenOutputs: [ + SHELL_OUTPUT_PROJECTION, + /(?:^|\n)(?:agent-device\s+)?snapshot\b[\s\S]*(?:^|\n)(?:agent-device\s+)?snapshot\b/i, + RAW_COORDINATE_TARGET, + ], + strictFinalOutput: true, + allowOnlyLocalCliHelpCommands: true, + }), + makeCase({ + id: 'raw-output-before-shell-projection', + contract: [ + 'App name: Agent Device Tester', + 'Current screen: Home tab', + 'The target control has no stable selector in the task context', + 'Fresh interactive snapshot will expose the target as @e12', + 'agent-device output includes refs, warnings, hints, and diagnostics needed for the next step', + ], + task: 'Plan robust commands to inspect interactive refs, press the discovered @e12 target, then verify the nearby change. Do not pipe, grep, jq, or hide command output while exploring.', + outputs: [ + /snapshot -i/i, + /(?:^|\n)(?:agent-device\s+)?(?:press|click)\s+@e12\b/i, + /(?:diff snapshot -i|snapshot\b.*(?:-i\b.*--diff|--diff\b.*-i\b)|wait\b|find\b)/i, + ], + forbiddenOutputs: [SHELL_OUTPUT_PROJECTION, RAW_COORDINATE_TARGET], + strictFinalOutput: true, + allowOnlyLocalCliHelpCommands: true, + }), + makeCase({ + id: 'selector-role-filter-not-role-key', + contract: [ + 'App name: Agent Device Tester', + 'Current screen: Feed directory', + 'Fresh interactive snapshot shows @e37 [button] "Search for more feeds"', + 'If a role filter is needed, the supported selector shape is role=button label="Search for more feeds"', + 'button="Search for more feeds" is not a valid selector key', + 'Need the resulting UI after pressing this control', + ], + task: 'Plan the command to press Search for more feeds with settle. Prefer the visible @e37 ref or a valid selector; do not invent a role-name selector key.', + outputs: [ + plannedCommandAlternatives(['press', 'click']), + /(?:@e37|label=(?:["']Search for more feeds["']|Search for more feeds)|role=button)/i, + /--settle\b/i, + ], + forbiddenOutputs: [ROLE_NAME_SELECTOR_KEY, RAW_COORDINATE_TARGET], + strictFinalOutput: true, + allowOnlyLocalCliHelpCommands: true, + }), + makeCase({ + id: 'settle-diff-is-observation', + contract: [ + 'App name: Agent Device Tester', + 'Previous command: press @e37 --settle', + 'The settled diff already showed + @e64 [text-field] "Search"', + 'The settled diff already showed + @e65 [text] "Recent searches"', + 'The task was to confirm the feed-search UI and close the session', + 'No more refs or visible evidence are needed', + ], + task: 'Plan the next command. Do not take another snapshot just to re-read evidence that was already present in the settled diff.', + outputs: [plannedCommand('close')], + forbiddenOutputs: [ + plannedCommand('snapshot'), + plannedCommand('wait'), + plannedCommand('find'), + plannedCommand('get'), + plannedCommand('is'), + plannedCommandAlternatives(['press', 'click']), + ], + strictFinalOutput: true, + allowOnlyLocalCliHelpCommands: true, + }), makeCase({ id: 'text-replace-uses-fill', contract: [