diff --git a/AxePlaygroundApp/AxePlayground/ContentView.swift b/AxePlaygroundApp/AxePlayground/ContentView.swift index 3a8b11e..f290ee6 100644 --- a/AxePlaygroundApp/AxePlayground/ContentView.swift +++ b/AxePlaygroundApp/AxePlayground/ContentView.swift @@ -65,6 +65,16 @@ struct ContentView: View { SearchableTestView() case "toolbar-picker-test": ToolbarPickerTestView() + case "alert-test": + AlertTestView() + case "sheet-test": + SheetTestView() + case "context-menu-test": + ContextMenuTestView() + case "modal-navigation-test": + ModalNavigationTestView() + case "long-scroll-test": + LongScrollTestView() // Input & Text case "text-input": @@ -117,6 +127,13 @@ struct MainMenuView: View { ("slider-value-test", "Slider Value Test", "Numeric AXValue with selector tap"), ("searchable-test", "Searchable Test", "Navigation search field targeting"), ("toolbar-picker-test", "Toolbar Picker Test", "Toolbar segmented picker targeting") + ]), + ("Presentation", [ + ("alert-test", "Alert Test", "Alert presentation and button targeting"), + ("sheet-test", "Sheet Test", "Sheet presentation and actions"), + ("context-menu-test", "Context Menu Test", "Long press menu targeting"), + ("modal-navigation-test", "Modal Navigation Test", "Modal route and nested navigation refresh"), + ("long-scroll-test", "Long Scroll Test", "Dedicated long scroll coverage") ]) ] diff --git a/AxePlaygroundApp/AxePlayground/Views/AccessibilityFixturesView.swift b/AxePlaygroundApp/AxePlayground/Views/AccessibilityFixturesView.swift index 93cda18..a2697e5 100644 --- a/AxePlaygroundApp/AxePlayground/Views/AccessibilityFixturesView.swift +++ b/AxePlaygroundApp/AxePlayground/Views/AccessibilityFixturesView.swift @@ -112,3 +112,193 @@ struct ToolbarPickerTestView: View { } } } + +struct AlertTestView: View { + @State private var state = "Initial" + @State private var isShowingAlert = false + + var body: some View { + VStack(spacing: 24) { + Text("Alert State: \(state)") + .accessibilityIdentifier("alert-test-state") + .accessibilityValue(state) + + Button("Show Alert") { + isShowingAlert = true + } + .buttonStyle(.borderedProminent) + .accessibilityIdentifier("alert-test-show-alert") + } + .padding() + .navigationTitle("Alert Test") + .navigationBarTitleDisplayMode(.inline) + .alert("Delete Draft?", isPresented: $isShowingAlert) { + Button("Cancel", role: .cancel) { + state = "Cancelled" + } + Button("Delete", role: .destructive) { + state = "Deleted" + } + } message: { + Text("This alert is used for deterministic automation coverage.") + } + } +} + +struct SheetTestView: View { + @State private var state = "Initial" + @State private var isShowingSheet = false + + var body: some View { + VStack(spacing: 24) { + Text("Sheet State: \(state)") + .accessibilityIdentifier("sheet-test-state") + .accessibilityValue(state) + + Button("Open Sheet") { + isShowingSheet = true + } + .buttonStyle(.borderedProminent) + .accessibilityIdentifier("sheet-test-open-sheet") + } + .padding() + .navigationTitle("Sheet Test") + .navigationBarTitleDisplayMode(.inline) + .sheet(isPresented: $isShowingSheet) { + NavigationStack { + VStack(spacing: 24) { + Text("Sheet Fixture") + .font(.title2) + .fontWeight(.bold) + + Button("Run Sheet Action") { + state = "Sheet action tapped" + } + .buttonStyle(.borderedProminent) + .accessibilityIdentifier("sheet-test-action") + + Button("Close Sheet") { + isShowingSheet = false + } + .buttonStyle(.bordered) + .accessibilityIdentifier("sheet-test-close") + } + .padding() + .navigationTitle("Sheet Fixture") + .navigationBarTitleDisplayMode(.inline) + .accessibilityIdentifier("sheet-test-sheet") + } + .presentationDetents([.medium]) + } + } +} + +struct ContextMenuTestView: View { + @State private var state = "Initial" + + var body: some View { + VStack(spacing: 24) { + Text("Context Menu State: \(state)") + .accessibilityIdentifier("context-menu-test-state") + .accessibilityValue(state) + + Text("Long Press Target") + .font(.headline) + .frame(maxWidth: .infinity) + .padding() + .background(.blue.opacity(0.15), in: RoundedRectangle(cornerRadius: 12)) + .accessibilityIdentifier("context-menu-test-target") + .contextMenu { + Button("Favorite") { + state = "Favorited" + } + Button("Archive") { + state = "Archived" + } + } + } + .padding() + .navigationTitle("Context Menu") + .navigationBarTitleDisplayMode(.inline) + } +} + +struct ModalNavigationTestView: View { + @State private var state = "Initial" + @State private var isShowingModal = false + + var body: some View { + VStack(spacing: 24) { + Text("Modal Navigation State: \(state)") + .accessibilityIdentifier("modal-navigation-test-state") + .accessibilityValue(state) + + Button("Open Modal Flow") { + isShowingModal = true + } + .buttonStyle(.borderedProminent) + .accessibilityIdentifier("modal-navigation-test-open") + } + .padding() + .navigationTitle("Modal Navigation") + .navigationBarTitleDisplayMode(.inline) + .sheet(isPresented: $isShowingModal) { + NavigationStack { + List { + NavigationLink("Open Detail") { + VStack(spacing: 24) { + Text("Modal Detail") + .font(.title2) + .accessibilityIdentifier("modal-navigation-test-detail") + + Button("Mark Complete") { + state = "Completed" + } + .buttonStyle(.borderedProminent) + .accessibilityIdentifier("modal-navigation-test-complete") + } + .padding() + .navigationTitle("Modal Detail") + .navigationBarTitleDisplayMode(.inline) + } + .accessibilityIdentifier("modal-navigation-test-detail-link") + } + .navigationTitle("Modal Flow") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .topBarTrailing) { + Button("Done") { + isShowingModal = false + } + .accessibilityIdentifier("modal-navigation-test-done") + } + } + .accessibilityIdentifier("modal-navigation-test-modal") + } + } + } +} + +struct LongScrollTestView: View { + private let rows = Array(1...80) + + var body: some View { + List { + Text("Long Scroll Start") + .font(.headline) + .accessibilityIdentifier("long-scroll-test-start") + + ForEach(rows, id: \.self) { row in + Text("Long Scroll Row \(row)") + .accessibilityIdentifier("long-scroll-test-row-\(row)") + } + + Text("Long Scroll End") + .font(.headline) + .accessibilityIdentifier("long-scroll-test-end") + } + .navigationTitle("Long Scroll") + .navigationBarTitleDisplayMode(.inline) + .accessibilityIdentifier("long-scroll-test-list") + } +} diff --git a/BATCHING.md b/BATCHING.md deleted file mode 100644 index 020f330..0000000 --- a/BATCHING.md +++ /dev/null @@ -1,286 +0,0 @@ -# AXe Batch Guide - -This guide explains `axe batch` in simple, behavior-first terms. - -## What `axe batch` does - -`axe batch` runs multiple interaction steps in order in a single command. - -Why this matters: -- Lower overhead than launching a new AXe process for every step. -- Better latency for automation flows. -- Easier scripting for multi-step interactions. - -## Quick example - -```bash -axe batch --udid SIMULATOR_UDID \ - --step "tap --id SearchField" \ - --step "type 'hello world'" \ - --step "key 40" -``` - -What this does: -1. Taps the element with accessibility id `SearchField`. -2. Types `hello world`. -3. Presses keycode `40` (Enter/Return). - -## Supported step commands - -You can use these inside `--step` lines or step files: -- `tap` -- `swipe` -- `gesture` -- `touch` -- `type` -- `button` -- `key` -- `key-sequence` -- `key-combo` - -Batch-only pseudo-step: -- `sleep ` - -Example: -```bash -axe batch --udid SIMULATOR_UDID \ - --step "tap -x 180 -y 360" \ - --step "sleep 0.5" \ - --step "tap -x 220 -y 420" -``` - -## Input source rules - -Batch accepts exactly one input source per run: -- `--step` (repeatable inline steps) -- `--file` (one step per line) -- `--stdin` (one step per line) - -If you combine sources in one command, batch fails validation. - -## Arguments and behavior - -### `--udid ` -Required. The simulator to target. - -### `--step "..."` -Adds one step inline. Repeat for multiple steps. - -Example: -```bash ---step "tap --id LoginButton" --step "type 'cam@example.com'" -``` - -### `--file ` -Reads steps from a file (one step per line). - -Rules: -- Empty lines are ignored. -- Lines that start with `#` are treated as comments. - -Example file: -```text -# login flow -tap --id EmailField -type 'cam@example.com' -key 43 -type 'super-secret' -key 40 -``` - -### `--stdin` -Reads steps from stdin (one step per line). - -Example: -```bash -cat steps.txt | axe batch --stdin --udid SIMULATOR_UDID -``` - -### `--ax-cache ` -Controls how accessibility snapshots are reused for selector-based taps (`tap --id` / `tap --label`). - -- `perBatch` (default): fetch once, reuse during the whole batch. - - Fastest. -- `perStep`: fetch again for each selector step. - - Better if UI changes a lot between steps. -- `none`: same behavior as per-step (no cache kept). - -### `--type-submission ` -Controls how `type` events are submitted. - -- `chunked` (default): splits typing HID events into chunks. - - Safer for very long text. -- `composite`: sends all typing HID events together. - - Fewer submissions, can be faster for moderate text sizes. - -### `--type-chunk-size ` -Used when `--type-submission chunked`. - -Default: `200` - -Larger value: -- fewer submissions -- potentially faster - -Smaller value: -- more submissions -- potentially more stable for very large text payloads - -### `--wait-timeout ` -Maximum time to poll for selector-based elements (`tap --id` / `tap --label`) before failing. - -Default: `0` (no waiting — fail immediately if the element is not found). - -When set to a positive value, batch polls the accessibility tree at regular intervals until the element appears or the timeout expires. This is useful for multi-screen flows where a tap triggers navigation and the next tap targets an element on the new screen. - -Example: -```bash -axe batch --udid SIMULATOR_UDID --wait-timeout 5 \ - --step "tap --id LoginButton" \ - --step "tap --id WelcomeMessage" -``` - -The second step polls for up to 5 seconds for `WelcomeMessage` to appear after the login tap triggers navigation. - -Only `.notFound` errors are retried. If the selector matches multiple elements or the matched element has an invalid frame, the step fails immediately without retrying. - -### `--poll-interval ` -How frequently the accessibility tree is re-fetched when `--wait-timeout` is active. - -Default: `0.25` - -Lower values poll more aggressively (faster detection, more overhead). Higher values reduce overhead but increase detection latency. - -### `--tap-style ` -Controls the default tap event style for tap steps. - -- `automatic` (default): uses physical touch down/up for matched switches/toggles and FBSimulator `tapAt` for other targets. -- `simulator`: always uses FBSimulator `tapAt`. -- `physical`: always uses touch down/up. - -A tap step can override the batch default with its own `--tap-style` option. - -### `--verbose` -Enables detailed stderr logging for troubleshooting. - -Default: disabled (quiet output, success/failure summary only). - -Use this when debugging selector resolution, setup, or retries. - -### `--continue-on-error` -Controls failure policy. - -Default behavior (without this flag): -- Fail fast. -- Batch stops at first failing step. - -With this flag: -- Batch keeps running later steps even if one step fails. -- At the end, it reports all failed steps. - -Use carefully, because later steps might run in an unexpected UI state. - -## Step-level options - -Each step command keeps its normal options and validation. - -Examples: -- `tap --id BackButton` -- `tap --label 'Weather Alerts'` -- `tap -x 200 -y 400 --pre-delay 0.2 --post-delay 0.2` -- `swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100` -- `tap -x 320 -y 780 --tap-style physical` -- `gesture scroll-down --duration 1.0` -- `touch -x 150 -y 300 --down --up --delay 1.0` - -Selector tap steps use the same activation behavior as direct `axe tap`: when a matched row or label contains exactly one UIKit `UISwitch` or SwiftUI `Toggle`/switch control, AXe taps the switch control instead of the row's geometric center. With the default `--tap-style automatic`, those switch/toggle activations use physical touch down/up; normal taps use FBSimulator `tapAt`. - -One important rule: -- Do not include `--udid` inside a step. Use the batch-level `--udid` only. - -Coordinate-based `tap`, `swipe`, and `touch` steps use the same orientation-aware coordinate mapping as the standalone commands. Use coordinates from `describe-ui` directly; AXe detects rotated landscape simulator orientation and letterboxed landscape-only app layouts automatically. - -## Verification strategy - -Batch is execution-focused. It does not do built-in assertions. - -Recommended pattern: -1. Run `axe batch ...` -2. Verify with `axe describe-ui ...` or `axe screenshot ...` - -## Real-world example: login flow - -```bash -axe batch --udid SIMULATOR_UDID --file login.steps -``` - -`login.steps`: -```text -tap --id EmailField -type 'cam@example.com' -key 43 -type 'super-secret' -key 40 -``` - -Then verify: -```bash -axe describe-ui --udid SIMULATOR_UDID > post-login-ui.json -``` - -## Real-world example: navigation flow - -```bash -axe batch --udid SIMULATOR_UDID \ - --step "gesture scroll-down" \ - --step "tap --label Settings" \ - --step "sleep 0.5" \ - --step "tap --id SaveButton" -``` - -## Real-world example: toggling a setting - -```bash -axe batch --udid SIMULATOR_UDID \ - --step "tap --label 'Weather Alerts'" -``` - -If the `Weather Alerts` row contains one switch/toggle control, AXe taps that control's activation point using physical touch in automatic tap style. - - -## Real-world example: multi-screen flow with element waiting - -```bash -axe batch --udid SIMULATOR_UDID --wait-timeout 5 \ - --step "tap --id LoginButton" \ - --step "tap --id DashboardTitle" \ - --step "tap --label Profile" -``` - -Each selector tap polls for up to 5 seconds, so if `LoginButton` triggers a screen transition, `DashboardTitle` will be found once the new screen loads. - -## Troubleshooting - -- Error: multiple input sources - - Use only one of: `--step`, `--file`, or `--stdin`. - -- Error: no element matched in `tap --id` / `tap --label` - - Confirm current screen. - - Run `axe describe-ui --udid ...` to refresh selectors. - - Use `--wait-timeout ` if the element appears after a previous step triggers navigation. - - Consider `--ax-cache perStep` if your UI changes between steps. - -- Error: multiple elements matched for `--label` - - Labels are not guaranteed to be unique. - - AXe prefers actionable matches first (for example, `Button`) when a label is shared with read-only text, but it still fails if multiple actionable matches remain. - - Prefer `--id` when available. - - If AXe reports that none of the matches expose `AXUniqueId`, use coordinate taps (`tap -x/-y`) for that step. - -- Output is too noisy - - Keep default quiet mode for normal runs. - - Add `--verbose` only when you need troubleshooting details. - -- Steps succeed but final state is wrong - - Add `sleep` between navigation-heavy steps. - - Verify after batch with `describe-ui` / `screenshot`. - - Avoid `--continue-on-error` unless you really want best-effort execution. diff --git a/Package.swift b/Package.swift index a9065c7..7d43852 100644 --- a/Package.swift +++ b/Package.swift @@ -56,10 +56,7 @@ let package = Package( .testTarget( name: "AXeTests", dependencies: ["AXe", "AXeCore"], - path: "Tests", - resources: [ - .copy("README.md") - ] + path: "Tests" ), .plugin( name: "VersionPlugin", diff --git a/README.md b/README.md index 411f0cc..c94269c 100644 --- a/README.md +++ b/README.md @@ -5,451 +5,52 @@ AXe is a comprehensive CLI tool for interacting with iOS Simulators using Apple' [![CI](https://github.com/cameroncooke/AXe/actions/workflows/release.yml/badge.svg)](https://github.com/cameroncooke/AXe/actions/workflows/release.yml) [![Licence: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -![AxeDemo](https://github.com/user-attachments/assets/9eafa5d5-3cef-4e39-82c5-7b9d41fe4548) - -- [Features](#features) - - [Touch \& Gestures](#touch--gestures) - - [Input \& Text](#input--text) - - [Hardware Buttons](#hardware-buttons) - - [Timing Controls](#timing-controls) - - [Accessibility](#accessibility) -- [Quick Start](#quick-start) - - [Installation](#installation) - - [Install via Homebrew](#install-via-homebrew) - - [Build from source](#build-from-source) - - [Basic Usage](#basic-usage) - - [Install AXe Skill](#install-axe-skill) -- [Commands Overview](#commands-overview) - - [**Touch \& Gestures**](#touch--gestures-1) - - [**Sliders**](#sliders) - - [**Gesture Presets**](#gesture-presets) - - [**Text Input**](#text-input) - - [**Hardware Buttons**](#hardware-buttons-1) - - [**Keyboard Control**](#keyboard-control) - - [**Batch Chaining**](#batch-chaining) - - [**Video Streaming**](#video-streaming) - - [**Screenshot**](#screenshot) - - [**Accessibility \& Info**](#accessibility--info) -- [Architecture](#architecture) - - [Why AXe?](#why-axe) -- [Gesture Presets Reference](#gesture-presets-reference) -- [Benchmarking](#benchmarking) -- [Batch Guide](#batch-guide) -- [Contributing](#contributing) -- [Licence](#licence) - - -## Features - -AXe provides complete iOS Simulator automation capabilities: - -### Touch & Gestures -- **Tap**: Precise touch events at specific coordinates with timing controls -- **Swipe**: Multi-touch gestures with configurable duration and delta -- **Drag**: Raw point-to-point low-level HID drags with explicit touch move events -- **Touch Control**: Low-level touch down/up events for advanced gesture control -- **Sliders**: Set slider controls to a verified 0-100 percentage tolerance by accessibility identifier or label -- **Gesture Presets**: Common gesture patterns (scroll-up, scroll-down, scroll-left, scroll-right, edge swipes) -- **Batch Chaining**: Execute ordered multi-step interaction workflows in one invocation - -### Input & Text -- **Text Input**: Comprehensive text typing with automatic shift key handling -- **Key Presses**: Individual key presses by HID keycode -- **Key Sequences**: Multi-key sequences with timing control -- **Key Combos**: Atomic modifier+key combinations (e.g., Cmd+A, Cmd+Shift+Z) -- **Multiple Input Methods**: Direct text, stdin, or file input - -### Hardware Buttons -- **Home Button**: iOS home button simulation -- **Lock/Power Button**: Power button with duration control -- **Side Button**: iPhone X+ side button -- **Siri Button**: Siri activation button -- **Apple Pay Button**: Apple Pay button simulation - -### Timing Controls -- **Pre/Post Delays**: Configurable delays before and after actions -- **Duration Control**: Precise timing for gestures and button presses -- **Sequence Timing**: Custom delays between key sequences -- **Complex Automation**: Multi-step workflows with precise timing - -### Video & Screenshots -- **Screenshot Capture**: Capture simulator display as PNG with automatic or custom filenames -- **Screenshot-based Streaming**: Capture simulator video at 1-30 FPS -- **Multiple Output Formats**: MJPEG, raw JPEG, ffmpeg-compatible, BGRA -- **H.264 Recording**: Use the `record-video` command to write MP4 files with hardware-friendly encoding -- **Configurable Quality**: Adjust JPEG quality and scale factor -- **Real-time Performance**: Efficient frame timing for smooth playback - -### Accessibility -- **UI Description**: Extract accessibility information from any point or full screen -- **Simulator Management**: List available simulators - -## Quick Start - -### Installation - -#### Install via Homebrew +## Install ```bash -# Install via Homebrew brew tap cameroncooke/axe brew install axe - -# Or single liner -brew install cameroncooke/axe/axe - -# Use directly -axe --help -``` - -#### Build from source - -For development work: - -```bash -# Clone the repository -git clone https://github.com/cameroncooke/AXe.git -cd AXe - -# Build required XCFrameworks (not checked into the repo) -./scripts/build.sh dev - -# Run directly with Swift after frameworks are built -swift run axe --help -swift run axe list-simulators - -# Build for development -swift build -.build/debug/axe --help ``` -#### Makefile shortcuts +Or install in one command: ```bash -make build # swift build -make test # default tests (non-E2E) -make e2e # full simulator E2E flow via test-runner.sh -make clean +brew install cameroncooke/axe/axe ``` -### Basic Usage +Verify the CLI: ```bash -# List available simulators +axe --help axe list-simulators - -# Get simulator UDID -UDID="B34FF305-5EA8-412B-943F-1D0371CA17FF" - -# Basic interactions -axe tap -x 100 -y 200 --udid $UDID -axe tap --id "Safari" --udid $UDID -axe tap --label "Safari" --udid $UDID -axe tap --label "Weather Alerts" --udid $UDID # Auto-uses physical touch for matched switches/toggles -axe tap -x 320 -y 780 --tap-style physical --udid $UDID # Force physical touch for coordinate taps -axe slider --id "volume-slider" --value 75 --udid $UDID -axe type 'Hello World!' --udid $UDID -axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid $UDID -axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --udid $UDID -axe button home --udid $UDID - -# Screenshot -axe screenshot --udid $UDID - -# Gesture presets -axe gesture scroll-up --udid $UDID -axe gesture swipe-from-left-edge --udid $UDID - -# With timing controls (NEW!) -axe tap -x 100 -y 200 --pre-delay 1.0 --post-delay 0.5 --udid $UDID -axe gesture scroll-down --pre-delay 0.5 --post-delay 1.0 --udid $UDID ``` -### Install AXe Skill +## Basic usage ```bash -# Auto-detect installed clients (~/.claude or ~/.agents) -axe init - -# Non-interactive usage: pass an explicit target -axe init --client claude - -# Install directly to a custom skills directory -axe init --dest ~/.claude/skills - -# Print bundled skill content only -axe init --print - -# Remove installed AXe skill directory -axe init --uninstall --client agents -``` - -## Commands Overview - -### **Touch & Gestures** - -```bash -# Tap at coordinates -axe tap -x 100 -y 200 --udid SIMULATOR_UDID -axe tap -x 100 -y 200 --pre-delay 1.0 --post-delay 0.5 --udid SIMULATOR_UDID - -# Tap by element (uses describe-ui view tree) -axe tap --id "Safari" --udid SIMULATOR_UDID -axe tap --label "Safari" --udid SIMULATOR_UDID -axe tap --label "Weather Alerts" --udid SIMULATOR_UDID # Switches/toggles use their activation point and physical touch when matched -axe tap --label "Submit" --tap-style simulator --udid SIMULATOR_UDID # Force FBSimulator tapAt -axe tap -x 320 -y 780 --tap-style physical --udid SIMULATOR_UDID # Force touch down/up - -# Swipe gestures -axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid SIMULATOR_UDID -axe swipe --start-x 50 --start-y 500 --end-x 350 --end-y 500 --duration 2.0 --delta 25 --udid SIMULATOR_UDID - -# Raw low-level drag using explicit touch move events -axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --udid SIMULATOR_UDID -axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --duration 0.4 --steps 40 --udid SIMULATOR_UDID - -# Orientation-aware coordinates -# AXe automatically maps logical UI coordinates for rotated landscape and letterboxed landscape-only apps. -# Use the coordinates from describe-ui directly; AXe detects the simulator orientation. -axe tap -x 100 -y 200 --udid SIMULATOR_UDID -axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid SIMULATOR_UDID -axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --udid SIMULATOR_UDID - -# Advanced touch control -axe touch -x 150 -y 250 --down --udid SIMULATOR_UDID -axe touch -x 150 -y 250 --up --udid SIMULATOR_UDID -axe touch -x 150 -y 250 --down --up --udid SIMULATOR_UDID -# Long press (hold for 1 second) -axe touch -x 150 -y 250 --down --up --delay 1.0 --udid SIMULATOR_UDID -``` - -### **Sliders** - -```bash -# Set a slider by accessibility identifier to a verified 0-100 percentage -axe slider --id slider-value-slider --value 75 --udid SIMULATOR_UDID - -# Set a slider by label and narrow matching to slider elements -axe slider --label "Volume" --value 40 --element-type Slider --udid SIMULATOR_UDID -``` - -### **Gesture Presets** - -```bash -# Scrolling gestures -axe gesture scroll-up --udid SIMULATOR_UDID -axe gesture scroll-down --udid SIMULATOR_UDID -axe gesture scroll-left --udid SIMULATOR_UDID -axe gesture scroll-right --udid SIMULATOR_UDID - -# Navigation gestures -axe gesture swipe-from-left-edge --udid SIMULATOR_UDID -axe gesture swipe-from-right-edge --udid SIMULATOR_UDID -axe gesture swipe-from-top-edge --udid SIMULATOR_UDID -axe gesture swipe-from-bottom-edge --udid SIMULATOR_UDID - -# With custom screen dimensions -axe gesture scroll-up --screen-width 430 --screen-height 932 --udid SIMULATOR_UDID - -# With timing controls -axe gesture scroll-down --pre-delay 1.0 --post-delay 0.5 --udid SIMULATOR_UDID -``` - -### **Text Input** - -```bash -# Simple text input (use single quotes for special characters) -axe type 'Hello World!' --udid SIMULATOR_UDID - -# From stdin (best for automation) -echo "Complex text" | axe type --stdin --udid SIMULATOR_UDID - -# From file -axe type --file input.txt --udid SIMULATOR_UDID -``` - -### **Hardware Buttons** - -```bash -# Available buttons: home, lock, side-button, siri, apple-pay -axe button home --udid SIMULATOR_UDID -axe button lock --duration 2.0 --udid SIMULATOR_UDID -axe button siri --udid SIMULATOR_UDID -``` - -### **Keyboard Control** - -```bash -# Individual key presses (by HID keycode) -axe key 40 --udid SIMULATOR_UDID # Enter key -axe key 42 --duration 1.0 --udid SIMULATOR_UDID # Hold Backspace - -# Key sequences -axe key-sequence --keycodes 11,8,15,15,18 --udid SIMULATOR_UDID # Type "hello" - -# Key combos (modifier + key as atomic operation) -axe key-combo --modifiers 227 --key 4 --udid SIMULATOR_UDID # Cmd+A (Select All) -axe key-combo --modifiers 227 --key 6 --udid SIMULATOR_UDID # Cmd+C (Copy) -axe key-combo --modifiers 227,225 --key 4 --udid SIMULATOR_UDID # Cmd+Shift+A -``` - -### **Batch Chaining** - -```bash -# Execute multiple interaction steps in one command -axe batch --udid SIMULATOR_UDID \ - --step "tap --id SearchField" \ - --step "type 'hello world'" \ - --step "key 40" - -# Toggle a switch by its label -axe batch --udid SIMULATOR_UDID \ - --step "tap --label 'Weather Alerts'" - -# Add explicit timing between steps -axe batch --udid SIMULATOR_UDID \ - --step "tap -x 200 -y 400" \ - --step "sleep 0.5" \ - --step "tap -x 200 -y 500" - -# Read steps from a file (one step per line) -axe batch --udid SIMULATOR_UDID --file steps.txt -``` - -`batch` behavior in plain terms: -- Runs steps in order in a single invocation. -- Accepts exactly one-step source: `--step`, `--file`, or `--stdin`. -- Uses fail-fast by default (stops on first error). -- `--continue-on-error` switches to best-effort mode. -- `--ax-cache perBatch` is default for faster selector-based taps. -- `--type-submission chunked` is default for safer long text typing. - -> [!TIP] -> Keep verification out-of-band by running `axe describe-ui` or `axe screenshot` after batch execution. - -### **Video Streaming** - -```bash -# Stream MJPEG frames over stdout (default format) -axe stream-video --udid SIMULATOR_UDID --fps 10 --format mjpeg > stream.mjpeg - -# Pipe JPEG frames directly into ffmpeg -axe stream-video --udid SIMULATOR_UDID --fps 30 --format ffmpeg | \ - ffmpeg -f image2pipe -framerate 30 -i - -c:v libx264 -preset ultrafast output.mp4 - -# Stream raw JPEG frames with length prefixes for custom servers -axe stream-video --udid SIMULATOR_UDID --fps 12 --format raw | custom-stream-consumer - -# Legacy BGRA stream for backward compatibility -axe stream-video --udid SIMULATOR_UDID --format bgra | \ - ffmpeg -f rawvideo -pixel_format bgra -video_size 393x852 -i - output.mp4 - -# Record directly to MP4 using the dedicated recorder -axe record-video --udid SIMULATOR_UDID --fps 15 --output recording.mp4 -``` - -### **Video Recording** - -```bash -# Record the simulator to an MP4 file (QuickTime compatible) -axe record-video --udid SIMULATOR_UDID --fps 15 --output recording.mp4 - -# Let AXe pick a timestamped filename in the current directory -axe record-video --udid SIMULATOR_UDID --fps 20 - -# Tweak quality/scale to reduce file size -axe record-video --udid SIMULATOR_UDID --fps 10 --quality 60 --scale 0.5 --output low-bandwidth.mp4 -``` - -> [!TIP] -> Press `Ctrl+C` to stop recording. AXe finalises the MP4 before exiting and prints the file path to stdout. - -### **Screenshot** - -```bash -# Capture screenshot with auto-generated filename -axe screenshot --udid SIMULATOR_UDID - -# Save to specific file -axe screenshot --output ~/Desktop/my-screenshot.png --udid SIMULATOR_UDID - -# Save to directory (auto-generates timestamped filename) -axe screenshot --output ~/Desktop/ --udid SIMULATOR_UDID -``` - -> [!TIP] -> The screenshot path is printed to stdout for easy scripting. Progress messages go to stderr. - -### **Accessibility & Info** - -```bash -# Get accessibility information -axe describe-ui --udid SIMULATOR_UDID # Full screen -axe describe-ui --point 100,200 --udid SIMULATOR_UDID # Specific point - -# Set slider controls discovered via describe-ui -axe slider --id slider-value-slider --value 75 --udid SIMULATOR_UDID - -# List simulators +# Find a booted simulator UDID axe list-simulators -``` +export UDID= -## Architecture +# Inspect the current UI +axe describe-ui --udid "$UDID" -### Why AXe? - -AXe directly utilises the lower-level frameworks provided by [idb](https://github.com/facebook/idb), Facebook's open-source suite for automating iOS Simulators and Devices. - -While `idb` offers a powerful client/server architecture and a broad set of device automation features via an RPC protocol, **AXe takes a different approach**: - -- **Single Binary:** AXe is distributed as a single, standalone CLI tool—no server or client setup required. -- **Focused Scope:** AXe is purpose-built for UI automation, streamlining accessibility testing and automation tasks. -- **Simple Integration:** With no external dependencies or daemons, AXe can be easily scripted and embedded into other tools or systems running on the same host. -- **Complete HID Coverage:** Full feature parity with idb's HID functionality plus gesture presets and timing controls. -- **Intelligent Automation:** Built-in gesture presets and coordinate helpers for common use cases. - -This makes AXe a lightweight and easily adoptable alternative for projects that need direct, scriptable access to Simulator automation. - -## Benchmarking - -Compare batched vs non-batched latency on AxePlayground: - -```bash -# Build AXe first -swift build - -# Use a booted simulator UDID -scripts/benchmark_batch.sh --udid SIMULATOR_UDID --iterations 20 --rounds 5 +# Interact with the simulator +axe tap --label "Continue" --udid "$UDID" +axe type 'Hello world' --udid "$UDID" +axe screenshot --output ./screen.png --udid "$UDID" ``` -The benchmark runs equivalent two-tap workflows in both modes and prints mean/median speedup. - -## Batch Guide - -For a full behavior-first explanation of every batch argument with practical examples, see [`BATCHING.md`](BATCHING.md). - -## Gesture Presets Reference +## Documentation -| Preset | Description | Use Case | -|--------|-------------|----------| -| `scroll-up` | Scroll up in center of screen | Content navigation | -| `scroll-down` | Scroll down in center of screen | Content navigation | -| `scroll-left` | Scroll left in center of screen | Horizontal scrolling | -| `scroll-right` | Scroll right in center of screen | Horizontal scrolling | -| `swipe-from-left-edge` | Left edge to right edge | Back navigation | -| `swipe-from-right-edge` | Right edge to left edge | Forward navigation | -| `swipe-from-top-edge` | Top to bottom | Dismiss/close | -| `swipe-from-bottom-edge` | Bottom to top | Open/reveal | - -## Contributing - -Contributions are welcome! Please feel free to submit a Pull Request. +Full documentation is available at [axe-cli.com/docs](https://axe-cli.com/docs). ## Disclaimer -AXe is an independent open-source iOS simulator automation project and is not affiliated with, endorsed by, or associated with Deque Systems or its axe® accessibility products. +AXe is an independent open-source iOS Simulator automation project and is not affiliated with, endorsed by, or associated with Deque Systems or its axe® accessibility products. ## Licence -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. -Third-party licensing notices (including Meta's IDB MIT attribution) are in [THIRD_PARTY_LICENSES](THIRD_PARTY_LICENSES). +This project is licensed under the MIT License. See [LICENSE](LICENSE) for details. + +Third-party licensing notices, including Meta's IDB MIT attribution, are in [THIRD_PARTY_LICENSES](THIRD_PARTY_LICENSES). diff --git a/TEST_RUNNER.md b/TEST_RUNNER.md deleted file mode 100644 index 06d460d..0000000 --- a/TEST_RUNNER.md +++ /dev/null @@ -1,207 +0,0 @@ -# AXe Test Runner - -The `test-runner.sh` script provides a comprehensive, automated solution for building and testing the AXe iOS testing framework. - -## Features - -✅ **Automated Build Process** -- Builds the AXe CLI executable (`swift build`) -- Builds and installs the AxePlayground app on iPhone 16 simulator -- Handles simulator setup and management - -✅ **Comprehensive Testing** -- Runs individual test suites or entire test plan -- Sequential execution to prevent state conflicts -- Proper environment variable setup (`SIMULATOR_UDID`, `AXE_E2E=1`, optional `AXE_LANDSCAPE_E2E=1`) - -✅ **Flexible Options** -- Build-only mode for CI/CD pipelines -- Test-only mode for rapid iteration -- Clean build support -- Verbose output for debugging - -## Prerequisites - -- **Xcode**: Latest version with iOS simulator support -- **Swift**: For building the AXe CLI tool -- **iPhone 16 Simulator**: Must be available in Xcode Simulator - -## Quick Start - -```bash -# Make script executable (first time only) -chmod +x test-runner.sh - -# Run everything (build + test) -./test-runner.sh - -# Show help -./test-runner.sh --help -``` - -## Usage Examples - -### Complete Build and Test Cycle -```bash -# Build AXe executable, install playground app, run all tests -./test-runner.sh - -# Same as above but with clean build -./test-runner.sh --clean -``` - -### Test-Specific Workflows -```bash -# Build everything and run only swipe tests -./test-runner.sh SwipeTests - -# Run only tap tests (skip building) -./test-runner.sh --tests-only TapTests - -# Run all tests with verbose output -./test-runner.sh --verbose - -# Run landscape orientation precision tests when Simulator menu automation is available -AXE_LANDSCAPE_E2E=1 ./test-runner.sh -``` - -### Build-Only Workflows -```bash -# Just build everything (useful for CI) -./test-runner.sh --build-only - -# Clean build without running tests -./test-runner.sh --clean --build-only -``` - -## Available Test Suites - -| Test Suite | Description | -|------------|-------------| -| `SwipeTests` | Swipe gesture testing with coordinate accuracy and direction detection | -| `TapTests` | Tap coordinate accuracy and timing | -| `KeyTests` | Keyboard input simulation | -| `TouchTests` | Low-level touch event sequences | -| `TypeTests` | Text input simulation | -| `ButtonTests` | Hardware button press simulation | -| `GestureTests` | Predefined gesture patterns | -| `ListSimulatorsTests` | Simulator discovery and listing | -| `BatchTests` | Batch workflow execution | - -## Command Line Options - -| Option | Description | -|--------|-------------| -| `-h, --help` | Show help message | -| `-b, --build-only` | Only build (skip tests) | -| `-t, --tests-only` | Only run tests (skip building) | -| `-c, --clean` | Clean build before building | -| `-s, --sequential` | Run tests sequentially (default) | -| `-v, --verbose` | Verbose output | - -## Configuration - -The script is pre-configured for: -- **Simulator**: iPhone 16 (UDID: `B34FF305-5EA8-412B-943F-1D0371CA17FF`) -- **App**: AxePlayground (`com.cameroncooke.AxePlayground`) -- **Project**: `AxePlaygroundApp/AxePlayground.xcodeproj` - -To change these settings, edit the configuration section at the top of `test-runner.sh`. - -## Test Environment - -The script automatically sets up the following environment: -- `SIMULATOR_UDID`: Set to the configured iPhone 16 simulator -- `AXE_E2E=1`: Enables simulator E2E tests -- `AXE_LANDSCAPE_E2E=0`: Landscape orientation precision tests are skipped by default. Set `AXE_LANDSCAPE_E2E=1` to run them; the runner preflights Simulator menu automation and skips them if the menu is unavailable. -- Sequential test execution (prevents state conflicts) -- Clean app state between test suites - -## Troubleshooting - -### Common Issues - -**"Simulator not found"** -```bash -# List available simulators -xcrun simctl list devices | grep iPhone - -# Update SIMULATOR_UDID in script if needed -``` - -**"Build failed"** -```bash -# Try a clean build -./test-runner.sh --clean --build-only - -# Check Xcode and Swift installation -xcode-select --print-path -swift --version -``` - -**"Tests failed"** -```bash -# Run specific test suite with verbose output -./test-runner.sh --verbose SwipeTests - -# Check if app is properly installed -xcrun simctl list apps B34FF305-5EA8-412B-943F-1D0371CA17FF -``` - -### Performance Notes - -- **Sequential Testing**: Tests run sequentially by default to prevent state conflicts -- **Build Time**: Initial builds take longer; subsequent builds are incremental -- **Test Duration**: Full test suite takes ~5-10 minutes depending on system performance - -## Integration with CI/CD - -### GitHub Actions Example -```yaml -- name: Run AXe Tests - run: | - chmod +x test-runner.sh - ./test-runner.sh --verbose -``` - -### Build-Only for Docker -```bash -# In containerized environments -./test-runner.sh --build-only -``` - -## Advanced Usage - -### Custom Test Execution -```bash -# Run multiple specific test suites -./test-runner.sh SwipeTests && ./test-runner.sh TapTests - -# Test-driven development workflow -./test-runner.sh --tests-only SwipeTests --verbose -``` - -### Debugging Failed Tests -```bash -# Run with maximum verbosity -./test-runner.sh --verbose --tests-only FailingTestSuite - -# Check simulator state -xcrun simctl list devices -``` - -## Success Indicators - -The script provides clear visual feedback: -- 🎯 **Blue**: Section headers -- ✅ **Green**: Success messages -- ℹ️ **Blue**: Information -- ⚠️ **Yellow**: Warnings -- ❌ **Red**: Errors - -## Exit Codes - -- `0`: Success -- `1`: Build or test failure -- `1`: Missing prerequisites -- `1`: Invalid command line arguments \ No newline at end of file diff --git a/Tests/README.md b/Tests/README.md deleted file mode 100644 index 70c5a5c..0000000 --- a/Tests/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# AXe Tests - -Clean, simple test structure following KISS principles. - -## Structure - -Each AXe command has its own dedicated test file: - -- `ListSimulatorsTests.swift` - Tests for `list-simulators` command -- `DescribeUITests.swift` - Tests for `describe-ui` command -- `TapTests.swift` - Tests for `tap` command -- `SliderTests.swift` - Tests for `slider` command -- `SwipeTests.swift` - Tests for `swipe` command -- `DragTests.swift` - Tests for `drag` command -- `TypeTests.swift` - Tests for `type` command -- `KeyTests.swift` - Tests for `key` and `key-sequence` commands -- `TouchTests.swift` - Tests for `touch` command -- `ButtonTests.swift` - Tests for `button` command -- `GestureTests.swift` - Tests for `gesture` command -- `BatchTests.swift` - E2E coverage for `batch` command variants - -## Running Tests - -Use Swift's built-in testing system: - -```bash -# Run default tests (unit/non-E2E) -swift test - -# Run simulator E2E tests explicitly -AXE_E2E=1 SIMULATOR_UDID= swift test - -# Run specific test files -swift test --filter TapTests -swift test --filter SliderTests -swift test --filter SwipeTests -swift test --filter DragTests -swift test --filter TypeTests -swift test --filter KeyTests -swift test --filter TouchTests -swift test --filter ButtonTests -swift test --filter GestureTests -swift test --filter BatchTests -swift test --filter ListSimulatorsTests -swift test --filter DescribeUITests - -# Run with verbose output -swift test --verbose -``` - -## Test Requirements - -- Simulator E2E tests require `AXE_E2E=1` and a booted iOS simulator -- Set `SIMULATOR_UDID` with: `axe list-simulators` or `xcrun simctl list devices` -- `swift test` without `AXE_E2E=1` runs non-E2E tests only -- Some tests use the AxePlaygroundApp for validation -- Each test file is self-contained and executable - -## Test Philosophy - -- **KISS**: Keep It Simple, Stupid -- **One responsibility**: Each file tests exactly one command -- **No code generation**: All tests are explicit and readable -- **Self-contained**: Each test file includes its own utilities -- **Executable**: Each test file can be run independently - -## Individual Test Files - -Each test file can be run directly: - -```bash -swift test --filter TapTests -swift test --filter SliderTests -swift test --filter SwipeTests -swift test --filter DragTests -``` - -## Test Coverage - -All tests validate: -- ✅ Command execution (exit codes) -- ✅ Basic functionality -- ✅ Edge cases and error conditions -- ✅ Integration with AxePlaygroundApp where applicable -- ✅ Input validation and error handling \ No newline at end of file diff --git a/USAGE_EXAMPLES.md b/USAGE_EXAMPLES.md deleted file mode 100644 index e0afebd..0000000 --- a/USAGE_EXAMPLES.md +++ /dev/null @@ -1,509 +0,0 @@ -# AXe Usage Examples - -This document shows how to use AXe's comprehensive HID functionality including gesture presets, delay controls, and coordinate helpers. - -## Basic Setup - -First, get the UDID of your simulator: - -```bash -# List available simulators -axe list-simulators - -# Find a booted simulator and copy its UDID -# Example: B34FF305-5EA8-412B-943F-1D0371CA17FF -``` - -## Install the AXe Skill - -```bash -# Install into detected clients (.claude/.agents) -axe init - -# Non-interactive usage: pass an explicit target -axe init --client claude - -# Install into a custom destination directory -axe init --dest ~/.agents/skills - -# Print the skill without writing files -axe init --print - -# Remove installed skill directories -axe init --uninstall --client agents -``` - -## HID Commands Overview - -AXe provides comprehensive HID (Human Interface Device) functionality matching idb capabilities: - -### **1. Text Input** - -```bash -# Simple text input -axe type 'Hello World!' --udid SIMULATOR_UDID - -# Use single quotes for special characters -axe type 'Special chars: @#$%^&*()' --udid SIMULATOR_UDID - -# From stdin (best for automation) -echo "Complex text with any characters!" | axe type --stdin --udid SIMULATOR_UDID - -# From file -echo "Multi-line text content" > input.txt -axe type --file input.txt --udid SIMULATOR_UDID -``` - -### **2. Touch & Gestures** - -```bash -# Simple tap -axe tap -x 100 -y 200 --udid SIMULATOR_UDID - -# Tap by accessibility element (uses describe-ui accessibility tree) -axe tap --id "Safari" --udid SIMULATOR_UDID -axe tap --label "Safari" --udid SIMULATOR_UDID -axe tap --label "Weather Alerts" --udid SIMULATOR_UDID # Auto-uses physical touch for matched switches/toggles -axe tap --label "Submit" --tap-style simulator --udid SIMULATOR_UDID # Force FBSimulator tapAt -axe tap -x 320 -y 780 --tap-style physical --udid SIMULATOR_UDID # Force touch down/up for coordinates - -# Tap with timing controls -axe tap -x 100 -y 200 --pre-delay 1.0 --post-delay 0.5 --udid SIMULATOR_UDID - -# Swipe gestures -axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid SIMULATOR_UDID -axe swipe --start-x 50 --start-y 500 --end-x 350 --end-y 500 --duration 2.0 --delta 25 --udid SIMULATOR_UDID - -# Raw low-level drag using explicit touch move events -axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --udid SIMULATOR_UDID -axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --duration 0.4 --steps 40 --udid SIMULATOR_UDID - -# Swipe and drag with timing controls -axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --pre-delay 1.0 --post-delay 0.5 --udid SIMULATOR_UDID -axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --pre-delay 1.0 --post-delay 0.5 --udid SIMULATOR_UDID - -# Advanced touch control -axe touch -x 150 -y 250 --down --udid SIMULATOR_UDID # Touch down only -axe touch -x 150 -y 250 --up --udid SIMULATOR_UDID # Touch up only -axe touch -x 150 -y 250 --down --up --delay 1.0 --udid SIMULATOR_UDID # Touch with delay -``` - -### **3. Sliders** - -```bash -# Set a slider by accessibility identifier to a verified 0-100 percentage -axe slider --id slider-value-slider --value 75 --udid SIMULATOR_UDID - -# Set a slider by accessibility label and narrow matching to slider elements -axe slider --label "Volume" --value 40 --element-type Slider --udid SIMULATOR_UDID -``` - -The `slider` command uses the slider's accessibility frame and current AXValue to perform one calibrated low-level HID drag through the same composite touch-move path as `drag`, then verifies the new AXValue. Since iOS slider controls quantize values to their rendered track resolution, AXe verifies that the observed value is within tolerance rather than retrying correction gestures to chase unreachable decimals. If the observed AXValue remains outside tolerance after that drag, the command fails clearly. Use `describe-ui` first to find reliable `--id` or `--label` selectors. - -### **4. Gesture Presets** 🆕 - -```bash -# Scrolling gestures -axe gesture scroll-up --udid SIMULATOR_UDID -axe gesture scroll-down --udid SIMULATOR_UDID -axe gesture scroll-left --udid SIMULATOR_UDID -axe gesture scroll-right --udid SIMULATOR_UDID - -# Navigation gestures -axe gesture swipe-from-left-edge --udid SIMULATOR_UDID -axe gesture swipe-from-right-edge --udid SIMULATOR_UDID -axe gesture swipe-from-top-edge --udid SIMULATOR_UDID -axe gesture swipe-from-bottom-edge --udid SIMULATOR_UDID - -# Edge swipe gestures -axe gesture swipe-from-left-edge --udid SIMULATOR_UDID -axe gesture swipe-from-bottom-edge --udid SIMULATOR_UDID - -# Gesture presets with custom parameters -axe gesture scroll-up --duration 2.0 --delta 100 --udid SIMULATOR_UDID -axe gesture scroll-down --screen-width 430 --screen-height 932 --udid SIMULATOR_UDID - -# Gesture presets with timing controls -axe gesture scroll-down --pre-delay 1.0 --post-delay 0.5 --udid SIMULATOR_UDID -axe gesture swipe-from-left-edge --pre-delay 2.0 --duration 0.8 --post-delay 1.0 --udid SIMULATOR_UDID -``` - -### **5. Hardware Buttons** - -```bash -# Available buttons: home, lock, side-button, siri, apple-pay -axe button home --udid SIMULATOR_UDID -axe button lock --udid SIMULATOR_UDID -axe button lock --duration 3.0 --udid SIMULATOR_UDID # Long press - -# Side button (iPhone X+) -axe button side-button --udid SIMULATOR_UDID - -# Siri button -axe button siri --udid SIMULATOR_UDID - -# Apple Pay button -axe button apple-pay --udid SIMULATOR_UDID -``` - -### **6. Keyboard Control** - -```bash -# Individual key presses by keycode -axe key 40 --udid SIMULATOR_UDID # Enter key -axe key 44 --udid SIMULATOR_UDID # Space key -axe key 42 --duration 1.0 --udid SIMULATOR_UDID # Hold Backspace for 1 second - -# Key sequences -axe key-sequence --keycodes 11,8,15,15,18 --udid SIMULATOR_UDID # Type "hello" -axe key-sequence --keycodes 40,40,40 --delay 0.5 --udid SIMULATOR_UDID # Press Enter 3 times - -# Key combos (hold modifiers + press key as atomic operation) -axe key-combo --modifiers 227 --key 4 --udid SIMULATOR_UDID # Cmd+A (Select All) -axe key-combo --modifiers 227 --key 6 --udid SIMULATOR_UDID # Cmd+C (Copy) -axe key-combo --modifiers 227 --key 25 --udid SIMULATOR_UDID # Cmd+V (Paste) -axe key-combo --modifiers 227,225 --key 4 --udid SIMULATOR_UDID # Cmd+Shift+A -``` - -### **7. Batch Workflows** - -```bash -# Chain steps with one simulator/HID session -axe batch --udid SIMULATOR_UDID \ - --step "tap --id SearchField" \ - --step "type 'AXe batch input'" \ - --step "key 40" - -# Add explicit per-step delay with sleep -axe batch --udid SIMULATOR_UDID \ - --step "tap -x 180 -y 360" \ - --step "sleep 0.75" \ - --step "tap -x 240 -y 420" - -# Read one step per line from a file -cat > steps.txt <<'EOF' -tap --id UsernameField -type 'cam@example.com' -key 43 -type 'super-secret' -key 40 -EOF -axe batch --udid SIMULATOR_UDID --file steps.txt - -# For large type steps, chunked mode is default; optional composite mode: -axe batch --udid SIMULATOR_UDID --type-submission composite \ - --step "type 'some long string here'" - -# Toggle a setting switch by label. Automatic tap style uses physical touch for matched switches/toggles. -axe batch --udid SIMULATOR_UDID \ - --step "tap --label 'Weather Alerts'" - -# Override tap style for all tap steps in a batch, or for one tap step. -axe batch --udid SIMULATOR_UDID --tap-style physical \ - --step "tap -x 320 -y 780" -axe batch --udid SIMULATOR_UDID \ - --step "tap --label 'Submit' --tap-style simulator" -``` - -Selector taps activate switch/toggle controls at the control's activation point when the matched row or label contains one switch/toggle. The default `--tap-style automatic` uses physical touch down/up for those switch/toggle activations and FBSimulator `tapAt` for normal taps. - -Simple behavior rules: -- One-step source only: `--step`, `--file`, or `--stdin`. -- Steps run in order. -- Default mode is fail-fast (stop at first error). -- Add `--continue-on-error` for best-effort execution. -- Keep assertions separate with `describe-ui` or `screenshot`. - -For argument-by-argument behavior explanations, see [`BATCHING.md`](BATCHING.md). - -## Advanced Timing Control 🆕 - -### **Pre/Post Delays** - -```bash -# Add delays before and after actions -axe tap -x 100 -y 200 --pre-delay 2.0 --post-delay 1.0 --udid SIMULATOR_UDID -axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --pre-delay 1.5 --udid SIMULATOR_UDID -axe gesture scroll-up --pre-delay 1.0 --post-delay 2.0 --udid SIMULATOR_UDID -``` - -### **Complex Timing Sequences** - -```bash -# Sequence with precise timing -axe tap -x 100 -y 200 --post-delay 1.0 --udid SIMULATOR_UDID -axe gesture scroll-down --pre-delay 0.5 --post-delay 1.0 --udid SIMULATOR_UDID -axe button home --udid SIMULATOR_UDID -``` - -## Common Use Cases - -### **App Navigation with Presets** - -```bash -# Modern app navigation using presets -axe tap -x 100 -y 200 --udid SIMULATOR_UDID # Launch app -axe gesture scroll-up --post-delay 1.0 --udid SIMULATOR_UDID # Scroll content -axe gesture swipe-from-left-edge --udid SIMULATOR_UDID # Navigate back -axe button home --udid SIMULATOR_UDID # Go to home screen -``` - -### **Pull-to-Refresh Testing** - -```bash -# Test scroll down functionality -axe tap -x 200 -y 400 --udid SIMULATOR_UDID # Tap list area -axe gesture scroll-down --pre-delay 1.0 --udid SIMULATOR_UDID # Scroll down -axe gesture scroll-down --post-delay 2.0 --udid SIMULATOR_UDID # Scroll to see results -``` - -### **Multi-Screen Testing** - -```bash -# Different screen sizes with presets -# iPhone 15 -axe gesture scroll-up --screen-width 390 --screen-height 844 --udid SIMULATOR_UDID - -# iPhone 15 Plus -axe gesture scroll-up --screen-width 430 --screen-height 932 --udid SIMULATOR_UDID - -# iPad Pro -axe gesture scroll-up --screen-width 1024 --screen-height 1366 --udid SIMULATOR_UDID -``` - -### **Form Input with Timing** - -```bash -# Fill a form with proper timing -axe tap -x 150 -y 300 --post-delay 0.5 --udid SIMULATOR_UDID # Tap text field, wait for focus -axe type 'john.doe@example.com' --udid SIMULATOR_UDID # Enter email -axe key 43 --udid SIMULATOR_UDID # Tab to next field -axe type 'SecurePassword123!' --udid SIMULATOR_UDID # Enter password -axe key 40 --pre-delay 0.5 --udid SIMULATOR_UDID # Wait then press Enter/Submit -``` - -### **Gaming & Interactive Apps** - -```bash -# Game controls with precise timing -axe touch -x 100 -y 500 --down --udid SIMULATOR_UDID # Start drag -axe gesture swipe-from-top-edge --pre-delay 0.2 --duration 0.3 --udid SIMULATOR_UDID # Quick swipe action -axe touch -x 300 -y 200 --up --udid SIMULATOR_UDID # End drag - -# Edge swipes with timing -axe gesture swipe-from-left-edge --duration 0.5 --udid SIMULATOR_UDID # Quick back -axe gesture swipe-from-right-edge --pre-delay 1.0 --duration 0.3 --udid SIMULATOR_UDID # Forward after delay -``` - -### **Accessibility Testing with Presets** - -```bash -# Get accessibility info -axe describe-ui --point 100,200 --udid SIMULATOR_UDID - -# Set a discovered slider control -axe slider --id slider-value-slider --value 75 --udid SIMULATOR_UDID - -# Navigate with presets and keyboard -axe gesture scroll-down --post-delay 1.0 --udid SIMULATOR_UDID # Scroll to content -axe key-sequence --keycodes 43,43,40 --delay 1.0 --udid SIMULATOR_UDID # Tab navigation -``` - -### **Video Recording** - -```bash -# Record to an MP4 (QuickTime compatible) in the current directory -axe record-video --udid SIMULATOR_UDID --fps 15 - -# Choose a custom destination -# --output automatically records using h264 -axe record-video --udid SIMULATOR_UDID --fps 20 --output recordings/run.mp4 - -# Reduce bandwidth/size by lowering quality and scale -axe record-video --udid SIMULATOR_UDID --fps 10 --quality 55 --scale 0.6 --output recordings/light.mp4 - -# Simple automation-friendly script -UDID=$(axe list-simulators | awk '/Booted/{print $NF; exit}') -OUTPUT="recording_$(date +%Y%m%d_%H%M%S).mp4" - -axe record-video --udid "$UDID" --fps 25 --output "$OUTPUT" & -RECORD_PID=$! - -# ...run automation commands here... -axe tap -x 100 -y 200 --udid "$UDID" -axe gesture scroll-down --udid "$UDID" - -sleep 2 -kill -INT $RECORD_PID -wait $RECORD_PID -printf 'Saved recording to %s\n' "$OUTPUT" -``` - -> [!NOTE] -> `kill -INT` sends the same signal as pressing `Ctrl+C`, giving AXe time to finalise the MP4 before the process exits. - -## Shell Escaping Solutions - -### ❌ Problematic Examples: -```bash -# This fails due to history expansion -axe type "Hello World!" --udid SIMULATOR_UDID - -# This fails due to variable expansion -axe type "Hello $USER" --udid SIMULATOR_UDID -``` - -### ✅ Recommended Solutions: - -#### **1. Use Single Quotes** -```bash -axe type 'Hello World!' --udid SIMULATOR_UDID -axe type 'Special chars: @#$%^&*()' --udid SIMULATOR_UDID -``` - -#### **2. Use stdin (Best for Scripts)** -```bash -echo "Hello World! Any characters work here: @#$%^&*()" | axe type --stdin --udid SIMULATOR_UDID - -# In scripts -TEXT="Complex text with $variables" -echo "$TEXT" | axe type --stdin --udid SIMULATOR_UDID -``` - -#### **3. Use Files** -```bash -echo "Complex multi-line text" > input.txt -axe type --file input.txt --udid SIMULATOR_UDID -``` - -## Automation Examples - -### **Advanced Shell Script Automation** -```bash -#!/bin/bash - -# Get simulator UDID -UDID=$(axe list-simulators | grep "Booted" | head -1 | grep -o '[A-F0-9-]\{36\}') - -# Advanced app testing with presets and timing -axe tap -x 100 -y 200 --post-delay 1.0 --udid "$UDID" # Launch app, wait for load -axe gesture scroll-up --pre-delay 0.5 --post-delay 1.0 --udid "$UDID" # Scroll with timing -axe gesture swipe-from-left-edge --udid "$UDID" # Navigate back -axe button home --pre-delay 0.5 --udid "$UDID" # Return home with delay -``` - -### **Python Script Automation with Presets** -```python -import subprocess -import time - -def run_axe_command(cmd): - subprocess.run(cmd, shell=True, check=True) - -udid = "YOUR_SIMULATOR_UDID" - -# Advanced interaction sequence with presets -run_axe_command(f"axe tap -x 150 -y 300 --post-delay 1.0 --udid {udid}") -run_axe_command(f"echo 'Automated input' | axe type --stdin --udid {udid}") -run_axe_command(f"axe gesture scroll-down --pre-delay 0.5 --post-delay 1.0 --udid {udid}") -run_axe_command(f"axe gesture scroll-down --udid {udid}") -run_axe_command(f"axe button home --pre-delay 1.0 --udid {udid}") -``` - -### **Performance Testing with Presets** -```bash -# Rapid gesture testing -for gesture in scroll-up scroll-down scroll-left scroll-right; do - axe gesture $gesture --duration 0.3 --post-delay 0.2 --udid SIMULATOR_UDID -done - -# Stress testing with timing control -for i in {1..10}; do - axe gesture swipe-from-left-edge --duration 0.4 --post-delay 0.1 --udid SIMULATOR_UDID - axe gesture swipe-from-right-edge --duration 0.4 --post-delay 0.1 --udid SIMULATOR_UDID -done -``` - -### **Multi-Device Testing** -```bash -# Test same gesture on different screen sizes -devices=("390,844" "430,932" "393,852" "1024,1366") -for device in "${devices[@]}"; do - IFS=',' read -r width height <<< "$device" - echo "Testing on ${width}x${height}" - axe gesture scroll-up --screen-width $width --screen-height $height --udid SIMULATOR_UDID - axe gesture scroll-down --screen-width $width --screen-height $height --udid SIMULATOR_UDID -done -``` - -## Gesture Preset Reference 🆕 - -### **Available Presets** - -| Preset | Description | Default Duration | Default Delta | Use Case | -|--------|-------------|------------------|---------------|----------| -| `scroll-up` | Scroll up in center | 0.5s | 25px | Content scrolling | -| `scroll-down` | Scroll down in center | 0.5s | 25px | Content scrolling | -| `scroll-left` | Scroll left in center | 0.5s | 25px | Horizontal scrolling | -| `scroll-right` | Scroll right in center | 0.5s | 25px | Horizontal scrolling | -| `swipe-from-left-edge` | Left edge to right | 0.3s | 50px | Back navigation | -| `swipe-from-right-edge` | Right edge to left | 0.3s | 50px | Forward navigation | -| `swipe-from-top-edge` | Top to bottom | 0.3s | 50px | Dismiss/close | -| `swipe-from-bottom-edge` | Bottom to top | 0.3s | 50px | Open/reveal | - -### **Timing Control Reference** 🆕 - -| Parameter | Range | Description | Available On | -|-----------|-------|-------------|--------------| -| `--pre-delay` | 0-10 seconds | Delay before action | tap, swipe, drag, gesture | -| `--post-delay` | 0-10 seconds | Delay after action | tap, swipe, drag, gesture | -| `--duration` | 0-10 seconds | Action duration | swipe, drag, gesture, button, key | -| `--steps` | 1-1000 | Touch move event count | drag | -| `--delay` | 0-5 seconds | Between-key delay | key-sequence, touch | - -## Benchmarking Batch vs Non-Batch - -```bash -# Build AXe binary -swift build - -# Run benchmark on a booted simulator -scripts/benchmark_batch.sh --udid SIMULATOR_UDID --iterations 20 --rounds 5 -``` - -This benchmark compares equivalent two-tap workflows and reports per-iteration latency plus speedup. - -## Key Advantages - -1. **Complete HID Coverage**: All idb functionality now available in AXe -2. **Gesture Presets**: 11 common patterns with intelligent defaults -3. **Precise Timing Control**: Pre/post delays for complex automation -4. **Multi-Screen Support**: Automatic coordinate calculation for different devices -5. **No Shell Escaping**: Use `--stdin` or `--file` for complex text -6. **Automation-Friendly**: Perfect for CI/CD and testing scripts -7. **Flexible Input Methods**: Multiple ways to provide input and control timing -8. **Slider Verification**: Slider controls use selector-resolved low-level HID dragging with AXValue tolerance verification -9. **Comprehensive Validation**: Built-in parameter validation and error handling - -## Common Keycodes Reference - -``` -# Letters (lowercase) -a=4, b=5, c=6, d=7, e=8, f=9, g=10, h=11, i=12, j=13, k=14, l=15, m=16 -n=17, o=18, p=19, q=20, r=21, s=22, t=23, u=24, v=25, w=26, x=27, y=28, z=29 - -# Numbers -1=30, 2=31, 3=32, 4=33, 5=34, 6=35, 7=36, 8=37, 9=38, 0=39 - -# Special keys -Enter=40, Escape=41, Backspace=42, Tab=43, Space=44 -Minus=45, Equal=46, LeftBracket=47, RightBracket=48, Backslash=49 -Semicolon=51, Quote=52, Grave=53, Comma=54, Period=55, Slash=56 - -# Function keys -F1=58, F2=59, F3=60, F4=61, F5=62, F6=63, F7=64, F8=65, F9=66, F10=67 - -# Modifier keys -LeftCtrl=224, LeftShift=225, LeftAlt=226, LeftGUI=227 -RightCtrl=228, RightShift=229, RightAlt=230, RightGUI=231 -``` diff --git a/docs/investigations/tabview-switching-e2e-2026-05-10.md b/docs/investigations/tabview-switching-e2e-2026-05-10.md deleted file mode 100644 index 35c2a2c..0000000 --- a/docs/investigations/tabview-switching-e2e-2026-05-10.md +++ /dev/null @@ -1,598 +0,0 @@ -# Investigation: SwiftUI TabView E2E Switching - -## Summary -Selector-based switching for the standard SwiftUI `TabView` fixture is now unblocked by setting the outgoing CoreSimulator accessibility request's `AXPTranslatorRequest.clientType` to `2` in the host-side FBSimulatorControl translation delegate. - -The earlier clean path sent requests with `clientType=0`, which exposed the tab bar as one leaf `AXGroup` labeled `Tab Bar`. A focused client-type probe showed that forcing the serialized request `clientType` to `2` makes the simulator-side AccessibilityPlatformTranslation path expose real SwiftUI tab item elements: - -- `Home`, `type='RadioButton'`, with a real frame -- `Settings`, `type='RadioButton'`, with a real frame -- parent `Tab Bar`, `type='Group'`, now with the tab item children - -This is a real metadata fix, not coordinate segmentation and not synthetic tab creation. - -## Goal -Prove whether AXe can expose and tap individual standard SwiftUI `TabView` tab items through discovered accessibility metadata, without XCTest, WDA, a test-runner host app, or coordinate segmentation heuristics. - -Completion required one of these outcomes: - -1. `axe describe-ui` shows `Home` and `Settings` tab item elements with real labels, types, and frames, and selector-based `tap` switches tabs. -2. The report contains concrete reverse-engineered evidence for the blocker, including the private request paths and constants attempted. - -This investigation reached outcome 1. - -## Current Fixture -The `tab-view-test` playground route renders a standard SwiftUI `TabView` with: - -- `Home` tab -- `Settings` tab -- visible state label: `Current Tab: Home` / `Current Tab: Settings` -- content marker: `Home panel active` / `Settings panel active` - -The route is valid: launching `screen=tab-view-test` renders `Current Tab: Home`. - -## Final `describe-ui` Evidence -Captured after removing temporary instrumentation, applying the production `clientType=2` request patch, rebuilding the vendored frameworks, and launching the fixture on simulator `A2C64636-37E9-4B68-B872-E7F0A82A5670` / iPhone 17 Pro. - -Command shape: - -```sh -UDID=${SIMULATOR_UDID:-A2C64636-37E9-4B68-B872-E7F0A82A5670} -AXE_BIN="$(swift build --show-bin-path)/axe" -xcrun simctl terminate "$UDID" com.cameroncooke.AxePlayground || true -xcrun simctl launch "$UDID" com.cameroncooke.AxePlayground --launch-arg "screen=tab-view-test" -"$AXE_BIN" describe-ui --udid "$UDID" > /tmp/axe-tabview-final-client2.json -``` - -Observed accessibility tree summary from the focused client-type capture: - -```text -('Tab Bar', 'Group', frame={x:0,y:790.9999999999999,width:402,height:83}, children=1) -('Home', 'RadioButton', frame={x:111,y:794.9999999999999,width:94,height:54}, children=0) -('Settings', 'RadioButton', frame={x:197,y:794.9999999999999,width:94,height:54}, children=0) -Home tab element matches: 1 -Settings tab element matches: 1 -``` - -The important difference was not a serializer change. The same `AXPMacPlatformElement.accessibilityChildren` walk returned real children once the CoreSimulator request carried `AXPTranslatorRequest.clientType=2` into the simulator-side AccessibilityPlatformTranslation service. - -## Vendored Serializer Baseline -Primary file inspected and temporarily patched during the probe: - -- `idb_checkout/FBSimulatorControl/Commands/FBSimulatorAccessibilityCommands.m` - -Baseline behavior: - -- Flat serialization walks `element.accessibilityChildren`. -- Nested serialization recursively walks `element.accessibilityChildren`. -- For this SwiftUI `TabView`, the `Tab Bar` element has no serialized children. - -Relevant private API surface inspected: - -- `idb_checkout/PrivateHeaders/AccessibilityPlatformTranslation/AXPMacPlatformElement.h` -- `idb_checkout/PrivateHeaders/AccessibilityPlatformTranslation/AXPTranslationObject.h` -- `idb_checkout/PrivateHeaders/AccessibilityPlatformTranslation/AXPTranslator.h` -- `idb_checkout/PrivateHeaders/AccessibilityPlatformTranslation/AXPTranslatorRequest.h` -- `idb_checkout/PrivateHeaders/AccessibilityPlatformTranslation/AXPTranslatorResponse.h` -- `idb_checkout/PrivateHeaders/AccessibilityPlatformTranslation/AXPTranslator_iOS.h` -- `idb_checkout/PrivateHeaders/CoreSimulator/SimDevice.h` - -`AXPMacPlatformElement` supports generic attribute access through methods such as `accessibilityAttributeValue:`, `accessibilityMultipleAttributes:`, `accessibilityAttributeNames`, and `accessibilityParameterizedAttributeNames`. - -## Lower-Level AccessibilityPlatformTranslation Evidence -The repo-visible headers do not publish the request or attribute constants, so the investigation moved to dyld-cache disassembly, Objective-C runtime probes, and temporary FBSimulatorControl instrumentation. - -### Binary and symbol commands - -```sh -xcrun dyld_info -exports /System/Library/PrivateFrameworks/AccessibilityPlatformTranslation.framework/Versions/A/AccessibilityPlatformTranslation -xcrun dyld_info -disassemble /System/Library/PrivateFrameworks/AccessibilityPlatformTranslation.framework/Versions/A/AccessibilityPlatformTranslation > /tmp/axe_apt_disasm.txt -nm -m /Library/Developer/PrivateFrameworks/CoreSimulator.framework/CoreSimulator | grep -i 'sendAccessibilityRequest\|accessibilityRequest\|Accessibility' -strings /Library/Developer/PrivateFrameworks/CoreSimulator.framework/CoreSimulator | grep -i 'sendAccessibilityRequest\|accessibilityRequest\|Accessibility' -xcrun dyld_info -objc /Library/Developer/PrivateFrameworks/CoreSimulator.framework/CoreSimulator | grep -i 'sendAccessibilityRequest\|accessibilityRequest\|Accessibility' -``` - -Relevant symbols found: - -```text -_OBJC_CLASS_$_AXPMacPlatformElement -_OBJC_CLASS_$_AXPTranslationObject -_OBJC_CLASS_$_AXPTranslator -_OBJC_CLASS_$_AXPTranslatorRequest -_OBJC_CLASS_$_AXPTranslatorResponse --[AXPTranslator_iOS attributeFromRequest:] --[AXPTranslator_iOS _processChildrenAttributeRequest:error:] --[AXPTranslator_iOS _processRawElementDataRequest:error:] --[AXPTranslator_iOS processAttributeRequest:] --[AXPMacPlatformElement _attributeTypeForMacAttribute:] --[SimDevice sendAccessibilityRequestAsync:completionQueue:completionHandler:] --[SimDevice accessibilityPlatformTranslationToken] -com.apple.CoreSimulator.accessibility -SimAccessibility_PayloadClassName -SimAccessibility_Payload -``` - -Disassembly findings: - -- `-[AXPTranslator_iOS attributeFromRequest:]` accepts attribute IDs through `0x81` / `129` before falling out to no direct attribute. -- `-[AXPTranslator_iOS processAttributeRequest:]` first checks special cases, then calls `attributeFromRequest:`, then the direct attribute path. -- `-[AXPTranslator_iOS _processChildrenAttributeRequest:error:]` is the concrete path for child requests. -- `-[AXPTranslator_iOS _processRawElementDataRequest:error:]` is the concrete path for raw element token requests. -- `AXPMacPlatformElement` creates translator requests with `requestWithTranslation:`, sets `requestType`, sets `attributeType`, and sends them via `AXPTranslator.sharedInstance sendTranslatorRequest:`. -- Normal attribute requests use `requestType=2`. -- Multiple attribute requests use `requestType=5` and the `_AXPParametersDictAttributesKey` parameter. -- Application object requests use `requestType=1`. -- Action requests use `requestType=7`. -- CoreSimulator owns `-[SimDevice sendAccessibilityRequestAsync:completionQueue:completionHandler:]`, which is the host-to-simulator request transport used behind this translation layer. - -### Runtime constant derivation - -A local Objective-C runtime probe called `AXPMacPlatformElement.applicationElement` and `_attributeTypeForMacAttribute:`. - -Important Mac attribute mappings: - -| Mac attribute string | AXP attribute ID | -| --- | ---: | -| `AXChildren` | `8` / `0x08` | -| `AXChildrenInNavigationOrder` | `9` / `0x09` | -| `AXSelectedChildren` | `81` / `0x51` | -| `AXRole` | `45` | -| `AXRoleDescription` | `46` | -| `AXSubrole` | `51` | -| `AXDescription` / label | `33` | -| `AXValue` | `53` | -| `AXFrame` | `21` | -| `AXIdentifier` | `25` | -| `AXEnabled` | `27` | - -Strings such as `AXTabs`, `AXContents`, `AXRows`, `AXColumns`, `AXVisibleChildren`, and `AXRawElementData` mapped to `0`, so they are not accepted Mac-side attribute names for this bridge. - -A second runtime probe used the exported `_AXPAttributeToString` plus `_macAttributeTypeForAXPAttribute:`. `_attributeTypeForMacAttribute:@"AXChildren"` had to be called first to initialize the internal mapping. - -Important AXP enum names: - -| AXP attribute ID | AXP name | Mac bridge name | -| ---: | --- | --- | -| `8` | `AXPAttributeChildren` | `AXChildren` | -| `9` | `AXPAttributeChildrenInNavigationOrder` | `AXChildrenInNavigationOrder` | -| `18` | `AXPAttributeFirstContainedElement` | none returned | -| `34` | `AXPAttributeLastContainedElement` | none returned | -| `37` | `AXPAttributeNextContentSibling` | none returned | -| `44` | `AXPAttributePreviousContentSibling` | none returned | -| `58` | `AXPAttributeVisibleOpaqueElements` | none returned | -| `60` | `AXPAttributeRawElementData` | `AXDeviceElementToken` | -| `76` | `AXPAttributeLinkedUIElements` | `AXLinkedUIElements` | -| `79` | `AXPAttributeWindowSections` | none returned | -| `81` | `AXPAttributeSelectedChildren` | `AXSelectedChildren` | -| `85` | `AXPAttributeFirstElementForFocus` | none returned | -| `113` | `AXPAttributeElementsForSearchParameters` | none returned | -| `128` | `AXPAttributeMemoryAddress` | none returned | - -Key finding: raw element data is requested through AXP attribute `60` and appears on the Mac bridge as `AXDeviceElementToken`, not as a Mac attribute named `AXRawElementData`. - -## Temporary FBSimulatorControl Direct Request Probe -I temporarily instrumented `FBSimulatorAccessibilityCommands.m` only around the element labeled `Tab Bar`. The instrumentation was removed before the final rebuild. - -The probe collected: - -- `accessibilityAttributeNames` -- `accessibilityParameterizedAttributeNames` -- selected `accessibilityAttributeValue:` results -- direct `AXPTranslatorRequest` results with `requestType=2` -- `AXPTranslatorResponse.responseObject` -- `AXPTranslatorResponse.translationResponse` -- `AXPTranslatorResponse.translationsResponse` -- `AXPTranslatorResponse.errorCode` - -The direct request path used runtime class lookup to avoid hard-linking private classes: - -```objc -Class requestClass = objc_getClass("AXPTranslatorRequest"); -Class translatorClass = objc_getClass("AXPTranslator"); -id request = [requestClass requestWithTranslation:element.translation]; -[request setRequestType:2]; -[request setAttributeType:attributeID]; -id response = [[translatorClass sharedInstance] sendTranslatorRequest:request]; -``` - -Important build finding: - -- Referencing private classes directly, such as `AXPTranslatorRequest` or `AXPMacPlatformElement.class`, caused architecture-specific undefined symbol failures. -- `objc_getClass("AXPTranslatorRequest")`, `objc_getClass("AXPTranslator")`, and `objc_getClass("AXPMacPlatformElement")` avoided direct symbol references and allowed the temporary probe to build. - -### Tab Bar attributes exposed by the bridge - -`Tab Bar` returned these attribute names: - -```text -AXChildren -AXChildrenInNavigationOrder -AXCustomContent -AXDescription -AXEnabled -AXFocused -AXHelp -AXLanguage -AXParent -AXPosition -AXRole -AXRoleDescription -AXSelected -AXLinkedUIElements -AXSize -AXSubrole -AXTopLevelUIElement -AXUserInputLabels -AXValue -AXWindow -AXIdentifier -``` - -`accessibilityParameterizedAttributeNames` returned an empty array. - -Important Mac attribute values: - -| Attribute | Result | -| --- | --- | -| `AXChildren` | empty array | -| `AXChildrenInNavigationOrder` | empty array | -| `AXSelectedChildren` | nil | -| `_AXVisibleOpaqueElements` | nil | -| `AXDeviceElementToken` | dictionary with `TokenType` and `ElementData` | -| `AXTraits` | `8589934592` | -| `AXSections` | nil | -| `AXLinkedUIElements` | nil | -| `AXFirstContainedElement` | nil | -| `AXLastContainedElement` | nil | -| `AXNextContentSibling` | nil | -| `AXPreviousContentSibling` | nil | -| `AXUIElementsForSearchPredicate` | nil | -| `_AXFirstFocusedElement` | nil | - -### Direct AXP request results for `Tab Bar` - -All requests used `AXPTranslatorRequest.requestType=2` against the `Tab Bar` translation object. - -| AXP attr | Name | Result | Error code | Translation objects | -| ---: | --- | --- | ---: | --- | -| `8` | children | `NSArray[0]` | `0` | `NSArray[0]` | -| `9` | children in navigation order | `NSArray[0]` | `0` | empty/nil | -| `18` | first contained element | nil | `18446744073709526411` | nil | -| `34` | last contained element | nil | `18446744073709526411` | nil | -| `37` | next content sibling | nil | `18446744073709526411` | nil | -| `44` | previous content sibling | nil | `18446744073709526411` | nil | -| `58` | visible opaque elements | nil | `18446744073709526411` | nil | -| `60` | raw element data | token dictionary | `0` | nil | -| `76` | linked UI elements | nil | `18446744073709526411` | nil | -| `79` | window sections | nil | `0` | nil | -| `81` | selected children | nil | `0` | nil | -| `85` | first element for focus | nil | `18446744073709526411` | nil | -| `113` | elements for search parameters | nil | `0` | nil | -| `128` | memory address | string-like pointer | `0` | nil | - -Raw element data returned only the simulator element token payload: - -```text -{ - TokenType = AXElementTokenSimulator; - ElementData = {length = 20, bytes = 0xe8a6000080f98905010000000900000000000000}; -} -``` - -That payload is useful for a deeper protocol investigation, but it does not contain tab labels, frames, types, or child translation objects at this layer. - -## Raw Token / CoreSimulator XPC Follow-up -Follow-up reverse engineering on Xcode 26.4 (`17E192`) and the iOS 26.4 simulator runtime did not find a bypass that exposes real `Home` / `Settings` tab item elements. - -### Raw `AXElementTokenSimulator` payload - -Inspected binary: - -```text -/Library/Developer/CoreSimulator/Volumes/iOS_23E244/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS 26.4.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AccessibilityPlatformTranslation.framework/AccessibilityPlatformTranslation -``` - -Relevant symbols: - -```text --[AXPTranslator_iOS _processRawElementDataRequest:error:] --[AXPTranslator_iOS translationObjectFromData:] --[AXPTranslator_iOS remoteTranslationDataWithTranslation:pid:] -__AXUIElementCreateData -__AXUIElementCreateWithData -__AXUIElementIDForElement -__AXUIElementCreateWithPIDAndID -``` - -Disassembly evidence: - -- `_processRawElementDataRequest:error:` calls `axElement`, then `__AXUIElementCreateData`, then returns a two-key dictionary: `TokenType = AXElementTokenSimulator` and `ElementData = `. It does not request descendants or attach attribute data. -- `translationObjectFromData:` is the inverse: it calls `__AXUIElementCreateWithData`, then `translationObjectFromPlatformElement:`. This can rehydrate an `AXUIElement` token into the normal translation path, but it does not introduce a new child enumeration source. -- `remoteTranslationDataWithTranslation:pid:` only rewrites a remote element PID when `__AXUIElementIDForElement` returns the remote-element sentinel shape, then serializes the element again with `__AXUIElementCreateData`. That is PID remapping, not tab-item discovery. - -The observed 20-byte `ElementData` decodes as little-endian chunks: - -```text -bytes: e8 a6 00 00 80 f9 89 05 01 00 00 00 09 00 00 00 00 00 00 00 -u32@0 = 42728 / 0xa6e8 -u64@4 = 4387895680 / 0x10589f980 -u32@12 = 9 / 0x9 -u32@16 = 0 / 0x0 -``` - -This looks like an opaque AXRuntime element token containing process/element identity fields. It has no room for, and does not contain, tab labels, frames, roles, or child references. The only supported use visible in `AccessibilityPlatformTranslation` is rehydrating the same `AXUIElement` with `__AXUIElementCreateWithData`. - -### CoreSimulator accessibility XPC path - -Inspected binaries: - -```text -/Library/Developer/PrivateFrameworks/CoreSimulator.framework/Versions/A/CoreSimulator -/Library/Developer/PrivateFrameworks/CoreSimulator.framework/Versions/A/Resources/Platforms/iphoneos/usr/libexec/CoreSimulatorBridge -``` - -Relevant host-side methods: - -```text --[SimDevice sendAccessibilityRequestAsync:completionQueue:completionHandler:] -+[SimDevice _xpcMessageWithAXRequest:] -+[SimDevice _axResponseFromXPCMessage:] --[SimDevice accessibilityPlatformTranslationToken] --[SimDevice accessibilityConnection] -``` - -Relevant simulator-side methods: - -```text --[CSBAccessibilityBridgeManager start] -+[CSBAccessibilityBridgeManager _responseMessageFromRequestMessage:] -+[CSBAccessibilityBridgeManager _axRequestFromXPCMessage:] -+[CSBAccessibilityBridgeManager _xpcMessageWithAXResponse:requestMessage:] -``` - -Transport shape: - -1. Host looks up Mach service `com.apple.CoreSimulator.accessibility`. -2. Host builds an XPC dictionary with `SimAccessibility_PayloadClassName = NSStringFromClass(AXPTranslatorRequest)`. -3. Host archives the `AXPTranslatorRequest` with secure coding and stores it as data under nested key `translatorRequest` inside `SimAccessibility_Payload`. -4. `CoreSimulatorBridge` accepts only that request class shape, unarchives the `AXPTranslatorRequest`, and calls `[[AXPTranslator sharedInstance] processTranslatorRequest:request]` inside the simulator. -5. The bridge archives the resulting `AXPTranslatorResponse` under nested key `translatorResponse` and sends it back with `SimAccessibility_PayloadClassName = NSStringFromClass(AXPTranslatorResponse)`. -6. Host accepts only `AXPTranslatorResponse` from `_axResponseFromXPCMessage:`. - -That means the CoreSimulator XPC layer is not a richer accessibility API. It is a narrow secure-coding carrier for the same `AXPTranslatorRequest` / `AXPTranslatorResponse` path already exercised by the direct request probe. Sending a different payload class would fail the class-name/unarchive checks unless `CoreSimulatorBridge` itself were changed, which is outside a maintainable AXe patch. - -### Request surface still available below this point - -`AXPTranslator processTranslatorRequest:` dispatches request types `1...11`; cache-aware request handling only covers request types `2`, `3`, `4`, `5`, `9`, and `10` (`shouldCheckTreeDumpCacheForRequestType:` mask `0x63c`). The normal request types used by AXe are already covered: - -- `requestType=2`: single attribute request -- `requestType=5`: multiple attribute request -- `requestType=1`: application object request -- `requestType=7`: action request - -The remaining interesting private surface is the AX tree-dump / cached-tree path: - -```text --[AXPTranslator_iOS setRequestResolvingBehavior:] --[AXPTranslator_iOS generateAXTreeDumpTypeOnBackgroundThread:completionHandler:] --[AXPTranslator_iOS _frontmostAppChildrenForXCTest] --[AXPTranslator_iOS axTreeDumpGenerateNextSetOfElementAttrsOnMainThread] --[AXPTranslator processPlatformAXTreeDump:] --[AXPTranslator checkTreeDumpCacheForRequest:] -``` - -This is not the same as decoding `AXElementTokenSimulator`. It appears to be an alternate in-simulator AX tree cache used by Apple clients such as Oneness / XCTest-style tree dumping. It was probed as the next focused metadata path. - -## AX Tree-Dump / Cache Follow-up - -A temporary FBSimulatorControl probe was added at the existing CoreSimulator translation hook, after `frontmostApplicationWithDisplayId:bridgeDelegateToken:` returned an `AXPTranslationObject` and after the bridge token was installed. The probe was removed before the final clean rebuild. - -Inspected binaries: - -```text -/System/Library/PrivateFrameworks/AccessibilityPlatformTranslation.framework/Versions/A/AccessibilityPlatformTranslation -/Library/Developer/CoreSimulator/Volumes/iOS_23E244/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS 26.4.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AccessibilityPlatformTranslation.framework/AccessibilityPlatformTranslation -``` - -Relevant symbols and entry points found by strings, `nm -m`, `xcrun dyld_info -objc`, and disassembly: - -```text --[AXPTranslator_iOS setRequestResolvingBehavior:] --[AXPTranslator_iOS generateAXTreeDumpTypeOnBackgroundThread:completionHandler:] --[AXPTranslator_iOS fetchNextSetOfElementAttrsOnBackgroundThreadWithEarlyTermination:] --[AXPTranslator_iOS _frontmostAppChildrenForXCTest] --[AXPTranslator_iOS axTreeDumpGenerateNextSetOfElementAttrsOnMainThread] --[AXPTranslator_iOS processPlatformAXTreeDump:] --[AXPTranslator processPlatformAXTreeDump:] --[AXPTranslator checkTreeDumpCacheForRequest:] --[AXPTranslator treeDumpResponseCacheForBridgeDelegateToken:] --[AXPTranslator processAXTreeElements:] --[AXPRemoteCacheManager initWithCachedTreeClientType:] --[AXPRemoteCacheManager start] --[AXPRemoteCacheManager _sendAXHierachyOnBackgroundQueue] --[AXPRemoteCacheManager axInitialTreeDumpGeneratedOnBackgroundThreadCallback:success:] -_AXRequestingClient -_AXOverrideRequestingClientType -AXPTreeDumpTypeInitialDump -AXPTreeDumpTypeAdditionalData -AXPTreeDumpTypeTreeDestroyed -``` - -Disassembly findings: - -- `AXPRemoteCacheManager.start` sets `[AXPTranslator sharedInstance] setRequestResolvingBehavior:2` and `setCachedTreeClientType:`. -- `AXPRemoteCacheManager._sendAXHierachyOnBackgroundQueue` calls `[AXPTranslator sharediOSInstance] generateAXTreeDumpTypeOnBackgroundThread:@"AXPTreeDumpTypeInitialDump" completionHandler:...]`. -- `generateAXTreeDumpTypeOnBackgroundThread:completionHandler:` expects to run on `axTreeDumpSharedBackgroundQueue` (`com.apple.accessibility.AXPRemoteCacheManager.axHierarchyGeneration`). -- The tree generator references `AXRequestingClient`; when that client is `2`, it can call `_frontmostAppChildrenForXCTest`. -- `_frontmostAppChildrenForXCTest` reads AX attribute `0x1389` from the frontmost app and converts each returned `AXUIElement` into an `AXPTranslationObject`. -- `AXPTranslatorResponse.treeDumpType` reads `resultData[@"treeDumpType"]`. -- `AXPTranslatorResponse.treeDumpResponse` reads `resultData[@"treeDump"]`. - -Runtime probe values from AXe/FBSimulatorControl, with the fixture launched to `screen=tab-view-test`: - -```text -translator class=AXPTranslator -responds setRequestResolvingBehavior=1 -responds generateAXTreeDump=1 -responds processPlatformAXTreeDump=1 -responds checkTreeDumpCache=1 -sharedIOS= -sharedIOS responds _frontmostAppChildrenForXCTest=1 -sharedIOS responds axTreeDumpSharedBackgroundQueue=1 -sharedIOS responds generateAXTreeDump=1 -``` - -Request/cache knobs attempted: - -```text --[AXPTranslator setRequestResolvingBehavior:] = 2 --[AXPTranslator setCachedTreeClientType:] = 2 --[AXPTranslator_iOS setRequestResolvingBehavior:] = 2 --[AXPTranslator_iOS setCachedTreeClientType:] = 2 -AXOverrideRequestingClientType(2): symbol available, AXRequestingClient before=0 after=0 -``` - -Direct request attempts through `-[SimDevice sendAccessibilityRequestAsync:completionQueue:completionHandler:]`: - -| Request | Client type | Response | Notes | -| --- | ---: | --- | --- | -| `requestType=11`, `attributeType=0` | `0` | nil | No `treeDump`, no matches | -| `requestType=11`, `attributeType=0` | `2` | nil | No `treeDump`, no matches | -| `requestType=11`, `attributeType=0` | `16` | timeout in the first probe | Dropped from later probes to avoid blocking | - -Private method outputs: - -```text -_frontmostAppChildrenForXCTest output=NSArray(count=0, first=) -Home matches: 0 -Settings matches: 0 -Tab Bar matches: 0 -``` - -Calling `generateAXTreeDumpTypeOnBackgroundThread:completionHandler:` on `axTreeDumpSharedBackgroundQueue` did complete once the temporary probe used the correct private completion signature, which is `(BOOL success, AXPTranslatorResponse *response)`. - -Observed completion: - -```text -generateAXTreeDump completion success=1 -treeDumpType=AXPTreeDumpTypeInitialDump -treeDump=NSArray(count=2) -``` - -Observed tree-dump payload: - -```text -AXPTranslatorResponse associatedRequestType=11 -resultData keys=(treeDump, treeDumpType) -treeDump[0] associatedRequestType=4, attribute=AXPAttributeUndefined, associatedTranslationObj=(null) -treeDump[1] associatedRequestType=2, attribute=AXPAttributeApplicationOrientation, associatedTranslationObj=(null) -treeDumpResponseCacheForBridgeDelegateToken(token) = nil -Home matches: 0 -Settings matches: 0 -Tab Bar matches: 0 -``` - -The first generator probe briefly crashed because the temporary completion block used `(AXPTranslatorResponse *, BOOL)`. The crash report confirmed APT invoked the block as `(BOOL, AXPTranslatorResponse *)`, causing the probe to retain pointer `0x1` as an object. After correcting the temporary probe, the generator completed successfully but still returned only the two response objects above. - -Conclusion: AXe can reach the private tree-dump entry point from the host-side FBSimulatorControl path, but this does not expose SwiftUI `TabView` tab item elements. The in-process tree generator returned no real `Home` / `Settings` labels, frames, roles, or translation objects, and the bridge-token cache stayed empty. No serializer/fetch patch was kept. - -## AXRuntime Client-Type Follow-up - -The final lower-level target was host-side AXRuntime client gating around `AXRequestingClient`, `AXOverrideRequestingClientType`, and CoreSimulator's serialized `AXPTranslatorRequest` path. - -### Host AXRuntime behavior - -A small host probe loaded AXRuntime from the dyld shared cache and called the available client APIs: - -```text -initial: AXRequestingClient=0, _AXRequestingClientForSelfMachMessage=7 -after AXOverrideRequestingClientType(2): AXRequestingClient=0, _AXRequestingClientForSelfMachMessage=2 -after _AXSetRequestingClient(2): AXRequestingClient=2, _AXRequestingClientForSelfMachMessage=2 -after _AXSetRequestingClient(0): AXRequestingClient=0, _AXRequestingClientForSelfMachMessage=2 -``` - -Disassembly matched those results: - -- `AXOverrideRequestingClientType` writes the self-Mach-message override global. -- `AXRequestingClient` reads a different requesting-client global. -- `_AXSetRequestingClient` writes the `AXRequestingClient` global. - -So `AXOverrideRequestingClientType(2)` leaving `AXRequestingClient` at `0` in the AXe host process is expected. It changes the client used for messages sent as this process, not the plain host getter. - -### Why host `_AXSetRequestingClient(2)` was not enough - -`AXPTranslatorRequest.requestWithTranslation:` sets `request.clientType` from AccessibilityPlatformTranslation's current-request client conversion, not from AXe's later CoreSimulator send call. A standalone host probe showed the request stayed at `clientType=0` even after both host calls: - -```text -AXOverrideRequestingClientType(2) -_AXSetRequestingClient(2) -AXPTranslatorRequest.requestWithTranslation:nil -> clientType=0 -``` - -That means mutating AXRuntime globals in the AXe host process does not reliably mutate the `clientType` serialized to CoreSimulator. - -### What actually gates simulator-side results - -`AXPTranslatorRequest` securely encodes `clientType`, and CoreSimulator carries that request across XPC to `CoreSimulatorBridge`. Simulator-side AccessibilityPlatformTranslation decodes the same request and uses `request.clientType` while processing attribute, multiple-attribute, and action requests. Disassembly showed simulator-side request processors reading `request.clientType` and calling `AXOverrideRequestingClientType(...)` before resolving attributes. - -A temporary FBSimulatorControl probe forced outgoing requests to `clientType=2` immediately before `-[SimDevice sendAccessibilityRequestAsync:completionQueue:completionHandler:]`. One focused capture then exposed real SwiftUI tab item metadata: - -```text -AXE_PROBE forcing requestType=... clientType=2 -Home exact matches: 1 -Settings exact matches: 1 -Home type: RadioButton -Settings type: RadioButton -``` - -After that proof, the temporary env gate and probe logging were removed. The kept patch sets `request.clientType = 2` unconditionally in the CoreSimulator accessibility translation delegate. - -## Root Cause -The selector blocker was not AXe's Swift target resolver and not a missing serializer fallback. It was the host-to-simulator request client type. - -With `AXPTranslatorRequest.clientType=0`, simulator-side AccessibilityPlatformTranslation resolves the standard SwiftUI `TabView` tab bar as a leaf `AXGroup`. With `clientType=2`, the same CoreSimulator bridge path resolves real tab item children with labels, roles, and frames. - -The practical cause is that AXe's vendored FBSimulatorControl path was forwarding whatever `AXPTranslatorRequest.requestWithTranslation:` produced in the host process. In this AXe host context, that value stayed at `0`; the XCTest-style client behavior only appeared after setting `request.clientType=2` on the outgoing request before CoreSimulator serialized it. - -## Decisions -- Set `AXPTranslatorRequest.clientType=2` in the FBSimulatorControl CoreSimulator accessibility translation delegate. -- Did not change `AccessibilityTargetResolver`; it can now consume real tab item metadata. -- Replaced the temporary coordinate TabView E2E with a selector-based `tap --label Settings --element-type RadioButton` test. -- Normalized scalar AX fields in the production and test decoders so numeric `AXValue` from the newly exposed tab radio buttons does not break selector resolution. -- Removed all temporary probe logging, environment gates, and AXRuntime/tree-dump instrumentation. -- Did not use XCTest, WDA, a test-runner host app, coordinate segmentation, or synthetic tab elements. - -## Checks and Commands Run -Passed after the production patch and probe removal: - -```sh -./scripts/build.sh frameworks -./scripts/build.sh install -./scripts/build.sh strip -./scripts/build.sh xcframeworks -./scripts/build.sh verify-xcframeworks -swift build -./test-runner.sh TapTests -AXE_E2E=1 SIMULATOR_UDID=A2C64636-37E9-4B68-B872-E7F0A82A5670 swift test --filter DescribeUITests -swift test -``` - -Focused `describe-ui` capture after removing probe instrumentation and using the kept `clientType=2` patch: - -```text -elements: 100 -Home tab element matches: 2 -Settings tab element matches: 2 -Home type: RadioButton, AXValue: 1 -Settings type: RadioButton, AXValue: 0 -``` - -Regression coverage added: - -- `TapTests.selectorTapSwitchesSwiftUITabViewTab` verifies that `describe-ui` exposes `Home` and `Settings` as `RadioButton` elements, then taps `Settings` with `--label Settings --element-type RadioButton` and waits for `Current Tab: Settings`. -- The production and test accessibility decoders now accept scalar AX string fields as strings, numbers, or booleans; this is required because the newly exposed tab radio buttons return numeric selected-state `AXValue` values. - -## Next Lower-Level Target -No lower-level host-side avenue remains for this specific TabView selector blocker. The client-gating path produced real metadata and the patch is intentionally small. - -Future work, if this regresses on another Xcode/iOS runtime, should start by verifying the AXP client mapping for `clientType=2` on that runtime and checking whether simulator-side AccessibilityPlatformTranslation still applies it before child/attribute resolution. Do not reintroduce coordinate segmentation unless the product explicitly chooses a best-effort fallback with clear UX tradeoffs.