diff --git a/AxePlaygroundApp/AxePlayground/Views/AccessibilityFixturesView.swift b/AxePlaygroundApp/AxePlayground/Views/AccessibilityFixturesView.swift index f9ad2a0..93cda18 100644 --- a/AxePlaygroundApp/AxePlayground/Views/AccessibilityFixturesView.swift +++ b/AxePlaygroundApp/AxePlayground/Views/AccessibilityFixturesView.swift @@ -1,9 +1,18 @@ +import Foundation import SwiftUI struct SliderValueTestView: View { @State private var sliderValue = 0.25 @State private var state = "Initial" + private var percentText: String { + String(format: "%.2f", locale: Locale(identifier: "en_US_POSIX"), sliderValue * 100.0) + } + + private var exactValueText: String { + String(format: "%.4f", locale: Locale(identifier: "en_US_POSIX"), sliderValue) + } + var body: some View { VStack(spacing: 24) { Text("Slider Value State: \(state)") @@ -20,6 +29,14 @@ struct SliderValueTestView: View { .accessibilityIdentifier("slider-position-value") .accessibilityValue(sliderValue.formatted(.number.precision(.fractionLength(2)))) + Text("Slider Percent State: \(percentText)") + .accessibilityIdentifier("slider-percent-state") + .accessibilityValue(percentText) + + Text("Slider Exact Value: \(exactValueText)") + .accessibilityIdentifier("slider-exact-value-state") + .accessibilityValue(exactValueText) + Slider(value: $sliderValue, in: 0...1) .accessibilityIdentifier("slider-value-slider") .accessibilityLabel("Slider Value Slider") diff --git a/CHANGELOG.md b/CHANGELOG.md index ba8efe6..6ee94af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Added `axe slider --id/--label --value 0...100` for selector-based slider setting with orientation-aware HID dragging and AXValue tolerance verification/failure reporting. +- Added `axe drag --start-x/--start-y --end-x/--end-y` for raw point-to-point low-level HID drag validation using explicit touch move events. + +### Changed + +- Changed `axe slider` to use the shared composite low-level HID drag path with AXValue tolerance verification instead of retrying with correction gestures. + ### Fixed - Fixed `describe-ui` and selector-based `tap --label` exposing and activating real SwiftUI `TabView` tab items, navigation search fields, toolbar segmented picker items, and generated navigation back buttons from the CoreSimulator accessibility bridge. Also fixed selector decoding when the accessibility tree contains numeric `AXValue` fields such as sliders. diff --git a/README.md b/README.md index d026c13..15d7216 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ AXe is a comprehensive CLI tool for interacting with iOS Simulators using Apple' - [Install AXe Skill](#install-axe-skill) - [Commands Overview](#commands-overview) - [**Touch \& Gestures**](#touch--gestures-1) + - [**Sliders**](#sliders) - [**Gesture Presets**](#gesture-presets) - [**Text Input**](#text-input) - [**Hardware Buttons**](#hardware-buttons-1) @@ -45,7 +46,9 @@ AXe provides complete iOS Simulator automation capabilities: ### Touch & Gestures - **Tap**: Precise touch events at specific coordinates with timing controls - **Swipe**: Multi-touch gestures with configurable duration and delta +- **Drag**: Raw point-to-point low-level HID drags with explicit touch move events - **Touch Control**: Low-level touch down/up events for advanced gesture control +- **Sliders**: Set slider controls to a verified 0-100 percentage tolerance by accessibility identifier or label - **Gesture Presets**: Common gesture patterns (scroll-up, scroll-down, scroll-left, scroll-right, edge swipes) - **Batch Chaining**: Execute ordered multi-step interaction workflows in one invocation @@ -144,8 +147,10 @@ axe tap --id "Safari" --udid $UDID axe tap --label "Safari" --udid $UDID axe tap --label "Weather Alerts" --udid $UDID # Auto-uses physical touch for matched switches/toggles axe tap -x 320 -y 780 --tap-style physical --udid $UDID # Force physical touch for coordinate taps +axe slider --id "volume-slider" --value 75 --udid $UDID axe type 'Hello World!' --udid $UDID axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid $UDID +axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --udid $UDID axe button home --udid $UDID # Screenshot @@ -199,11 +204,16 @@ axe tap -x 320 -y 780 --tap-style physical --udid SIMULATOR_UDID # Force touch axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid SIMULATOR_UDID axe swipe --start-x 50 --start-y 500 --end-x 350 --end-y 500 --duration 2.0 --delta 25 --udid SIMULATOR_UDID +# Raw low-level drag using explicit touch move events +axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --udid SIMULATOR_UDID +axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --duration 0.4 --steps 40 --udid SIMULATOR_UDID + # Orientation-aware coordinates # AXe automatically maps logical UI coordinates for rotated landscape and letterboxed landscape-only apps. # Use the coordinates from describe-ui directly; AXe detects the simulator orientation. axe tap -x 100 -y 200 --udid SIMULATOR_UDID axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid SIMULATOR_UDID +axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --udid SIMULATOR_UDID # Advanced touch control axe touch -x 150 -y 250 --down --udid SIMULATOR_UDID @@ -213,6 +223,18 @@ axe touch -x 150 -y 250 --down --up --udid SIMULATOR_UDID axe touch -x 150 -y 250 --down --up --delay 1.0 --udid SIMULATOR_UDID ``` +### **Sliders** + +```bash +# Set a slider by accessibility identifier to a verified 0-100 percentage +axe slider --id slider-value-slider --value 75 --udid SIMULATOR_UDID + +# Set a slider by label and narrow matching to slider elements +axe slider --label "Volume" --value 40 --element-type Slider --udid SIMULATOR_UDID +``` + +`slider` resolves the real accessibility slider element, performs one calibrated low-level HID drag from its current AXValue-derived position toward the requested percentage using the same composite touch-move path as `drag`, and re-reads AXValue to verify the result. Since iOS slider controls quantize values to their rendered track resolution, AXe verifies that the observed value is within tolerance rather than retrying correction gestures to chase unreachable decimals. If the observed AXValue remains outside tolerance after that drag, the command fails clearly. + ### **Gesture Presets** ```bash @@ -367,6 +389,9 @@ axe screenshot --output ~/Desktop/ --udid SIMULATOR_UDID axe describe-ui --udid SIMULATOR_UDID # Full screen axe describe-ui --point 100,200 --udid SIMULATOR_UDID # Specific point +# Set slider controls discovered via describe-ui +axe slider --id slider-value-slider --value 75 --udid SIMULATOR_UDID + # List simulators axe list-simulators ``` diff --git a/Skills/CLI/axe/SKILL.md b/Skills/CLI/axe/SKILL.md index 3cba505..58a3fa2 100644 --- a/Skills/CLI/axe/SKILL.md +++ b/Skills/CLI/axe/SKILL.md @@ -1,23 +1,26 @@ --- name: axe -description: Provides agent-ready AXe CLI usage guidance for iOS Simulator automation. Use when asked to "use AXe", "automate a simulator", "tap/swipe/type on simulator", "describe UI", "take a screenshot", "record video", "batch steps", or "interact with an iOS app". Covers all commands including touch, gestures, text input, keyboard, buttons, accessibility, screenshots, video, and batch workflows. +description: Provides agent-ready AXe CLI usage guidance for iOS Simulator automation. Use when asked to "use AXe", "automate a simulator", "tap/swipe/type on simulator", "set a slider", "describe UI", "take a screenshot", "record video", "batch steps", or "interact with an iOS app". Covers all commands including touch, gestures, sliders, text input, keyboard, buttons, accessibility, screenshots, video, and batch workflows. --- ## Step 1: Confirm runtime context 1. Identify simulator UDID target first (`axe list-simulators`). 2. Simulator-interaction AXe commands require `--udid `. Commands like `list-simulators` and `init` do not. -3. Run `axe describe-ui --udid ` to inspect the full current screen. Use `axe describe-ui --point --udid ` to inspect the element at a specific coordinate. Use the output to discover available `--id` and `--label` values for selector taps, and to confirm coordinates for coordinate-based taps. -4. Prefer selector taps (`tap --id` / `tap --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting (`--wait-timeout`) in batch flows. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. +3. Run `axe describe-ui --udid ` to inspect the full current screen. Use `axe describe-ui --point --udid ` to inspect the element at a specific coordinate. Use the output to discover available `--id` and `--label` values for selector taps and slider setting, and to confirm coordinates for coordinate-based taps. +4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. ## Step 2: Choose the right command -Available commands: `init`, `tap`, `swipe`, `gesture`, `touch`, `type`, `button`, `key`, `key-sequence`, `key-combo`, `batch`, `describe-ui`, `screenshot`, `record-video`, `stream-video`, `list-simulators`. Run `axe --help` or `axe --help` for full options. +Available commands: `init`, `tap`, `slider`, `swipe`, `drag`, `gesture`, `touch`, `type`, `button`, `key`, `key-sequence`, `key-combo`, `batch`, `describe-ui`, `screenshot`, `record-video`, `stream-video`, `list-simulators`. Run `axe --help` or `axe --help` for full options. Common examples: ```bash axe tap --id --udid axe tap --label --udid axe tap --label 'Weather Alerts' --udid +axe slider --id --value 75 --udid +axe slider --label --value 40 --element-type Slider --udid +axe drag --start-x --start-y --end-x --end-y --udid axe tap -x -y --tap-style physical --udid axe tap -x -y --udid axe type 'text' --udid @@ -28,14 +31,15 @@ axe screenshot --udid --output screenshot.png ## Step 3: Understand the execution model -HID commands (`tap`, `swipe`, `type`, `key`, etc.) are fire-and-forget — AXe confirms the event was dispatched to the simulator but cannot verify the app actually processed it. A tap may land before a view is interactive, or during a transition. This means: -- Always verify outcomes separately with `describe-ui` or `screenshot`. -- Use `--wait-timeout` in batch to wait for elements to appear, and `sleep` steps or `--pre-delay` / `--post-delay` to allow animations to settle. +Most HID commands (`tap`, `swipe`, `drag`, `type`, `key`, etc.) are fire-and-forget — AXe confirms the event was dispatched to the simulator but cannot verify the app actually processed it. A tap may land before a view is interactive, or during a transition. `slider` is the exception: it performs one selector-resolved low-level HID drag, re-reads the matched slider AXValue, and fails if the observed 0-100 value is outside tolerance. iOS slider controls quantize values to their rendered track resolution, so AXe does not retry correction gestures to chase unreachable decimals. This means: +- Always verify outcomes separately with `describe-ui` or `screenshot` when app behavior matters beyond the direct command result. +- Use `--wait-timeout` in batch to wait for tap elements to appear, and `sleep` steps or `--pre-delay` / `--post-delay` to allow animations to settle. ## Step 4: Apply timing and input best practices - Use `--pre-delay` / `--post-delay` on tap, swipe, and gesture commands for fixed delays around actions. - Use `--duration` to control how long a swipe, gesture, button press, or key press lasts. -- Coordinate-based `tap`, `swipe`, and `touch` accept coordinates from `describe-ui` directly; AXe detects rotated landscape simulator orientation and letterboxed landscape-only app layouts automatically. +- Coordinate-based `tap`, `swipe`, `drag`, and `touch` accept coordinates from `describe-ui` directly; AXe detects rotated landscape simulator orientation and letterboxed landscape-only app layouts automatically. +- Use `axe slider --id --value <0-100>` for sliders instead of approximating with raw swipe coordinates; it uses one calibrated low-level HID drag from the resolved slider frame/current AXValue, through the same composite touch-move path as `drag`, verifies the result within tolerance, and fails clearly if the observed AXValue remains outside tolerance. - For text with shell-sensitive characters, prefer `--stdin` or `--file` over inline quotes. - Use single quotes for inline text arguments to avoid shell expansion issues. @@ -48,6 +52,7 @@ HID commands (`tap`, `swipe`, `type`, `key`, etc.) are fire-and-forget — AXe c **Fall back to discrete commands** when: - A step's parameters depend on runtime inspection of a previous step's result (e.g. parsing `describe-ui` JSON to choose coordinates dynamically). +- Using `slider`; batch steps do not support slider verification. **Handling animations and transitions in batch:** - Use `--wait-timeout ` so selector taps (`--id` / `--label`) poll the accessibility tree until the element appears or the timeout expires. This is the primary mechanism for multi-screen flows. diff --git a/Skills/CLI/axe/references/cli-quick-reference.md b/Skills/CLI/axe/references/cli-quick-reference.md index eef7737..e3560a0 100644 --- a/Skills/CLI/axe/references/cli-quick-reference.md +++ b/Skills/CLI/axe/references/cli-quick-reference.md @@ -29,6 +29,16 @@ axe tap -x 320 -y 780 --tap-style physical --udid axe tap -x 100 -y 200 --pre-delay 1.0 --post-delay 0.5 --udid ``` +## Slider + +```bash +# Value is a percentage from 0 to 100 +axe slider --id "volume-slider" --value 75 --udid +axe slider --label "Volume" --value 40 --element-type Slider --udid +``` + +`slider` resolves the matched accessibility slider, uses its frame/current AXValue for one calibrated low-level HID drag through the same composite touch-move path as `drag`, and re-reads AXValue. Since iOS slider controls quantize values to their rendered track resolution, AXe verifies that the observed value is within tolerance rather than retrying correction gestures to chase unreachable decimals. If the observed value remains outside tolerance, the command fails clearly. + ## Swipe ```bash @@ -41,6 +51,15 @@ axe swipe --start-x 50 --start-y 500 --end-x 350 --end-y 500 --duration 2.0 --de axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --pre-delay 1.0 --post-delay 0.5 --udid ``` +## Drag (low-level) + +```bash +axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --udid +axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --duration 0.4 --steps 40 --udid +``` + +`drag` emits one composite low-level HID event: touch down at the start point, a sequence of explicit touch move events, then touch up at the end point. + ## Touch (low-level) ```bash @@ -215,13 +234,15 @@ axe stream-video --udid --fps 30 --format ffmpeg | \ | Parameter | Range | Description | Available on | |---|---|---|---| -| `--pre-delay` | 0–10s | Delay before action | tap, swipe, gesture | -| `--post-delay` | 0–10s | Delay after action | tap, swipe, gesture | -| `--duration` | 0–10s | Action duration | swipe, gesture, button, key | +| `--pre-delay` | 0–10s | Delay before action | tap, swipe, drag, gesture | +| `--post-delay` | 0–10s | Delay after action | tap, swipe, drag, gesture | +| `--duration` | 0–10s | Action duration | swipe, drag, gesture, button, key | +| `--steps` | 1–1000 | Touch move event count | drag | +| `--value` | 0–100 | Target slider percentage | slider | | `--delay` | 0–5s | Between-item delay | key-sequence, touch | ## Best practices -- Prefer `--id` / `--label` taps over coordinates for resilience. +- Prefer `--id` / `--label` selectors over coordinates for resilience; use `slider` for selector-resolved low-level HID slider dragging with AXValue tolerance verification instead of raw swipe coordinates, and use `drag` when you specifically need raw point-to-point HID drag behavior. - Selector taps activate a contained UIKit `UISwitch` or SwiftUI `Toggle` when the matched row or label contains exactly one switch/toggle. - Default `--tap-style automatic` uses physical touch for matched switches/toggles and simulator `tapAt` for normal taps; use `--tap-style physical|simulator` to override. - Use single quotes for inline text to avoid shell expansion. diff --git a/Sources/AXe/Commands/Drag.swift b/Sources/AXe/Commands/Drag.swift new file mode 100644 index 0000000..1dd7e1f --- /dev/null +++ b/Sources/AXe/Commands/Drag.swift @@ -0,0 +1,104 @@ +import ArgumentParser +import Foundation + +struct Drag: AsyncParsableCommand { + static let configuration = CommandConfiguration( + abstract: "Perform a low-level point-to-point drag using explicit touch move events." + ) + + private static let defaultDuration: TimeInterval = 0.6 + private static let defaultSteps = 60 + private static let maxSteps = 1_000 + private static let initialHold: TimeInterval = 0.05 + private static let finalHold: TimeInterval = 0.05 + + @Option(name: .customLong("start-x"), help: "The X coordinate of the starting point.") + var startX: Double + + @Option(name: .customLong("start-y"), help: "The Y coordinate of the starting point.") + var startY: Double + + @Option(name: .customLong("end-x"), help: "The X coordinate of the ending point.") + var endX: Double + + @Option(name: .customLong("end-y"), help: "The Y coordinate of the ending point.") + var endY: Double + + @Option(name: .customLong("duration"), help: "Duration of the drag movement in seconds.") + var duration: Double = Self.defaultDuration + + @Option(name: .customLong("steps"), help: "Number of touch move events to emit during the drag.") + var steps: Int = Self.defaultSteps + + @Option(name: .customLong("pre-delay"), help: "Delay before starting the drag in seconds.") + var preDelay: Double? + + @Option(name: .customLong("post-delay"), help: "Delay after completing the drag in seconds.") + var postDelay: Double? + + @Option(name: .customLong("udid"), help: "The UDID of the simulator.") + var simulatorUDID: String + + func validate() throws { + guard startX >= 0, startY >= 0, endX >= 0, endY >= 0 else { + throw ValidationError("Coordinates must be non-negative values.") + } + guard startX != endX || startY != endY else { + throw ValidationError("Start and end points must be different.") + } + guard duration > 0 else { + throw ValidationError("Duration must be greater than 0.") + } + guard (1...Self.maxSteps).contains(steps) else { + throw ValidationError("Steps must be between 1 and \(Self.maxSteps).") + } + if let preDelay { + guard preDelay >= 0 && preDelay <= 10.0 else { + throw ValidationError("Pre-delay must be between 0 and 10 seconds.") + } + } + if let postDelay { + guard postDelay >= 0 && postDelay <= 10.0 else { + throw ValidationError("Post-delay must be between 0 and 10 seconds.") + } + } + } + + func run() async throws { + let logger = AxeLogger() + try await setup(logger: logger) + try await performGlobalSetup(logger: logger) + + logger.info().log("Performing low-level drag from (\(startX), \(startY)) to (\(endX), \(endY))") + logger.info().log("Duration: \(duration)s, steps: \(steps)") + + if let preDelay, preDelay > 0 { + logger.info().log("Pre-delay: \(preDelay)s") + try await Task.sleep(for: .seconds(preDelay)) + } + + let physicalPoints = try await OrientationAwareCoordinates.translateBatch( + points: [(x: startX, y: startY), (x: endX, y: endY)], + for: simulatorUDID, + logger: logger + ) + + try await HIDInteractor.performCompositeDrag( + from: physicalPoints[0], + to: physicalPoints[1], + duration: duration, + steps: steps, + initialHold: Self.initialHold, + finalHold: Self.finalHold, + for: simulatorUDID, + logger: logger + ) + + if let postDelay, postDelay > 0 { + logger.info().log("Post-delay: \(postDelay)s") + try await Task.sleep(for: .seconds(postDelay)) + } + + logger.info().log("Low-level drag completed successfully") + } +} diff --git a/Sources/AXe/Commands/Slider.swift b/Sources/AXe/Commands/Slider.swift new file mode 100644 index 0000000..dac12de --- /dev/null +++ b/Sources/AXe/Commands/Slider.swift @@ -0,0 +1,345 @@ +import ArgumentParser +import Foundation + +struct Slider: AsyncParsableCommand { + static let configuration = CommandConfiguration( + abstract: "Set a slider to a deterministic value from 0 to 100 using accessibility selector targeting." + ) + + private static let directDragSteps = 120 + private static let directDragDuration: TimeInterval = 2.4 + private static let directDragInitialHold: TimeInterval = 0.05 + private static let directDragFinalHold: TimeInterval = 0.2 + private static let verificationTimeout: TimeInterval = 1.5 + private static let verificationPollInterval: TimeInterval = 0.1 + private static let verificationStabilityDelay: TimeInterval = 0.3 + private static let valueTolerance = 0.0007 + private static let alreadyAtTargetTolerance = valueTolerance + private static let lowRangeCoordinateOffset = 0.0268 + private static let highRangeCoordinateOffset = 0.0271 + + @Option(name: [.customLong("id")], help: "Set the slider matching AXUniqueId (accessibilityIdentifier).") + var elementID: String? + + @Option(name: [.customLong("label")], help: "Set the slider matching AXLabel (accessibilityLabel).") + var elementLabel: String? + + @Option(name: [.customLong("element-type")], help: "Filter matches to this accessibility type, usually Slider.") + var elementType: String? + + @Option(name: [.customLong("value")], help: "Target slider value as a percentage from 0 to 100.") + var value: Double + + @Option(name: .customLong("wait-timeout"), help: "Maximum seconds to poll for the slider before failing (0 = no waiting, default).") + var waitTimeout: Double = 0 + + @Option(name: .customLong("poll-interval"), help: "Seconds between accessibility tree polls when --wait-timeout is active (default: 0.25).") + var pollInterval: Double = 0.25 + + @Option(name: .customLong("udid"), help: "The UDID of the simulator.") + var simulatorUDID: String + + func validate() throws { + let selectorCount = [elementID != nil, elementLabel != nil].filter { $0 }.count + guard selectorCount == 1 else { + throw ValidationError("Use exactly one of --id or --label to target a slider.") + } + + if let elementID, elementID.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + throw ValidationError("--id must not be empty.") + } + if let elementLabel, elementLabel.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + throw ValidationError("--label must not be empty.") + } + + guard value.isFinite, (0...100).contains(value) else { + throw ValidationError("--value must be a finite number between 0 and 100.") + } + guard waitTimeout >= 0 else { + throw ValidationError("--wait-timeout must be non-negative.") + } + if waitTimeout > 0 { + guard pollInterval > 0 else { + throw ValidationError("--poll-interval must be greater than 0 when --wait-timeout is active.") + } + } + } + + func run() async throws { + let logger = AxeLogger() + try await setup(logger: logger) + try await performGlobalSetup(logger: logger) + + let query = try accessibilityQuery() + let targetNormalized = value / 100.0 + + let match = try await AccessibilityPoller.resolveElementWithPolling( + query: query, + simulatorUDID: simulatorUDID, + waitTimeout: waitTimeout, + pollInterval: pollInterval, + elementType: elementType, + logger: logger + ) + + let observedValue = try await setAndVerifySliderValue( + initialMatch: match, + query: query, + targetNormalized: targetNormalized, + logger: logger + ) + + logger.info().log("Slider set completed successfully") + print("✓ Slider set to \(formatPercent(value)) successfully (AXValue: \(observedValue))") + } + + private func accessibilityQuery() throws -> AccessibilityQuery { + if let elementID { + return .id(elementID) + } + if let elementLabel { + return .label(elementLabel) + } + throw CLIError(errorDescription: "Unexpected state: no slider selector.") + } + + private func makeDragPlan( + for element: AccessibilityElement, + applicationFrame: AccessibilityElement.Frame?, + targetNormalized: Double + ) throws -> SliderDragPlan { + guard element.isSliderLikeControl else { + let typeDescription = element.type ?? element.role ?? "unknown" + throw CLIError(errorDescription: "Matched element is not a slider (type: \(typeDescription)). Use --element-type Slider or a more specific --id/--label selector.") + } + guard let frame = element.frame else { + throw ElementResolutionError.invalidFrame(reason: "Matched slider has no frame.") + } + guard frame.width > 0, frame.height > 0 else { + throw ElementResolutionError.invalidFrame(reason: "Matched slider has an invalid frame size (\(frame.width)x\(frame.height)).") + } + + let currentNormalized = try parseNormalizedAXValue(element.normalizedValue) + let centerY = frame.y + (frame.height / 2.0) + let commandedNormalized = Self.commandedNormalizedValue( + currentNormalized: currentNormalized, + targetNormalized: targetNormalized + ) + let nominalStartX = frame.x + (frame.width * currentNormalized) + let startX = dragStartX( + frame: frame, + nominalStartX: nominalStartX, + currentNormalized: currentNormalized, + targetNormalized: targetNormalized + ) + let fingerOffsetFromNominalStart = startX - nominalStartX + let rawEndX = frame.x + (frame.width * commandedNormalized) + fingerOffsetFromNominalStart + let endX = Self.clampedDragEndX(rawEndX, applicationFrame: applicationFrame) + return SliderDragPlan( + logicalStart: (x: startX, y: centerY), + logicalEnd: (x: endX, y: centerY), + currentNormalized: currentNormalized, + targetNormalized: targetNormalized, + commandedNormalized: commandedNormalized + ) + } + + private func setAndVerifySliderValue( + initialMatch: AccessibilityMatch, + query: AccessibilityQuery, + targetNormalized: Double, + logger: AxeLogger + ) async throws -> String { + let dragPlan = try makeDragPlan( + for: initialMatch.element, + applicationFrame: initialMatch.applicationFrame, + targetNormalized: targetNormalized + ) + logger.info().log( + "Setting slider \(initialMatch.selectorDescription) from AXValue \(formatNormalized(dragPlan.currentNormalized)) toward \(formatNormalized(dragPlan.targetNormalized)) with low-level HID drag" + ) + + if abs(dragPlan.currentNormalized - targetNormalized) > Self.alreadyAtTargetTolerance { + try await performSliderDrag(dragPlan, logger: logger) + } + + let observedValue = try await pollObservedSliderValue( + query: query, + targetNormalized: targetNormalized, + logger: logger + ) + guard observedValue.isWithinTolerance else { + throw CLIError( + errorDescription: "Slider value did not reach requested value \(formatPercent(value)) after direct drag. Observed AXValue: \(observedValue.rawValue ?? "none")." + ) + } + return observedValue.rawValue ?? formatNormalized(observedValue.normalizedValue) + } + + private func performSliderDrag(_ dragPlan: SliderDragPlan, logger: AxeLogger) async throws { + let physicalPoints = try await OrientationAwareCoordinates.translateBatch( + points: [dragPlan.logicalStart, dragPlan.logicalEnd], + for: simulatorUDID, + logger: logger + ) + let physicalStart = physicalPoints[0] + let physicalEnd = physicalPoints[1] + + try await HIDInteractor.performCompositeDrag( + from: physicalStart, + to: physicalEnd, + duration: Self.directDragDuration, + steps: Self.directDragSteps, + initialHold: Self.directDragInitialHold, + finalHold: Self.directDragFinalHold, + for: simulatorUDID, + logger: logger + ) + } + + private func resolveSliderElement(query: AccessibilityQuery, logger: AxeLogger) async throws -> AccessibilityMatch { + let roots = try await AccessibilityFetcher.fetchAccessibilityElements(for: simulatorUDID, logger: logger) + let match = try AccessibilityTargetResolver.resolveElement( + roots: roots, + query: query, + elementType: elementType + ) + guard match.element.isSliderLikeControl else { + throw CLIError(errorDescription: "Matched element is no longer a slider.") + } + return match + } + + private func pollObservedSliderValue( + query: AccessibilityQuery, + targetNormalized: Double, + logger: AxeLogger + ) async throws -> SliderObservedValue { + let clock = ContinuousClock() + let deadline = clock.now + .seconds(Self.verificationTimeout) + var lastObservedValue: SliderObservedValue? + + repeat { + let match = try await resolveSliderElement(query: query, logger: logger) + let rawValue = match.element.normalizedValue + let normalizedValue = try parseNormalizedAXValue(rawValue) + let observedValue = SliderObservedValue( + match: match, + rawValue: rawValue, + normalizedValue: normalizedValue, + isWithinTolerance: abs(normalizedValue - targetNormalized) <= Self.valueTolerance + ) + if observedValue.isWithinTolerance { + try await Task.sleep(for: .seconds(Self.verificationStabilityDelay)) + if clock.now >= deadline { + return observedValue + } + let stableMatch = try await resolveSliderElement(query: query, logger: logger) + let stableRawValue = stableMatch.element.normalizedValue + let stableNormalizedValue = try parseNormalizedAXValue(stableRawValue) + let stableObservedValue = SliderObservedValue( + match: stableMatch, + rawValue: stableRawValue, + normalizedValue: stableNormalizedValue, + isWithinTolerance: abs(stableNormalizedValue - targetNormalized) <= Self.valueTolerance + ) + if stableObservedValue.isWithinTolerance { + return stableObservedValue + } + lastObservedValue = stableObservedValue + } else { + lastObservedValue = observedValue + } + + if clock.now < deadline { + try await Task.sleep(for: .seconds(Self.verificationPollInterval)) + } + } while clock.now < deadline + + if let lastObservedValue { + return lastObservedValue + } + throw CLIError(errorDescription: "Slider value could not be verified because AXValue was unavailable after dragging.") + } + + private func dragStartX( + frame: AccessibilityElement.Frame, + nominalStartX: Double, + currentNormalized: Double, + targetNormalized: Double + ) -> Double { + guard currentNormalized >= 1.0 - Self.valueTolerance, targetNormalized < currentNormalized else { + return nominalStartX + } + return nominalStartX - (frame.height / 2.0) + } + + static func commandedNormalizedValue(currentNormalized: Double, targetNormalized: Double) -> Double { + if abs(currentNormalized - targetNormalized) <= Self.alreadyAtTargetTolerance { + return currentNormalized + } + if targetNormalized < currentNormalized { + return targetNormalized - Self.lowRangeCoordinateOffset + } + return targetNormalized + Self.highRangeCoordinateOffset + } + + static func clampedDragEndX( + _ x: Double, + applicationFrame: AccessibilityElement.Frame? + ) -> Double { + guard let applicationFrame, applicationFrame.width > 0 else { + return x + } + return min(max(x, applicationFrame.x), applicationFrame.x + applicationFrame.width) + } + + private func parseNormalizedAXValue(_ rawValue: String?) throws -> Double { + guard let rawValue else { + throw CLIError(errorDescription: "Matched slider does not expose a numeric AXValue, so AXe cannot deterministically set it.") + } + + let trimmedValue = rawValue.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedValue.isEmpty else { + throw CLIError(errorDescription: "Matched slider does not expose a numeric AXValue, so AXe cannot deterministically set it.") + } + + let isPercent = trimmedValue.hasSuffix("%") + let numericText = trimmedValue.replacingOccurrences(of: "%", with: "") + guard let parsedValue = Double(numericText.trimmingCharacters(in: .whitespacesAndNewlines)), parsedValue.isFinite else { + throw CLIError(errorDescription: "Matched slider does not expose a numeric AXValue, so AXe cannot deterministically set it.") + } + + let normalizedValue = isPercent || parsedValue > 1.0 ? parsedValue / 100.0 : parsedValue + guard (0...1).contains(normalizedValue) else { + throw CLIError(errorDescription: "Matched slider AXValue is outside the supported 0...100 range: \(rawValue).") + } + return normalizedValue + } + + private func formatPercent(_ value: Double) -> String { + if value.rounded() == value { + return String(Int(value)) + } + return String(format: "%.2f", value) + } + + private func formatNormalized(_ value: Double) -> String { + String(format: "%.3f", value) + } +} + +private struct SliderDragPlan { + let logicalStart: (x: Double, y: Double) + let logicalEnd: (x: Double, y: Double) + let currentNormalized: Double + let targetNormalized: Double + let commandedNormalized: Double +} + +private struct SliderObservedValue { + let match: AccessibilityMatch + let rawValue: String? + let normalizedValue: Double + let isWithinTolerance: Bool +} diff --git a/Sources/AXe/Resources/skills/axe/SKILL.md b/Sources/AXe/Resources/skills/axe/SKILL.md index 0e690f8..e2aa89b 100644 --- a/Sources/AXe/Resources/skills/axe/SKILL.md +++ b/Sources/AXe/Resources/skills/axe/SKILL.md @@ -1,23 +1,26 @@ --- name: axe -description: Provides agent-ready AXe CLI usage guidance for iOS Simulator automation. Use when asked to "use AXe", "automate a simulator", "tap/swipe/type on simulator", "describe UI", "take a screenshot", "record video", "batch steps", or "interact with an iOS app". Covers all commands including touch, gestures, text input, keyboard, buttons, accessibility, screenshots, video, and batch workflows. +description: Provides agent-ready AXe CLI usage guidance for iOS Simulator automation. Use when asked to "use AXe", "automate a simulator", "tap/swipe/type on simulator", "set a slider", "describe UI", "take a screenshot", "record video", "batch steps", or "interact with an iOS app". Covers all commands including touch, gestures, sliders, text input, keyboard, buttons, accessibility, screenshots, video, and batch workflows. --- ## Step 1: Confirm runtime context 1. Identify simulator UDID target first (`axe list-simulators`). 2. Simulator-interaction AXe commands require `--udid `. Commands like `list-simulators` and `init` do not. -3. Run `axe describe-ui --udid ` to inspect the full current screen. Use `axe describe-ui --point --udid ` to inspect the element at a specific coordinate. Use the output to discover available `--id` and `--label` values for selector taps, and to confirm coordinates for coordinate-based taps. -4. Prefer selector taps (`tap --id` / `tap --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting (`--wait-timeout`) in batch flows. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. +3. Run `axe describe-ui --udid ` to inspect the full current screen. Use `axe describe-ui --point --udid ` to inspect the element at a specific coordinate. Use the output to discover available `--id` and `--label` values for selector taps and slider setting, and to confirm coordinates for coordinate-based taps. +4. Prefer selectors (`tap --id` / `tap --label`, `slider --id` / `slider --label`) over raw coordinates. Selectors are resilient to layout changes, work across device sizes, and support element waiting where documented. For UIKit `UISwitch` and SwiftUI `Toggle` rows, selector taps activate the contained switch/toggle when the match contains exactly one such control. Default tap style is `automatic`: switches/toggles use physical touch down/up, while normal taps use simulator `tapAt`. ## Step 2: Choose the right command -Available commands: `init`, `tap`, `swipe`, `gesture`, `touch`, `type`, `button`, `key`, `key-sequence`, `key-combo`, `batch`, `describe-ui`, `screenshot`, `record-video`, `stream-video`, `list-simulators`. Run `axe --help` or `axe --help` for full options. +Available commands: `init`, `tap`, `slider`, `swipe`, `drag`, `gesture`, `touch`, `type`, `button`, `key`, `key-sequence`, `key-combo`, `batch`, `describe-ui`, `screenshot`, `record-video`, `stream-video`, `list-simulators`. Run `axe --help` or `axe --help` for full options. Common examples: ```bash axe tap --id --udid axe tap --label --udid axe tap --label 'Weather Alerts' --udid +axe slider --id --value 75 --udid +axe slider --label --value 40 --element-type Slider --udid +axe drag --start-x --start-y --end-x --end-y --udid axe tap -x -y --tap-style physical --udid axe tap -x -y --udid axe type 'text' --udid @@ -28,14 +31,15 @@ axe screenshot --udid --output screenshot.png ## Step 3: Understand the execution model -HID commands (`tap`, `swipe`, `type`, `key`, etc.) are fire-and-forget — AXe confirms the event was dispatched to the simulator but cannot verify the app actually processed it. A tap may land before a view is interactive, or during a transition. This means: -- Always verify outcomes separately with `describe-ui` or `screenshot`. -- Use `--wait-timeout` in batch to wait for elements to appear, and `sleep` steps or `--pre-delay` / `--post-delay` to allow animations to settle. +Most HID commands (`tap`, `swipe`, `drag`, `type`, `key`, etc.) are fire-and-forget — AXe confirms the event was dispatched to the simulator but cannot verify the app actually processed it. A tap may land before a view is interactive, or during a transition. `slider` is the exception: it performs one selector-resolved low-level HID drag, re-reads the matched slider AXValue, and fails if the observed 0-100 value is outside tolerance. iOS slider controls quantize values to their rendered track resolution, so AXe does not retry correction gestures to chase unreachable decimals. This means: +- Always verify outcomes separately with `describe-ui` or `screenshot` when app behavior matters beyond the direct command result. +- Use `--wait-timeout` in batch to wait for tap elements to appear, and `sleep` steps or `--pre-delay` / `--post-delay` to allow animations to settle. ## Step 4: Apply timing and input best practices - Use `--pre-delay` / `--post-delay` on tap, swipe, and gesture commands for fixed delays around actions. - Use `--duration` to control how long a swipe, gesture, button press, or key press lasts. -- Coordinate-based `tap`, `swipe`, and `touch` accept coordinates from `describe-ui` directly; AXe detects rotated landscape simulator orientation and letterboxed landscape-only app layouts automatically. +- Coordinate-based `tap`, `swipe`, `drag`, and `touch` accept coordinates from `describe-ui` directly; AXe detects rotated landscape simulator orientation and letterboxed landscape-only app layouts automatically. +- Use `axe slider --id --value <0-100>` for sliders instead of approximating with raw swipe coordinates; it uses one calibrated low-level HID drag from the resolved slider frame/current AXValue, through the same composite touch-move path as `drag`, verifies the result within tolerance, and fails clearly if the observed AXValue remains outside tolerance. - For text with shell-sensitive characters, prefer `--stdin` or `--file` over inline quotes. - Use single quotes for inline text arguments to avoid shell expansion issues. @@ -48,6 +52,7 @@ HID commands (`tap`, `swipe`, `type`, `key`, etc.) are fire-and-forget — AXe c **Fall back to discrete commands** when: - A step's parameters depend on runtime inspection of a previous step's result (e.g. parsing `describe-ui` JSON to choose coordinates dynamically). +- Using `slider`; batch steps do not support slider verification. **Handling animations and transitions in batch:** - Use `--wait-timeout ` so selector taps (`--id` / `--label`) poll the accessibility tree until the element appears or the timeout expires. This is the primary mechanism for multi-screen flows. diff --git a/Sources/AXe/Utilities/AccessibilityElement.swift b/Sources/AXe/Utilities/AccessibilityElement.swift index e54bbf0..775cb41 100644 --- a/Sources/AXe/Utilities/AccessibilityElement.swift +++ b/Sources/AXe/Utilities/AccessibilityElement.swift @@ -11,6 +11,7 @@ struct AccessibilityElement: Decodable { "RadioButton", "SecureTextField", "SegmentedControl", + "Slider", "Switch", "Tab", "TabBarButton", @@ -107,7 +108,21 @@ struct AccessibilityElement: Decodable { } var isActionable: Bool { - isSwitchLikeControl || type.map(Self.actionableTypes.contains) == true + isSwitchLikeControl || isSliderLikeControl || type.map(Self.actionableTypes.contains) == true + } + + var isSliderLikeControl: Bool { + if type == "Slider" { + return true + } + if role == "AXSlider" || subrole == "AXSlider" { + return true + } + if let roleDescription = trimmed(roleDescription)?.lowercased(), + roleDescription.contains("slider") { + return true + } + return false } var isSwitchLikeControl: Bool { diff --git a/Sources/AXe/Utilities/AccessibilityPoller.swift b/Sources/AXe/Utilities/AccessibilityPoller.swift index 90a8ef5..ad7cc79 100644 --- a/Sources/AXe/Utilities/AccessibilityPoller.swift +++ b/Sources/AXe/Utilities/AccessibilityPoller.swift @@ -15,7 +15,28 @@ struct AccessibilityPoller { waitTimeout: waitTimeout, pollInterval: pollInterval, elementType: elementType, - logger: logger + logger: logger, + resolver: AccessibilityTargetResolver.resolveTap + ) { + try await AccessibilityFetcher.fetchAccessibilityElements(for: simulatorUDID, logger: logger) + } + } + + static func resolveElementWithPolling( + query: AccessibilityQuery, + simulatorUDID: String, + waitTimeout: TimeInterval, + pollInterval: TimeInterval, + elementType: String? = nil, + logger: AxeLogger + ) async throws -> AccessibilityMatch { + try await pollForResolution( + query: query, + waitTimeout: waitTimeout, + pollInterval: pollInterval, + elementType: elementType, + logger: logger, + resolver: AccessibilityTargetResolver.resolveElement ) { try await AccessibilityFetcher.fetchAccessibilityElements(for: simulatorUDID, logger: logger) } @@ -29,9 +50,29 @@ struct AccessibilityPoller { logger: AxeLogger, rootsFetcher: () async throws -> [AccessibilityElement] ) async throws -> TapResolution { + try await pollForResolution( + query: query, + waitTimeout: waitTimeout, + pollInterval: pollInterval, + elementType: elementType, + logger: logger, + resolver: AccessibilityTargetResolver.resolveTap, + rootsFetcher: rootsFetcher + ) + } + + private static func pollForResolution( + query: AccessibilityQuery, + waitTimeout: TimeInterval, + pollInterval: TimeInterval, + elementType: String?, + logger: AxeLogger, + resolver: ([AccessibilityElement], AccessibilityQuery, String?) throws -> T, + rootsFetcher: () async throws -> [AccessibilityElement] + ) async throws -> T { let roots = try await rootsFetcher() do { - return try AccessibilityTargetResolver.resolveTap(roots: roots, query: query, elementType: elementType) + return try resolver(roots, query, elementType) } catch let error as ElementResolutionError where error.isNotFound && waitTimeout > 0 { let clock = ContinuousClock() let deadline = clock.now + .seconds(waitTimeout) @@ -43,7 +84,7 @@ struct AccessibilityPoller { let freshRoots = try await rootsFetcher() do { - return try AccessibilityTargetResolver.resolveTap(roots: freshRoots, query: query, elementType: elementType) + return try resolver(freshRoots, query, elementType) } catch let retryError as ElementResolutionError where retryError.isNotFound { lastError = retryError continue diff --git a/Sources/AXe/Utilities/AccessibilityTargetResolver.swift b/Sources/AXe/Utilities/AccessibilityTargetResolver.swift index 3bd4a61..956caed 100644 --- a/Sources/AXe/Utilities/AccessibilityTargetResolver.swift +++ b/Sources/AXe/Utilities/AccessibilityTargetResolver.swift @@ -44,6 +44,12 @@ enum ElementResolutionError: LocalizedError { } } +struct AccessibilityMatch { + let element: AccessibilityElement + let selectorDescription: String + let applicationFrame: AccessibilityElement.Frame? +} + struct AccessibilityTargetResolver { static let describeUITip = "Make sure the app is on the expected screen, then run `axe describe-ui --udid ` and prefer --id when available." @@ -58,11 +64,11 @@ struct AccessibilityTargetResolver { try resolveTap(roots: roots, query: query, elementType: elementType).point } - static func resolveTap( + static func resolveElement( roots: [AccessibilityElement], query: AccessibilityQuery, elementType: String? = nil - ) throws -> TapResolution { + ) throws -> AccessibilityMatch { var allElements = roots.flatMap { $0.flattened() } if let elementType { @@ -90,10 +96,24 @@ struct AccessibilityTargetResolver { selectorDescription = "--value '\(rawValue)'" } + return AccessibilityMatch( + element: matchedElement, + selectorDescription: selectorDescription, + applicationFrame: applicationFrame(from: roots) + ) + } + + static func resolveTap( + roots: [AccessibilityElement], + query: AccessibilityQuery, + elementType: String? = nil + ) throws -> TapResolution { + let match = try resolveElement(roots: roots, query: query, elementType: elementType) + let activationElement = try selectActivationElement( - from: matchedElement, + from: match.element, roots: roots, - selectorDescription: selectorDescription, + selectorDescription: match.selectorDescription, allowSiblingRedirection: query.allowsSiblingRedirection ) @@ -123,6 +143,10 @@ struct AccessibilityTargetResolver { return (x: frame.x + (frame.width / 2.0), y: centerY) } + private static func applicationFrame(from roots: [AccessibilityElement]) -> AccessibilityElement.Frame? { + roots.first { $0.type == "Application" }?.frame ?? roots.first?.frame + } + private static func selectUniqueMatch( _ matches: [AccessibilityElement], kind: String, diff --git a/Sources/AXe/Utilities/HIDInteractor.swift b/Sources/AXe/Utilities/HIDInteractor.swift index c3bb1e4..4b83c6f 100644 --- a/Sources/AXe/Utilities/HIDInteractor.swift +++ b/Sources/AXe/Utilities/HIDInteractor.swift @@ -1,6 +1,7 @@ import Foundation import FBControlCore import FBSimulatorControl +import ObjectiveC // MARK: - HID Interactor @MainActor @@ -74,6 +75,94 @@ struct HIDInteractor { try await performHIDEvent(event, in: session, logger: logger) } + static func makeCompositeDragEvent( + from start: (x: Double, y: Double), + to end: (x: Double, y: Double), + duration: TimeInterval, + steps: Int, + initialHold: TimeInterval, + finalHold: TimeInterval + ) throws -> FBSimulatorHIDEvent { + guard duration >= 0 else { + throw CLIError(errorDescription: "Drag duration must be non-negative.") + } + guard steps > 0 else { + throw CLIError(errorDescription: "Drag steps must be greater than 0.") + } + guard initialHold >= 0, finalHold >= 0 else { + throw CLIError(errorDescription: "Drag hold durations must be non-negative.") + } + + let movePoints = try compositeDragMovePoints(from: start, to: end, steps: steps) + let stepDelay = duration / Double(steps) + var events: [FBSimulatorHIDEvent] = [ + .touchDownAt(x: start.x, y: start.y), + .delay(initialHold) + ] + + for point in movePoints { + events.append(.delay(stepDelay)) + events.append(try touchMoveEvent(x: point.x, y: point.y)) + } + + events.append(.delay(finalHold)) + events.append(.touchUpAt(x: end.x, y: end.y)) + + return FBSimulatorHIDEvent(events: events) + } + + static func compositeDragMovePoints( + from start: (x: Double, y: Double), + to end: (x: Double, y: Double), + steps: Int + ) throws -> [(x: Double, y: Double)] { + guard steps > 0 else { + throw CLIError(errorDescription: "Drag steps must be greater than 0.") + } + + return (1...steps).map { step in + let progress = Double(step) / Double(steps) + return ( + x: start.x + ((end.x - start.x) * progress), + y: start.y + ((end.y - start.y) * progress) + ) + } + } + + private static func touchMoveEvent(x: Double, y: Double) throws -> FBSimulatorHIDEvent { + typealias TouchMoveIMP = @convention(c) (AnyClass, Selector, Double, Double) -> FBSimulatorHIDEvent + + let selector = NSSelectorFromString("touchMoveAtX:y:") + guard let method = class_getClassMethod(FBSimulatorHIDEvent.self, selector) else { + throw CLIError(errorDescription: "FBSimulatorHIDEvent does not support touch move events.") + } + + let implementation = method_getImplementation(method) + let touchMove = unsafeBitCast(implementation, to: TouchMoveIMP.self) + return touchMove(FBSimulatorHIDEvent.self, selector, x, y) + } + + static func performCompositeDrag( + from start: (x: Double, y: Double), + to end: (x: Double, y: Double), + duration: TimeInterval, + steps: Int, + initialHold: TimeInterval, + finalHold: TimeInterval, + for simulatorUDID: String, + logger: AxeLogger + ) async throws { + let event = try makeCompositeDragEvent( + from: start, + to: end, + duration: duration, + steps: steps, + initialHold: initialHold, + finalHold: finalHold + ) + try await performHIDEvent(event, for: simulatorUDID, logger: logger) + } + static func performPhysicalTap( at point: (x: Double, y: Double), preDelay: Double?, diff --git a/Sources/AXe/main.swift b/Sources/AXe/main.swift index d07121a..0b41ab4 100644 --- a/Sources/AXe/main.swift +++ b/Sources/AXe/main.swift @@ -18,8 +18,10 @@ struct Axe: AsyncParsableCommand { ListSimulators.self, Init.self, Tap.self, + Slider.self, Type.self, Swipe.self, + Drag.self, Button.self, Key.self, KeySequence.self, diff --git a/Tests/AccessibilityTargetResolverTests.swift b/Tests/AccessibilityTargetResolverTests.swift index c8343e7..e79b98b 100644 --- a/Tests/AccessibilityTargetResolverTests.swift +++ b/Tests/AccessibilityTargetResolverTests.swift @@ -259,6 +259,31 @@ struct AccessibilityTargetResolverTests { #expect(!resolution.isSwitchLikeControl) } + @Test("Resolved element preserves slider frame and AXValue") + func resolvedElementPreservesSliderFrameAndAXValue() throws { + let roots = try decodeElements( + """ + [ + { + "type": "Slider", + "role": "AXSlider", + "frame": { "x": 40, "y": 200, "width": 300, "height": 40 }, + "AXLabel": "Volume", + "AXUniqueId": "volume-slider", + "AXValue": "0.25" + } + ] + """ + ) + + let match = try AccessibilityTargetResolver.resolveElement(roots: roots, query: .label("Volume"), elementType: "Slider") + + #expect(match.selectorDescription == "--label 'Volume'") + #expect(match.element.isSliderLikeControl) + #expect(match.element.frame?.x == 40) + #expect(match.element.normalizedValue == "0.25") + } + private func decodeElements(_ json: String) throws -> [AccessibilityElement] { let data = try #require(json.data(using: .utf8)) return try JSONDecoder().decode([AccessibilityElement].self, from: data) diff --git a/Tests/DragTests.swift b/Tests/DragTests.swift new file mode 100644 index 0000000..62b3210 --- /dev/null +++ b/Tests/DragTests.swift @@ -0,0 +1,119 @@ +import Testing +import Foundation +@testable import AXe + +@Suite("Drag Command Surface Tests") +struct DragCommandSurfaceTests { + @Test("Drag help includes coordinate and timing options") + func dragHelpIncludesCoordinateAndTimingOptions() async throws { + let result = try await TestHelpers.runAxeCommand("drag --help") + + #expect(result.output.contains("--start-x")) + #expect(result.output.contains("--start-y")) + #expect(result.output.contains("--end-x")) + #expect(result.output.contains("--end-y")) + #expect(result.output.contains("--duration")) + #expect(result.output.contains("--steps")) + } + + @Test("Invalid drag coordinates fail validation") + func invalidDragCoordinatesFailValidation() async throws { + let result = try await TestHelpers.runAxeCommandAllowFailure( + "drag --start-x 100 --start-y 100 --end-x 100 --end-y 100 --udid invalid" + ) + + #expect(result.exitCode != 0) + #expect(result.output.contains("Start and end points must be different.")) + } + + @Test("Too many drag steps fails validation") + func tooManyDragStepsFailsValidation() async throws { + let result = try await TestHelpers.runAxeCommandAllowFailure( + "drag --start-x 100 --start-y 100 --end-x 100 --end-y 200 --steps 1001 --udid invalid" + ) + + #expect(result.exitCode != 0) + #expect(result.output.contains("Steps must be between 1 and 1000.")) + } + + @Test("Composite drag plan includes move points between touch down and touch up") + @MainActor + func compositeDragPlanIncludesMovePoints() throws { + let movePoints = try HIDInteractor.compositeDragMovePoints( + from: (x: 100, y: 200), + to: (x: 300, y: 600), + steps: 4 + ) + + #expect(movePoints.count == 4) + #expect(movePoints.first?.x == 150) + #expect(movePoints.first?.y == 300) + #expect(movePoints.last?.x == 300) + #expect(movePoints.last?.y == 600) + #expect(movePoints.contains { point in + point.x > 100 && point.x < 300 && point.y > 200 && point.y < 600 + }) + } +} + +@Suite("Drag Command Tests", .serialized, .enabled(if: isE2EEnabled)) +struct DragTests { + @Test("Low-level drag records requested start and end points") + func lowLevelDragRecordsRequestedStartAndEndPoints() async throws { + try await TestHelpers.launchPlaygroundApp(to: "touch-control") + + _ = try await TestHelpers.waitForLabel(containing: "Touch Control Playground", timeout: 3) { + $0 == "Touch Control Playground" + } + + let start = (x: 250, y: 450) + let end = (x: 250, y: 650) + + try await TestHelpers.runAxeCommand( + "drag --start-x \(start.x) --start-y \(start.y) --end-x \(end.x) --end-y \(end.y) --duration 0.4 --steps 40", + simulatorUDID: defaultSimulatorUDID + ) + + try await waitForRecordedDrag(start: start, end: end, timeout: 3) + } + + private func waitForRecordedDrag( + start: (x: Int, y: Int), + end: (x: Int, y: Int), + timeout: TimeInterval + ) async throws { + let deadline = Date().addingTimeInterval(timeout) + var didSeeStart = false + var didSeeEnd = false + var lastTouchHistory: String? + + while Date() < deadline { + let uiState = try await TestHelpers.getUIState() + didSeeStart = hasTouchEvent(in: uiState, type: "down", near: start) + didSeeEnd = hasTouchEvent(in: uiState, type: "up", near: end) + lastTouchHistory = UIStateParser.findElement(in: uiState, withIdentifier: "touch-history")?.value + + if didSeeStart && didSeeEnd { + return + } + + try await Task.sleep(nanoseconds: 200_000_000) + } + + throw TestError.unexpectedState( + "Timed out waiting for drag start (\(start.x), \(start.y)) and end (\(end.x), \(end.y)). Saw start: \(didSeeStart), saw end: \(didSeeEnd), last touch history: \(lastTouchHistory ?? "none")" + ) + } + + private func hasTouchEvent(in uiState: UIElement, type: String, near expected: (x: Int, y: Int)) -> Bool { + UIStateParser.findElement(in: uiState) { element in + guard let value = element.value, + value.hasPrefix("\(type):"), + let point = CoordinateParser.parseNamedCoordinates(from: value) else { + return false + } + + return abs(point.x - expected.x) <= 1 && abs(point.y - expected.y) <= 1 + } != nil + } +} diff --git a/Tests/README.md b/Tests/README.md index 904e348..70c5a5c 100644 --- a/Tests/README.md +++ b/Tests/README.md @@ -9,7 +9,9 @@ Each AXe command has its own dedicated test file: - `ListSimulatorsTests.swift` - Tests for `list-simulators` command - `DescribeUITests.swift` - Tests for `describe-ui` command - `TapTests.swift` - Tests for `tap` command +- `SliderTests.swift` - Tests for `slider` command - `SwipeTests.swift` - Tests for `swipe` command +- `DragTests.swift` - Tests for `drag` command - `TypeTests.swift` - Tests for `type` command - `KeyTests.swift` - Tests for `key` and `key-sequence` commands - `TouchTests.swift` - Tests for `touch` command @@ -30,7 +32,9 @@ AXE_E2E=1 SIMULATOR_UDID= swift test # Run specific test files swift test --filter TapTests +swift test --filter SliderTests swift test --filter SwipeTests +swift test --filter DragTests swift test --filter TypeTests swift test --filter KeyTests swift test --filter TouchTests @@ -66,7 +70,9 @@ Each test file can be run directly: ```bash swift test --filter TapTests +swift test --filter SliderTests swift test --filter SwipeTests +swift test --filter DragTests ``` ## Test Coverage diff --git a/Tests/SliderTests.swift b/Tests/SliderTests.swift new file mode 100644 index 0000000..e76b38c --- /dev/null +++ b/Tests/SliderTests.swift @@ -0,0 +1,178 @@ +import Testing +import Foundation +@testable import AXe + +@Suite("Slider Command Surface Tests") +struct SliderCommandSurfaceTests { + @Test("Slider help includes selector and value options") + func sliderHelpIncludesSelectorAndValueOptions() async throws { + let result = try await TestHelpers.runAxeCommand("slider --help") + + #expect(result.output.contains("--id")) + #expect(result.output.contains("--label")) + #expect(result.output.contains("--value")) + #expect(result.output.contains("--element-type")) + } + + @Test("Invalid slider value fails validation") + func invalidSliderValueFailsValidation() async throws { + let result = try await TestHelpers.runAxeCommandAllowFailure("slider --id slider-value-slider --value 101 --udid invalid") + + #expect(result.exitCode != 0) + #expect(result.output.contains("--value must be a finite number between 0 and 100.")) + } + + @Test("Missing slider selector fails validation") + func missingSliderSelectorFailsValidation() async throws { + let result = try await TestHelpers.runAxeCommandAllowFailure("slider --value 75 --udid invalid") + + #expect(result.exitCode != 0) + #expect(result.output.contains("Use exactly one of --id or --label to target a slider.")) + } + + @Test("Slider drag endpoints stay within the application frame") + func sliderDragEndpointsStayWithinApplicationFrame() { + let applicationFrame = AccessibilityElement.Frame(x: 0, y: 0, width: 390, height: 844) + + #expect(Slider.clampedDragEndX(-12, applicationFrame: applicationFrame) == 0) + #expect(Slider.clampedDragEndX(402, applicationFrame: applicationFrame) == 390) + #expect(Slider.clampedDragEndX(120, applicationFrame: applicationFrame) == 120) + } + + @Test("Slider command skips overdrive when already within verification tolerance") + func sliderCommandSkipsOverdriveWhenAlreadyWithinVerificationTolerance() { + #expect(Slider.commandedNormalizedValue(currentNormalized: 0.3994, targetNormalized: 0.4) == 0.3994) + #expect(Slider.commandedNormalizedValue(currentNormalized: 0.4006, targetNormalized: 0.4) == 0.4006) + #expect(Slider.commandedNormalizedValue(currentNormalized: 0.398, targetNormalized: 0.4) > 0.4) + } +} + +@Suite("Slider Command Tests", .serialized, .enabled(if: isE2EEnabled)) +struct SliderTests { + private let observableValueTolerance = 0.0007 + + @Test("Slider command reaches observable AXValue tolerance with one command execution", arguments: [ + 0.0, + 0.1, + 1.50, + 40.0, + 75.0, + 78.25, + 100.0 + ]) + func sliderCommandReachesObservableAXValueToleranceWithOneCommand(requestedPercent: Double) async throws { + try await TestHelpers.launchPlaygroundApp(to: "slider-value-test") + + _ = try await TestHelpers.runAxeCommand( + "slider --id slider-value-slider --value \(formatCommandValue(requestedPercent)) --element-type Slider", + simulatorUDID: defaultSimulatorUDID + ) + + let state = try await waitForSliderState(requestedPercent: requestedPercent) + #expect(state.slider.type == "Slider") + } + + @Test("Slider command sets value by accessibility label") + func sliderCommandSetsValueByAccessibilityLabel() async throws { + try await TestHelpers.launchPlaygroundApp(to: "slider-value-test") + + try await TestHelpers.runAxeCommand( + "slider --label 'Slider Value Slider' --value 40 --element-type Slider", + simulatorUDID: defaultSimulatorUDID + ) + + let state = try await waitForSliderState(requestedPercent: 40) + #expect(state.slider.type == "Slider") + } + + @Test("Slider command reaches observable AXValue tolerance across sequential moves") + func sliderCommandReachesObservableAXValueToleranceAcrossSequentialMoves() async throws { + try await TestHelpers.launchPlaygroundApp(to: "slider-value-test") + + for requestedPercent in [0.0, 0.1, 1.50, 100.0, 78.25, 40.0, 75.0] { + try await TestHelpers.runAxeCommand( + "slider --id slider-value-slider --value \(formatCommandValue(requestedPercent)) --element-type Slider", + simulatorUDID: defaultSimulatorUDID + ) + + let state = try await waitForSliderState(requestedPercent: requestedPercent) + #expect(state.slider.type == "Slider") + } + } + + private func waitForSliderState(requestedPercent: Double) async throws -> SliderVerificationState { + let expectedNormalizedValue = requestedPercent / 100.0 + let deadline = Date().addingTimeInterval(3) + var lastPercentText: String? + var lastExactText: String? + var lastSliderValue: String? + + while Date() < deadline { + let uiState = try await TestHelpers.getUIState() + let percentElement = UIStateParser.findElement(in: uiState, withIdentifier: "slider-percent-state") + let exactElement = UIStateParser.findElement(in: uiState, withIdentifier: "slider-exact-value-state") + let slider = UIStateParser.findElement(in: uiState, withIdentifier: "slider-value-slider") + + lastPercentText = percentElement?.label + lastExactText = exactElement?.label + lastSliderValue = slider?.value + + if let slider, + let observedSliderValue = normalizedSliderValue(slider.value), + let observedPercent = numericLabelValue(percentElement?.label, prefix: "Slider Percent State:"), + let observedExactValue = numericLabelValue(exactElement?.label, prefix: "Slider Exact Value:"), + abs(observedSliderValue - expectedNormalizedValue) <= observableValueTolerance, + abs(observedExactValue - expectedNormalizedValue) <= observableValueTolerance, + abs((observedPercent / 100.0) - expectedNormalizedValue) <= observableValueTolerance { + return SliderVerificationState( + slider: slider, + observedNormalizedValue: observedSliderValue, + observedExactValue: observedExactValue, + observedPercent: observedPercent + ) + } + + try await Task.sleep(nanoseconds: 200_000_000) + } + + throw TestError.unexpectedState( + "Timed out waiting for requested slider target \(formatCommandValue(requestedPercent))% (normalized \(formatNormalized(expectedNormalizedValue))). Last percent text: \(lastPercentText ?? "none"), last exact text: \(lastExactText ?? "none"), last AXValue: \(lastSliderValue ?? "none")" + ) + } + + private func normalizedSliderValue(_ rawValue: String?) -> Double? { + guard let rawValue else { return nil } + + let trimmedValue = rawValue.trimmingCharacters(in: .whitespacesAndNewlines) + let isPercent = trimmedValue.hasSuffix("%") + let numericText = trimmedValue.replacingOccurrences(of: "%", with: "") + guard let parsedValue = Double(numericText.trimmingCharacters(in: .whitespacesAndNewlines)) else { + return nil + } + + return isPercent || parsedValue > 1.0 ? parsedValue / 100.0 : parsedValue + } + + private func numericLabelValue(_ label: String?, prefix: String) -> Double? { + guard let label, label.hasPrefix(prefix) else { return nil } + return Double(label.dropFirst(prefix.count).trimmingCharacters(in: .whitespacesAndNewlines)) + } + + private func formatCommandValue(_ value: Double) -> String { + if value.rounded() == value { + return String(Int(value)) + } + return String(format: "%.2f", locale: Locale(identifier: "en_US_POSIX"), value) + } + + private func formatNormalized(_ value: Double) -> String { + String(format: "%.4f", locale: Locale(identifier: "en_US_POSIX"), value) + } +} + +private struct SliderVerificationState { + let slider: UIElement + let observedNormalizedValue: Double + let observedExactValue: Double + let observedPercent: Double +} diff --git a/USAGE_EXAMPLES.md b/USAGE_EXAMPLES.md index 829b724..e0afebd 100644 --- a/USAGE_EXAMPLES.md +++ b/USAGE_EXAMPLES.md @@ -74,8 +74,13 @@ axe tap -x 100 -y 200 --pre-delay 1.0 --post-delay 0.5 --udid SIMULATOR_UDID axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid SIMULATOR_UDID axe swipe --start-x 50 --start-y 500 --end-x 350 --end-y 500 --duration 2.0 --delta 25 --udid SIMULATOR_UDID -# Swipe with timing controls +# Raw low-level drag using explicit touch move events +axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --udid SIMULATOR_UDID +axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --duration 0.4 --steps 40 --udid SIMULATOR_UDID + +# Swipe and drag with timing controls axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --pre-delay 1.0 --post-delay 0.5 --udid SIMULATOR_UDID +axe drag --start-x 100 --start-y 400 --end-x 300 --end-y 400 --pre-delay 1.0 --post-delay 0.5 --udid SIMULATOR_UDID # Advanced touch control axe touch -x 150 -y 250 --down --udid SIMULATOR_UDID # Touch down only @@ -83,7 +88,19 @@ axe touch -x 150 -y 250 --up --udid SIMULATOR_UDID # Touch up only axe touch -x 150 -y 250 --down --up --delay 1.0 --udid SIMULATOR_UDID # Touch with delay ``` -### **3. Gesture Presets** 🆕 +### **3. Sliders** + +```bash +# Set a slider by accessibility identifier to a verified 0-100 percentage +axe slider --id slider-value-slider --value 75 --udid SIMULATOR_UDID + +# Set a slider by accessibility label and narrow matching to slider elements +axe slider --label "Volume" --value 40 --element-type Slider --udid SIMULATOR_UDID +``` + +The `slider` command uses the slider's accessibility frame and current AXValue to perform one calibrated low-level HID drag through the same composite touch-move path as `drag`, then verifies the new AXValue. Since iOS slider controls quantize values to their rendered track resolution, AXe verifies that the observed value is within tolerance rather than retrying correction gestures to chase unreachable decimals. If the observed AXValue remains outside tolerance after that drag, the command fails clearly. Use `describe-ui` first to find reliable `--id` or `--label` selectors. + +### **4. Gesture Presets** 🆕 ```bash # Scrolling gestures @@ -111,7 +128,7 @@ axe gesture scroll-down --pre-delay 1.0 --post-delay 0.5 --udid SIMULATOR_UDID axe gesture swipe-from-left-edge --pre-delay 2.0 --duration 0.8 --post-delay 1.0 --udid SIMULATOR_UDID ``` -### **4. Hardware Buttons** +### **5. Hardware Buttons** ```bash # Available buttons: home, lock, side-button, siri, apple-pay @@ -129,7 +146,7 @@ axe button siri --udid SIMULATOR_UDID axe button apple-pay --udid SIMULATOR_UDID ``` -### **5. Keyboard Control** +### **6. Keyboard Control** ```bash # Individual key presses by keycode @@ -148,7 +165,7 @@ axe key-combo --modifiers 227 --key 25 --udid SIMULATOR_UDID # Cmd+V axe key-combo --modifiers 227,225 --key 4 --udid SIMULATOR_UDID # Cmd+Shift+A ``` -### **6. Batch Workflows** +### **7. Batch Workflows** ```bash # Chain steps with one simulator/HID session @@ -284,6 +301,9 @@ axe gesture swipe-from-right-edge --pre-delay 1.0 --duration 0.3 --udid SIMULATO # Get accessibility info axe describe-ui --point 100,200 --udid SIMULATOR_UDID +# Set a discovered slider control +axe slider --id slider-value-slider --value 75 --udid SIMULATOR_UDID + # Navigate with presets and keyboard axe gesture scroll-down --post-delay 1.0 --udid SIMULATOR_UDID # Scroll to content axe key-sequence --keycodes 43,43,40 --delay 1.0 --udid SIMULATOR_UDID # Tab navigation @@ -435,9 +455,10 @@ done | Parameter | Range | Description | Available On | |-----------|-------|-------------|--------------| -| `--pre-delay` | 0-10 seconds | Delay before action | tap, swipe, gesture | -| `--post-delay` | 0-10 seconds | Delay after action | tap, swipe, gesture | -| `--duration` | 0-10 seconds | Action duration | swipe, gesture, button, key | +| `--pre-delay` | 0-10 seconds | Delay before action | tap, swipe, drag, gesture | +| `--post-delay` | 0-10 seconds | Delay after action | tap, swipe, drag, gesture | +| `--duration` | 0-10 seconds | Action duration | swipe, drag, gesture, button, key | +| `--steps` | 1-1000 | Touch move event count | drag | | `--delay` | 0-5 seconds | Between-key delay | key-sequence, touch | ## Benchmarking Batch vs Non-Batch @@ -461,7 +482,8 @@ This benchmark compares equivalent two-tap workflows and reports per-iteration l 5. **No Shell Escaping**: Use `--stdin` or `--file` for complex text 6. **Automation-Friendly**: Perfect for CI/CD and testing scripts 7. **Flexible Input Methods**: Multiple ways to provide input and control timing -8. **Comprehensive Validation**: Built-in parameter validation and error handling +8. **Slider Verification**: Slider controls use selector-resolved low-level HID dragging with AXValue tolerance verification +9. **Comprehensive Validation**: Built-in parameter validation and error handling ## Common Keycodes Reference diff --git a/test-runner.sh b/test-runner.sh index 690b13f..cceb079 100755 --- a/test-runner.sh +++ b/test-runner.sh @@ -59,6 +59,12 @@ show_usage() { echo "" echo "Test Filters (optional):" echo " SwipeTests Run only swipe tests" + echo " DragTests Run only drag tests" + echo " SliderTests Run only slider tests" + echo " DescribeUITests Run only describe-ui tests" + echo " InitTests Run only init tests" + echo " KeyComboTests Run only key-combo tests" + echo " KeySequenceTests Run only key-sequence tests" echo " TapTests Run only tap tests" echo " KeyTests Run only key tests" echo " TouchTests Run only touch tests" @@ -66,10 +72,14 @@ show_usage() { echo " ButtonTests Run only button tests" echo " GestureTests Run only gesture tests" echo " ListSimulatorsTests Run only list simulators tests" + echo " RecordVideoTests Run only record video tests" + echo " StreamVideoDebugTests Run only stream video debug tests" + echo " StreamVideoTests Run only stream video tests" echo "" echo "Examples:" echo " $0 # Build everything and run all tests" echo " $0 SwipeTests # Build everything and run only swipe tests" + echo " $0 DragTests # Build everything and run only drag tests" echo " $0 -t SwipeTests # Skip building, run only swipe tests" echo " $0 -b # Only build, skip tests" echo " $0 -c # Clean build and run all tests" @@ -109,7 +119,7 @@ while [[ $# -gt 0 ]]; do VERBOSE=true shift ;; - BatchTests|SwipeTests|TapTests|KeyTests|TouchTests|TypeTests|ButtonTests|GestureTests|ListSimulatorsTests) + BatchTests|ButtonTests|DescribeUITests|GestureTests|InitTests|KeyComboTests|KeySequenceTests|KeyTests|ListSimulatorsTests|RecordVideoTests|StreamVideoDebugTests|StreamVideoTests|SwipeTests|DragTests|SliderTests|TapTests|TouchTests|TypeTests) TEST_FILTER="$1" shift ;; @@ -339,6 +349,8 @@ run_tests() { "StreamVideoDebugTests" "StreamVideoTests" "SwipeTests" + "DragTests" + "SliderTests" "TapTests" "TouchTests" "TypeTests"