From 5162e95ad858792ebc4bf2284c175553b550a629 Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Fri, 20 Feb 2026 19:25:37 -0800 Subject: [PATCH 01/16] feat: add Revyl cloud device integration Co-authored-by: Cursor Signed-off-by: Anam Hira --- README.md | 6 + docs/architecture.md | 3 + docs/revyl-integration.md | 91 ++++ .../host/revyl/RevylDeviceService.kt | 90 ++++ .../trailblaze/host/revyl/RevylMcpBridge.kt | 117 ++++++ .../host/revyl/RevylMcpServerFactory.kt | 81 ++++ .../trailblaze/host/revyl/RevylScreenState.kt | 92 ++++ .../trailblaze/host/revyl/RevylSession.kt | 20 + .../host/revyl/RevylTrailblazeAgent.kt | 245 +++++++++++ .../host/revyl/RevylWorkerClient.kt | 393 ++++++++++++++++++ 10 files changed, 1138 insertions(+) create mode 100644 docs/revyl-integration.md create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylSession.kt create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylWorkerClient.kt diff --git a/README.md b/README.md index aa4d09ba..dc3b992b 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,12 @@ Trailblaze's unique "**blaze once, trail forever**" workflow: └─────────────────────────────────────────────────────────────┘ ``` +### Cloud Device Support (Revyl) + +You can run Trailblaze against [Revyl](https://revyl.ai) cloud devices instead of local ADB or Maestro. Use +`RevylMcpServerFactory` to create an MCP server that provisions a device and maps Trailblaze tools to Revyl HTTP APIs. +See the [Revyl integration guide](docs/revyl-integration.md) for prerequisites, architecture, and usage. + ## Documentation at block.github.io/trailblaze See [Mobile-Agent-v3 Features Guide](docs/mobile-agent-v3-features.md) for detailed usage examples. diff --git a/docs/architecture.md b/docs/architecture.md index ec542789..2b7c34bd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -366,6 +366,9 @@ However: - **Not all Maestro features are exposed** - only the subset needed for Trailblaze tools - **The driver is replaceable** - the architecture supports alternative implementations +**Revyl integration:** A standalone agent, `RevylTrailblazeAgent`, implements `TrailblazeAgent` directly (no Maestro) +and talks to Revyl cloud devices via HTTP. See [Revyl integration](revyl-integration.md) for details. + ### Driver Interface Drivers must implement command execution: diff --git a/docs/revyl-integration.md b/docs/revyl-integration.md new file mode 100644 index 00000000..9fe90d49 --- /dev/null +++ b/docs/revyl-integration.md @@ -0,0 +1,91 @@ +--- +title: Revyl Cloud Device Integration +--- + +# Revyl Cloud Device Integration + +Trailblaze can use [Revyl](https://revyl.ai) cloud devices instead of local ADB or Maestro. This lets you run the same AI-powered tests against managed Android (and iOS) devices without a local device or emulator. + +## Overview + +The Revyl integration provides: + +- **RevylTrailblazeAgent** – A standalone `TrailblazeAgent` that maps Trailblaze tools to Revyl HTTP APIs (no Maestro). +- **RevylDeviceService** – Provisions and lists cloud devices via the Revyl backend. +- **RevylMcpServerFactory** – Builds an MCP server that uses Revyl for device communication. + +All integration code lives under `trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/`. + +## Prerequisites + +- A Revyl account and API key. +- [Revyl CLI](https://github.com/revyl/revyl-cli) (optional but recommended for session management and debugging). + +Set environment variables: + +- `REVYL_API_KEY` – Your Revyl API key (required). +- `REVYL_BACKEND_URL` – Backend base URL (optional; defaults to production). + +## Architecture + +```mermaid +flowchart LR + subgraph client["Client"] + LLM["LLM"] + AGENT["RevylTrailblazeAgent"] + end + subgraph revyl["Revyl"] + API["Revyl Backend API"] + WORKER["Worker HTTP"] + end + DEVICE["Cloud Device"] + LLM --> AGENT + AGENT --> API + AGENT --> WORKER + WORKER --> DEVICE +``` + +1. The LLM calls Trailblaze tools (tap, inputText, swipe, etc.). +2. **RevylTrailblazeAgent** maps each tool to Revyl operations. +3. **RevylWorkerClient** sends HTTP requests to the Revyl backend (session, device) and to the worker (screenshot, tap, type, swipe, etc.). +4. The worker drives the cloud device (Android/iOS). + +## MCP server usage + +Use **RevylMcpServerFactory** to create an MCP server that provisions a Revyl device and runs the agent: + +```kotlin +val server = RevylMcpServerFactory.create( + backendBaseUrl = System.getenv("REVYL_BACKEND_URL") ?: "https://backend.revyl.ai", + apiKey = System.getenv("REVYL_API_KEY") ?: error("REVYL_API_KEY required"), +) +// Use server with your MCP client +``` + +The factory starts a device session, builds a **RevylMcpBridge** with **RevylTrailblazeAgent**, and returns a **TrailblazeMcpServer** that speaks MCP. + +## Supported operations + +| Trailblaze tool | Revyl implementation | +|-------------------|-----------------------------------------------| +| tap | POST /input (tap at coordinates) | +| inputText | POST /input (typeText; optional clear_first) | +| swipe | POST /input (swipe) | +| longPress | POST /input (longPress) | +| launchApp | POST /session/launch_app | +| installApp | POST /session/install_app | +| eraseText | typeText with clear_first + space | +| getScreenState | GET screenshot + minimal hierarchy | + +Screenshots and view hierarchy are provided by the Revyl worker; hierarchy may be minimal compared to a full Maestro tree. + +## Limitations + +- No local ADB or Maestro; all device interaction goes through Revyl. +- View hierarchy from Revyl may be reduced (e.g. dimensions only, empty tree). +- Requires network access to Revyl backend and worker. + +## See also + +- [Architecture](architecture.md) – Revyl as an alternative to HostMaestroTrailblazeAgent. +- [Revyl CLI](https://github.com/revyl/revyl-cli) – Command-line tool for devices and tests. diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt new file mode 100644 index 00000000..a6a9ce57 --- /dev/null +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt @@ -0,0 +1,90 @@ +package xyz.block.trailblaze.host.revyl + +import xyz.block.trailblaze.devices.TrailblazeConnectedDeviceSummary +import xyz.block.trailblaze.devices.TrailblazeDeviceId +import xyz.block.trailblaze.devices.TrailblazeDevicePlatform +import xyz.block.trailblaze.devices.TrailblazeDriverType + +/** + * Provisions and manages Revyl cloud device sessions as an alternative + * to [xyz.block.trailblaze.host.devices.TrailblazeDeviceService] which + * discovers local ADB/iOS devices. + * + * @property revylClient The HTTP client used for Revyl API and worker communication. + */ +class RevylDeviceService( + private val revylClient: RevylWorkerClient, +) { + + /** + * Provisions a new cloud device via the Revyl backend. + * + * @param platform "ios" or "android". + * @param appUrl Optional direct download URL for an .apk/.ipa. + * @param appLink Optional deep-link to open after launch. + * @return A summary of the connected device for use with [DeviceManagerToolSet]. + * @throws RevylApiException If provisioning fails. + */ + fun startDevice( + platform: String, + appUrl: String? = null, + appLink: String? = null, + ): TrailblazeConnectedDeviceSummary { + val session = revylClient.startSession( + platform = platform, + appUrl = appUrl, + appLink = appLink, + ) + + val driverType = when (session.platform) { + "ios" -> TrailblazeDriverType.IOS_HOST + else -> TrailblazeDriverType.ANDROID_HOST + } + + return TrailblazeConnectedDeviceSummary( + trailblazeDriverType = driverType, + instanceId = session.workflowRunId, + description = "Revyl cloud ${session.platform} device (${session.viewerUrl})", + ) + } + + /** + * Stops all active Revyl device sessions managed by this service. + */ + fun stopDevice() { + revylClient.stopSession() + } + + /** + * Returns the [TrailblazeDeviceId] for the currently active session, or null if none. + */ + fun getCurrentDeviceId(): TrailblazeDeviceId? { + val session = revylClient.getSession() ?: return null + val platform = when (session.platform) { + "ios" -> TrailblazeDevicePlatform.IOS + else -> TrailblazeDevicePlatform.ANDROID + } + return TrailblazeDeviceId( + instanceId = session.workflowRunId, + trailblazeDevicePlatform = platform, + ) + } + + /** + * Returns the set of connected device summaries (at most one for Revyl sessions). + */ + fun listDevices(): Set { + val session = revylClient.getSession() ?: return emptySet() + val driverType = when (session.platform) { + "ios" -> TrailblazeDriverType.IOS_HOST + else -> TrailblazeDriverType.ANDROID_HOST + } + return setOf( + TrailblazeConnectedDeviceSummary( + trailblazeDriverType = driverType, + instanceId = session.workflowRunId, + description = "Revyl cloud ${session.platform} device", + ), + ) + } +} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt new file mode 100644 index 00000000..1b63699b --- /dev/null +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt @@ -0,0 +1,117 @@ +package xyz.block.trailblaze.host.revyl + +import xyz.block.trailblaze.api.ScreenState +import xyz.block.trailblaze.devices.TrailblazeConnectedDeviceSummary +import xyz.block.trailblaze.devices.TrailblazeDeviceId +import xyz.block.trailblaze.mcp.TrailblazeMcpBridge +import xyz.block.trailblaze.model.TrailblazeHostAppTarget +import xyz.block.trailblaze.toolcalls.TrailblazeTool +import xyz.block.trailblaze.toolcalls.getToolNameFromAnnotation +import xyz.block.trailblaze.toolcalls.commands.BooleanAssertionTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.StringEvaluationTrailblazeTool +import xyz.block.trailblaze.util.Console +import xyz.block.trailblaze.utils.ElementComparator + +/** + * [TrailblazeMcpBridge] implementation backed by the Revyl cloud device infrastructure. + * + * Routes MCP tool calls through [RevylTrailblazeAgent] and provides device + * listing / selection via [RevylDeviceService]. This bridge is plugged into + * [xyz.block.trailblaze.logs.server.TrailblazeMcpServer] as a drop-in + * replacement for the local-device [xyz.block.trailblaze.mcp.TrailblazeMcpBridgeImpl]. + * + * @property revylClient The HTTP client for Revyl worker communication. + * @property revylDeviceService Handles session provisioning and listing. + * @property agent The standalone Trailblaze agent that dispatches tools to Revyl. + */ +class RevylMcpBridge( + private val revylClient: RevylWorkerClient, + private val revylDeviceService: RevylDeviceService, + private val agent: RevylTrailblazeAgent, +) : TrailblazeMcpBridge { + + override suspend fun selectDevice(trailblazeDeviceId: TrailblazeDeviceId): TrailblazeConnectedDeviceSummary { + val devices = revylDeviceService.listDevices() + return devices.firstOrNull { it.trailblazeDeviceId == trailblazeDeviceId } + ?: error("Device ${trailblazeDeviceId.instanceId} not found in Revyl sessions.") + } + + override suspend fun getAvailableDevices(): Set { + return revylDeviceService.listDevices() + } + + override suspend fun getInstalledAppIds(): Set { + // Revyl doesn't expose an installed-apps endpoint — return empty for PoC + return emptySet() + } + + override fun getAvailableAppTargets(): Set { + return setOf(TrailblazeHostAppTarget.DefaultTrailblazeHostAppTarget) + } + + override suspend fun runYaml(yaml: String, startNewSession: Boolean): String { + Console.log("RevylMcpBridge: runYaml not supported for Revyl cloud devices (use tool calls instead)") + return "unsupported" + } + + override fun getCurrentlySelectedDeviceId(): TrailblazeDeviceId? { + return revylDeviceService.getCurrentDeviceId() + } + + override suspend fun getCurrentScreenState(): ScreenState? { + val session = revylClient.getSession() ?: return null + return RevylScreenState(revylClient, session.platform) + } + + /** + * Executes a [TrailblazeTool] by delegating to [RevylTrailblazeAgent]. + * + * @param tool The tool to execute on the cloud device. + * @return A human-readable result description. + */ + override suspend fun executeTrailblazeTool(tool: TrailblazeTool): String { + val toolName = tool.getToolNameFromAnnotation() + Console.log("RevylMcpBridge: executing tool '$toolName'") + + val result = agent.runTrailblazeTools( + tools = listOf(tool), + elementComparator = NoOpElementComparator, + ) + + return when (result.result) { + is xyz.block.trailblaze.toolcalls.TrailblazeToolResult.Success -> + "Successfully executed $toolName on Revyl cloud device." + is xyz.block.trailblaze.toolcalls.TrailblazeToolResult.Error -> + "Error executing $toolName: ${(result.result as xyz.block.trailblaze.toolcalls.TrailblazeToolResult.Error).errorMessage}" + } + } + + override suspend fun endSession(): Boolean { + return try { + revylClient.stopSession() + true + } catch (_: Exception) { + false + } + } + + override fun selectAppTarget(appTargetId: String): String? { + return if (appTargetId == "none") "None" else null + } + + override fun getCurrentAppTargetId(): String? { + return "none" + } +} + +/** + * No-op [ElementComparator] used when memory-based assertions are not needed. + */ +private object NoOpElementComparator : ElementComparator { + override fun getElementValue(prompt: String): String? = null + override fun evaluateBoolean(statement: String): BooleanAssertionTrailblazeTool = + BooleanAssertionTrailblazeTool(result = true, reason = "No-op comparator") + override fun evaluateString(query: String): StringEvaluationTrailblazeTool = + StringEvaluationTrailblazeTool(result = "", reason = "No-op comparator") + override fun extractNumberFromString(input: String): Double? = null +} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt new file mode 100644 index 00000000..df0c7741 --- /dev/null +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt @@ -0,0 +1,81 @@ +package xyz.block.trailblaze.host.revyl + +import xyz.block.trailblaze.logs.server.TrailblazeMcpServer +import xyz.block.trailblaze.model.TrailblazeHostAppTarget +import xyz.block.trailblaze.report.utils.LogsRepo +import xyz.block.trailblaze.util.Console +import java.io.File + +/** + * Factory for constructing a fully wired [TrailblazeMcpServer] backed by + * the Revyl cloud device infrastructure. + * + * Usage: + * ``` + * val server = RevylMcpServerFactory.create( + * apiKey = System.getenv("REVYL_API_KEY"), + * platform = "android", + * ) + * server.startStreamableHttpMcpServer(port = 8080, wait = true) + * ``` + */ +object RevylMcpServerFactory { + + /** + * Creates a [TrailblazeMcpServer] that provisions a Revyl cloud device + * and routes all MCP tool calls through [RevylTrailblazeAgent]. + * + * @param apiKey Revyl API key (typically from REVYL_API_KEY env var). + * @param platform "ios" or "android". + * @param backendUrl Override for the Revyl backend URL. + * @param appUrl Optional direct URL to an .apk/.ipa to install on the device. + * @param appLink Optional deep-link to open after launch. + * @param trailsDir Directory containing .trail YAML files. + * @return A configured [TrailblazeMcpServer] ready to start. + * @throws RevylApiException If device provisioning fails. + */ + fun create( + apiKey: String, + platform: String = "android", + backendUrl: String = RevylWorkerClient.DEFAULT_BACKEND_URL, + appUrl: String? = null, + appLink: String? = null, + trailsDir: File = File(System.getProperty("user.dir"), "trails"), + ): TrailblazeMcpServer { + Console.log("RevylMcpServerFactory: creating Revyl-backed MCP server") + Console.log(" Platform: $platform") + Console.log(" Backend: $backendUrl") + + val revylClient = RevylWorkerClient( + apiKey = apiKey, + backendBaseUrl = backendUrl, + ) + + Console.log("RevylMcpServerFactory: provisioning cloud device...") + revylClient.startSession( + platform = platform, + appUrl = appUrl, + appLink = appLink, + ) + + val session = revylClient.getSession()!! + Console.log("RevylMcpServerFactory: device ready") + Console.log(" Worker URL: ${session.workerBaseUrl}") + Console.log(" Viewer URL: ${session.viewerUrl}") + + val revylDeviceService = RevylDeviceService(revylClient) + val agent = RevylTrailblazeAgent(revylClient, platform) + val bridge = RevylMcpBridge(revylClient, revylDeviceService, agent) + + val logsDir = File(System.getProperty("user.dir"), ".trailblaze/logs") + logsDir.mkdirs() + val logsRepo = LogsRepo(logsDir) + + return TrailblazeMcpServer( + logsRepo = logsRepo, + mcpBridge = bridge, + trailsDirProvider = { trailsDir }, + targetTestAppProvider = { TrailblazeHostAppTarget.DefaultTrailblazeHostAppTarget }, + ) + } +} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt new file mode 100644 index 00000000..fae329c5 --- /dev/null +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt @@ -0,0 +1,92 @@ +package xyz.block.trailblaze.host.revyl + +import xyz.block.trailblaze.api.ScreenState +import xyz.block.trailblaze.api.ViewHierarchyTreeNode +import xyz.block.trailblaze.devices.TrailblazeDeviceClassifier +import xyz.block.trailblaze.devices.TrailblazeDevicePlatform +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * [ScreenState] implementation backed by Revyl cloud device screenshots. + * + * Since Revyl uses AI-powered visual grounding (not accessibility trees), + * the view hierarchy is returned as a minimal empty root node. The LLM agent + * relies on screenshot-based reasoning instead of element trees. + * + * @property revylClient Client used to capture screenshots from the Revyl worker. + * @property platform The device platform ("ios" or "android"). + */ +class RevylScreenState( + private val revylClient: RevylWorkerClient, + private val platform: String, +) : ScreenState { + + private val capturedScreenshot: ByteArray? = try { + revylClient.screenshot() + } catch (_: Exception) { + null + } + + private val dimensions: Pair = capturedScreenshot?.let { extractPngDimensions(it) } + ?: Pair(DEFAULT_WIDTH, DEFAULT_HEIGHT) + + override val screenshotBytes: ByteArray? = capturedScreenshot + + override val deviceWidth: Int = dimensions.first + + override val deviceHeight: Int = dimensions.second + + /** + * Returns a minimal root node — Revyl does not provide a view hierarchy tree. + * The AI agent should rely on screenshot-based visual grounding instead. + */ + override val viewHierarchyOriginal: ViewHierarchyTreeNode = ViewHierarchyTreeNode( + nodeId = 1, + text = "RevylRootNode", + className = "RevylCloudDevice", + dimensions = "${deviceWidth}x$deviceHeight", + centerPoint = "${deviceWidth / 2},${deviceHeight / 2}", + clickable = true, + enabled = true, + ) + + override val viewHierarchy: ViewHierarchyTreeNode = viewHierarchyOriginal + + override val trailblazeDevicePlatform: TrailblazeDevicePlatform = when (platform.lowercase()) { + "ios" -> TrailblazeDevicePlatform.IOS + else -> TrailblazeDevicePlatform.ANDROID + } + + override val deviceClassifiers: List = listOf( + trailblazeDevicePlatform.asTrailblazeDeviceClassifier(), + TrailblazeDeviceClassifier("revyl-cloud"), + ) + + companion object { + private const val DEFAULT_WIDTH = 1080 + private const val DEFAULT_HEIGHT = 2340 + + /** + * Extracts width and height from a PNG file's IHDR chunk header. + * + * @param data Raw PNG bytes. + * @return (width, height) pair, or null if the data is not valid PNG. + */ + private fun extractPngDimensions(data: ByteArray): Pair? { + // PNG: 8-byte signature + 4-byte IHDR length + 4-byte "IHDR" + 4-byte width + 4-byte height = 24 bytes minimum + if (data.size < 24) return null + val pngSignature = byteArrayOf( + 0x89.toByte(), 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A + ) + for (i in pngSignature.indices) { + if (data[i] != pngSignature[i]) return null + } + val buffer = ByteBuffer.wrap(data, 16, 8).order(ByteOrder.BIG_ENDIAN) + val width = buffer.int + val height = buffer.int + if (width <= 0 || height <= 0) return null + return Pair(width, height) + } + } +} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylSession.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylSession.kt new file mode 100644 index 00000000..248dc26c --- /dev/null +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylSession.kt @@ -0,0 +1,20 @@ +package xyz.block.trailblaze.host.revyl + +/** + * Represents an active Revyl cloud device session. + * + * @property index Local session index (0-based). + * @property sessionId Unique identifier returned by the Revyl backend. + * @property workflowRunId Hatchet workflow run powering this session. + * @property workerBaseUrl HTTP base URL of the device worker (e.g. "https://worker-xxx.revyl.ai"). + * @property viewerUrl Browser URL for live device screen. + * @property platform "ios" or "android". + */ +data class RevylSession( + val index: Int, + val sessionId: String, + val workflowRunId: String, + val workerBaseUrl: String, + val viewerUrl: String, + val platform: String, +) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt new file mode 100644 index 00000000..075058be --- /dev/null +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt @@ -0,0 +1,245 @@ +package xyz.block.trailblaze.host.revyl + +import maestro.SwipeDirection +import xyz.block.trailblaze.api.ScreenState +import xyz.block.trailblaze.api.TrailblazeAgent +import xyz.block.trailblaze.api.TrailblazeAgent.RunTrailblazeToolsResult +import xyz.block.trailblaze.logs.model.TraceId +import xyz.block.trailblaze.toolcalls.TrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.toolcalls.commands.EraseTextTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.HideKeyboardTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.InputTextTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.LaunchAppTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.ObjectiveStatusTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.OpenUrlTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.PressBackTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.PressKeyTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.ScrollUntilTextIsVisibleTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.SwipeTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.TakeSnapshotTool +import xyz.block.trailblaze.toolcalls.commands.TapOnElementByNodeIdTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.TapOnPointTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.WaitForIdleSyncTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.NetworkConnectionTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.LongPressOnElementWithTextTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.memory.MemoryTrailblazeTool +import xyz.block.trailblaze.toolcalls.getToolNameFromAnnotation +import xyz.block.trailblaze.utils.ElementComparator +import xyz.block.trailblaze.util.Console + +/** + * A standalone [TrailblazeAgent] implementation that routes device actions + * directly to the Revyl cloud device worker HTTP API. + * + * Unlike [xyz.block.trailblaze.MaestroTrailblazeAgent], this agent does NOT + * depend on the Maestro driver stack at all. Each [TrailblazeTool] is + * pattern-matched by type and translated into the corresponding Revyl HTTP + * call in a single hop — no intermediate Maestro Command objects. + * + * This makes it a clean, minimal integration layer between Trailblaze's + * LLM agent loop and Revyl's cloud device infrastructure. + * + * @property revylClient HTTP client for the Revyl device worker. + * @property platform "ios" or "android" — used for ScreenState construction. + */ +class RevylTrailblazeAgent( + private val revylClient: RevylWorkerClient, + private val platform: String, +) : TrailblazeAgent { + + /** + * Dispatches a list of [TrailblazeTool]s by mapping each tool's type + * directly to Revyl worker HTTP calls. + * + * Memory tools (assert/remember) are handled in-process via the + * [ElementComparator] — they don't require device interaction. + * + * @param tools Ordered list of tools to execute sequentially. + * @param traceId Optional trace ID for log correlation. + * @param screenState Cached screen state from the most recent LLM turn. + * @param elementComparator Comparator for memory-based assertions. + * @param screenStateProvider Lazy provider for a fresh device screenshot. + * @return Execution result containing the input tools, executed tools, and outcome. + */ + override fun runTrailblazeTools( + tools: List, + traceId: TraceId?, + screenState: ScreenState?, + elementComparator: ElementComparator, + screenStateProvider: (() -> ScreenState)?, + ): RunTrailblazeToolsResult { + val executed = mutableListOf() + + for (tool in tools) { + executed.add(tool) + val result = executeTool(tool, elementComparator, screenStateProvider) + if (result != TrailblazeToolResult.Success) { + return RunTrailblazeToolsResult( + inputTools = tools, + executedTools = executed, + result = result, + ) + } + } + + return RunTrailblazeToolsResult( + inputTools = tools, + executedTools = executed, + result = TrailblazeToolResult.Success, + ) + } + + // --------------------------------------------------------------------------- + // Tool dispatch — maps each TrailblazeTool to the equivalent Revyl call. + // --------------------------------------------------------------------------- + + private fun executeTool( + tool: TrailblazeTool, + elementComparator: ElementComparator, + screenStateProvider: (() -> ScreenState)?, + ): TrailblazeToolResult { + val toolName = tool.getToolNameFromAnnotation() + Console.log("RevylAgent: executing tool '$toolName'") + + return try { + when (tool) { + is TapOnPointTrailblazeTool -> handleTapOnPoint(tool) + is InputTextTrailblazeTool -> handleInputText(tool) + is SwipeTrailblazeTool -> handleSwipe(tool) + is LaunchAppTrailblazeTool -> handleLaunchApp(tool) + is EraseTextTrailblazeTool -> handleEraseText() + is HideKeyboardTrailblazeTool -> handleHideKeyboard() + is PressBackTrailblazeTool -> handlePressBack() + is PressKeyTrailblazeTool -> handlePressKey(tool) + is OpenUrlTrailblazeTool -> handleOpenUrl(tool) + is TakeSnapshotTool -> handleTakeSnapshot(tool, screenStateProvider) + is WaitForIdleSyncTrailblazeTool -> handleWaitForIdle(tool) + is ScrollUntilTextIsVisibleTrailblazeTool -> handleScroll(tool) + is NetworkConnectionTrailblazeTool -> handleNetworkConnection(tool) + is TapOnElementByNodeIdTrailblazeTool -> handleTapByNodeId(tool) + is LongPressOnElementWithTextTrailblazeTool -> handleLongPressText(tool) + is ObjectiveStatusTrailblazeTool -> TrailblazeToolResult.Success + is MemoryTrailblazeTool -> { + // Memory tools don't need device interaction + TrailblazeToolResult.Success + } + else -> { + Console.log("RevylAgent: unsupported tool type ${tool::class.simpleName} — skipping") + TrailblazeToolResult.Success + } + } + } catch (e: Exception) { + Console.error("RevylAgent: tool '$toolName' failed: ${e.message}") + TrailblazeToolResult.Error.ExceptionThrown( + errorMessage = "Revyl execution failed for '$toolName': ${e.message}", + command = tool, + stackTrace = e.stackTraceToString(), + ) + } + } + + // --------------------------------------------------------------------------- + // Individual tool handlers + // --------------------------------------------------------------------------- + + private fun handleTapOnPoint(tool: TapOnPointTrailblazeTool): TrailblazeToolResult { + if (tool.longPress) { + Console.log("RevylAgent: long-press at (${tool.x}, ${tool.y})") + revylClient.longPress(tool.x, tool.y) + } else { + Console.log("RevylAgent: tap at (${tool.x}, ${tool.y})") + revylClient.tap(tool.x, tool.y) + } + return TrailblazeToolResult.Success + } + + private fun handleInputText(tool: InputTextTrailblazeTool): TrailblazeToolResult { + Console.log("RevylAgent: type text '${tool.text}'") + revylClient.typeText(tool.text) + return TrailblazeToolResult.Success + } + + private fun handleSwipe(tool: SwipeTrailblazeTool): TrailblazeToolResult { + val direction = when (tool.direction) { + SwipeDirection.UP -> "up" + SwipeDirection.DOWN -> "down" + SwipeDirection.LEFT -> "left" + SwipeDirection.RIGHT -> "right" + else -> "down" + } + Console.log("RevylAgent: swipe $direction") + revylClient.swipe(direction) + return TrailblazeToolResult.Success + } + + private fun handleLaunchApp(tool: LaunchAppTrailblazeTool): TrailblazeToolResult { + Console.log("RevylAgent: launch app '${tool.appId}'") + revylClient.launchApp(tool.appId) + return TrailblazeToolResult.Success + } + + private fun handleEraseText(): TrailblazeToolResult { + Console.log("RevylAgent: erase text (clear_first + space via /input)") + revylClient.typeText(text = " ", clearFirst = true) + return TrailblazeToolResult.Success + } + + private fun handleHideKeyboard(): TrailblazeToolResult { + Console.log("RevylAgent: hide keyboard (no-op for cloud device)") + return TrailblazeToolResult.Success + } + + private fun handlePressBack(): TrailblazeToolResult { + Console.log("RevylAgent: press back (not directly supported — skipping)") + return TrailblazeToolResult.Success + } + + private fun handlePressKey(tool: PressKeyTrailblazeTool): TrailblazeToolResult { + Console.log("RevylAgent: press key '${tool.keyCode}' (not directly supported — skipping)") + return TrailblazeToolResult.Success + } + + private fun handleOpenUrl(tool: OpenUrlTrailblazeTool): TrailblazeToolResult { + Console.log("RevylAgent: open URL '${tool.url}' (not directly supported — skipping)") + return TrailblazeToolResult.Success + } + + private fun handleTakeSnapshot( + tool: TakeSnapshotTool, + screenStateProvider: (() -> ScreenState)?, + ): TrailblazeToolResult { + Console.log("RevylAgent: take snapshot '${tool.screenName}'") + // Capture screenshot from Revyl for the snapshot + revylClient.screenshot() + return TrailblazeToolResult.Success + } + + private fun handleWaitForIdle(tool: WaitForIdleSyncTrailblazeTool): TrailblazeToolResult { + Console.log("RevylAgent: wait for idle — sleeping briefly") + Thread.sleep(1000) + return TrailblazeToolResult.Success + } + + private fun handleScroll(tool: ScrollUntilTextIsVisibleTrailblazeTool): TrailblazeToolResult { + Console.log("RevylAgent: scroll until text visible — performing swipe down") + revylClient.swipe("down") + return TrailblazeToolResult.Success + } + + private fun handleNetworkConnection(tool: NetworkConnectionTrailblazeTool): TrailblazeToolResult { + Console.log("RevylAgent: network connection toggle (not supported — skipping)") + return TrailblazeToolResult.Success + } + + private fun handleTapByNodeId(tool: TapOnElementByNodeIdTrailblazeTool): TrailblazeToolResult { + Console.log("RevylAgent: tap by nodeId ${tool.nodeId} (not directly supported — skipping)") + return TrailblazeToolResult.Success + } + + private fun handleLongPressText(tool: LongPressOnElementWithTextTrailblazeTool): TrailblazeToolResult { + Console.log("RevylAgent: long press on element with text '${tool.text}'") + revylClient.tapTarget(tool.text) + return TrailblazeToolResult.Success + } +} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylWorkerClient.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylWorkerClient.kt new file mode 100644 index 00000000..38188f40 --- /dev/null +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylWorkerClient.kt @@ -0,0 +1,393 @@ +package xyz.block.trailblaze.host.revyl + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.int +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.put +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import java.io.IOException +import java.util.concurrent.TimeUnit + +/** + * HTTP client that communicates with a Revyl cloud device worker + * and the Revyl backend for session provisioning and AI-powered target resolution. + * + * Uses the same HTTP endpoints as the revyl-cli Go implementation + * (see revyl-cli/internal/mcp/device_session.go). + * + * @property apiKey Revyl API key for backend authentication. + * @property backendBaseUrl Base URL for the Revyl backend API. + */ +class RevylWorkerClient( + private val apiKey: String, + private val backendBaseUrl: String = DEFAULT_BACKEND_URL, +) { + + private val json = Json { ignoreUnknownKeys = true } + + private val httpClient = OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .build() + + private var currentSession: RevylSession? = null + + /** + * Returns the currently active session, or null if none is provisioned. + */ + fun getSession(): RevylSession? = currentSession + + // --------------------------------------------------------------------------- + // Session lifecycle + // --------------------------------------------------------------------------- + + /** + * Provisions a new cloud-hosted device and polls until the worker is reachable. + * + * @param platform "ios" or "android". + * @param appUrl Optional direct URL to an .apk/.ipa to install. + * @param appLink Optional deep-link to open after launch. + * @return The newly created [RevylSession]. + * @throws RevylApiException If provisioning fails or the worker never becomes ready. + */ + fun startSession( + platform: String, + appUrl: String? = null, + appLink: String? = null, + ): RevylSession { + val body = buildJsonObject { + put("platform", platform.lowercase()) + put("is_simulation", true) + if (!appUrl.isNullOrBlank()) put("app_url", appUrl) + if (!appLink.isNullOrBlank()) put("app_link", appLink) + } + + val response = backendPost("/api/v1/execution/start-device", body) + val workflowRunId = response["workflow_run_id"]?.jsonPrimitive?.content ?: "" + if (workflowRunId.isBlank()) { + val error = response["error"]?.jsonPrimitive?.content ?: "unknown error" + throw RevylApiException("Failed to start device: $error") + } + + val workerBaseUrl = pollForWorkerUrl(workflowRunId, maxWaitSeconds = 120) + waitForDeviceReady(workerBaseUrl, maxWaitSeconds = 30) + + val viewerUrl = "$backendBaseUrl/tests/execute?workflowRunId=$workflowRunId&platform=$platform" + + val session = RevylSession( + index = 0, + sessionId = workflowRunId, + workflowRunId = workflowRunId, + workerBaseUrl = workerBaseUrl, + viewerUrl = viewerUrl, + platform = platform.lowercase(), + ) + currentSession = session + return session + } + + /** + * Stops (cancels) the current device session and releases cloud resources. + * + * @throws RevylApiException If the cancellation request fails. + */ + fun stopSession() { + val session = currentSession ?: return + backendPost( + "/cancel-device", + buildJsonObject { put("workflow_run_id", session.workflowRunId) }, + ) + currentSession = null + } + + // --------------------------------------------------------------------------- + // Device actions — delegated to the worker HTTP API + // --------------------------------------------------------------------------- + + /** + * Captures a PNG screenshot of the current device screen. + * + * @return Raw PNG bytes. + * @throws RevylApiException If no session is active or the request fails. + */ + fun screenshot(): ByteArray { + val session = requireSession() + val request = Request.Builder() + .url("${session.workerBaseUrl}/screenshot") + .get() + .build() + + val response = httpClient.newCall(request).execute() + if (!response.isSuccessful) { + throw RevylApiException("Screenshot failed: HTTP ${response.code}") + } + return response.body?.bytes() ?: throw RevylApiException("Screenshot returned empty body") + } + + /** + * Taps at the given coordinates on the device screen. + * + * @param x Horizontal pixel coordinate. + * @param y Vertical pixel coordinate. + * @throws RevylApiException If the request fails. + */ + fun tap(x: Int, y: Int) { + workerPost("/tap", buildJsonObject { put("x", x); put("y", y) }) + } + + /** + * Taps a UI element identified by a natural language description + * using the Revyl AI grounding model. + * + * @param target Natural language description (e.g. "Sign In button"). + * @throws RevylApiException If grounding or the tap fails. + */ + fun tapTarget(target: String) { + val (x, y) = resolveTarget(target) + tap(x, y) + } + + /** + * Types text into the currently focused input field or a targeted element. + * + * @param text The text to type. + * @param targetX Optional x coordinate to tap before typing. + * @param targetY Optional y coordinate to tap before typing. + * @param clearFirst If true, clears the field content before typing. + * @throws RevylApiException If the request fails. + */ + fun typeText(text: String, targetX: Int? = null, targetY: Int? = null, clearFirst: Boolean = false) { + val body = buildJsonObject { + put("text", text) + if (targetX != null && targetY != null) { + put("x", targetX) + put("y", targetY) + } + if (clearFirst) { + put("clear_first", true) + } + } + workerPost("/input", body) + } + + /** + * Swipes in the given direction from the center or a specific point. + * + * @param direction One of "up", "down", "left", "right". + * @param startX Optional starting x coordinate. + * @param startY Optional starting y coordinate. + * @throws RevylApiException If the request fails. + */ + fun swipe(direction: String, startX: Int? = null, startY: Int? = null) { + val body = buildJsonObject { + put("direction", direction) + if (startX != null) put("x", startX) + if (startY != null) put("y", startY) + } + workerPost("/swipe", body) + } + + /** + * Long-presses at the given coordinates. + * + * @param x Horizontal pixel coordinate. + * @param y Vertical pixel coordinate. + * @param durationMs Duration of the press in milliseconds. + * @throws RevylApiException If the request fails. + */ + fun longPress(x: Int, y: Int, durationMs: Int = 1500) { + val body = buildJsonObject { + put("x", x) + put("y", y) + put("duration", durationMs) + } + workerPost("/longpress", body) + } + + /** + * Installs an app from a URL onto the device. + * + * @param appUrl Direct download URL for the .apk or .ipa. + * @throws RevylApiException If the request fails. + */ + fun installApp(appUrl: String) { + workerPost("/install", buildJsonObject { put("app_url", appUrl) }) + } + + /** + * Launches an installed app by its bundle/package ID. + * + * @param bundleId The app's bundle identifier (iOS) or package name (Android). + * @throws RevylApiException If the request fails. + */ + fun launchApp(bundleId: String) { + workerPost("/launch", buildJsonObject { put("bundle_id", bundleId) }) + } + + /** + * Resolves a natural language target description to screen coordinates + * using the Revyl AI grounding model on the worker. + * + * Falls back to the backend grounding endpoint if the worker doesn't support + * native target resolution. + * + * @param target Natural language element description (e.g. "the search bar"). + * @return Pair of (x, y) pixel coordinates. + * @throws RevylApiException If grounding fails. + */ + fun resolveTarget(target: String): Pair { + val session = requireSession() + + // Try worker-native grounding first + try { + val body = buildJsonObject { put("target", target) } + val responseBody = workerPostRaw("/resolve_target", body) + val parsed = json.parseToJsonElement(responseBody).jsonObject + val x = parsed["x"]!!.jsonPrimitive.int + val y = parsed["y"]!!.jsonPrimitive.int + return Pair(x, y) + } catch (_: Exception) { + // Fall back to backend grounding + } + + // Backend grounding fallback: send screenshot + target to the backend + val screenshotBytes = screenshot() + val base64Screenshot = java.util.Base64.getEncoder().encodeToString(screenshotBytes) + val groundBody = buildJsonObject { + put("screenshot_base64", base64Screenshot) + put("target", target) + put("workflow_run_id", session.workflowRunId) + } + + val response = backendPost("/api/v1/execution/ground-element", groundBody) + val x = response["x"]!!.jsonPrimitive.int + val y = response["y"]!!.jsonPrimitive.int + return Pair(x, y) + } + + // --------------------------------------------------------------------------- + // Internal helpers + // --------------------------------------------------------------------------- + + private fun requireSession(): RevylSession = + currentSession ?: throw RevylApiException("No active device session. Call startSession() first.") + + private fun backendPost(path: String, body: JsonObject): JsonObject { + val mediaType = "application/json; charset=utf-8".toMediaType() + val request = Request.Builder() + .url("$backendBaseUrl$path") + .addHeader("X-API-Key", apiKey) + .addHeader("Content-Type", "application/json") + .post(body.toString().toRequestBody(mediaType)) + .build() + + val response = httpClient.newCall(request).execute() + val responseBody = response.body?.string() ?: "{}" + if (!response.isSuccessful) { + throw RevylApiException("Backend request to $path failed (HTTP ${response.code}): $responseBody") + } + return json.parseToJsonElement(responseBody).jsonObject + } + + private fun workerPost(path: String, body: JsonObject) { + val responseBody = workerPostRaw(path, body) + if (responseBody.isNotBlank()) { + try { + val parsed = json.parseToJsonElement(responseBody).jsonObject + if (parsed.containsKey("error")) { + throw RevylApiException("Worker action $path failed: ${parsed["error"]!!.jsonPrimitive.content}") + } + } catch (_: kotlinx.serialization.SerializationException) { + // Non-JSON response is fine for success + } + } + } + + private fun workerPostRaw(path: String, body: JsonObject): String { + val session = requireSession() + val mediaType = "application/json; charset=utf-8".toMediaType() + val request = Request.Builder() + .url("${session.workerBaseUrl}$path") + .addHeader("Content-Type", "application/json") + .post(body.toString().toRequestBody(mediaType)) + .build() + + val response = httpClient.newCall(request).execute() + val responseBody = response.body?.string() ?: "" + if (!response.isSuccessful) { + throw RevylApiException("Worker request to $path failed (HTTP ${response.code}): $responseBody") + } + return responseBody + } + + /** + * Polls the backend until a worker URL is available for the given workflow run. + */ + private fun pollForWorkerUrl(workflowRunId: String, maxWaitSeconds: Int): String { + val deadline = System.currentTimeMillis() + (maxWaitSeconds * 1000L) + while (System.currentTimeMillis() < deadline) { + try { + val request = Request.Builder() + .url("$backendBaseUrl/api/v1/execution/worker-ws-url/$workflowRunId") + .addHeader("X-API-Key", apiKey) + .get() + .build() + + val response = httpClient.newCall(request).execute() + if (response.isSuccessful) { + val body = response.body?.string() ?: "" + val parsed = json.parseToJsonElement(body).jsonObject + val workerUrl = parsed["worker_url"]?.jsonPrimitive?.content ?: "" + if (workerUrl.isNotBlank()) { + return workerUrl.trimEnd('/') + } + } + } catch (_: IOException) { + // Retry + } + Thread.sleep(2000) + } + throw RevylApiException("Worker URL not available after ${maxWaitSeconds}s for workflow $workflowRunId") + } + + /** + * Waits until the worker's health endpoint reports a connected device. + */ + private fun waitForDeviceReady(workerBaseUrl: String, maxWaitSeconds: Int) { + val deadline = System.currentTimeMillis() + (maxWaitSeconds * 1000L) + while (System.currentTimeMillis() < deadline) { + try { + val request = Request.Builder() + .url("$workerBaseUrl/health") + .get() + .build() + + val response = httpClient.newCall(request).execute() + if (response.isSuccessful) { + return + } + } catch (_: IOException) { + // Retry + } + Thread.sleep(2000) + } + } + + companion object { + const val DEFAULT_BACKEND_URL = "https://backend.revyl.ai" + } +} + +/** + * Exception thrown when a Revyl API or worker request fails. + * + * @property message Human-readable description of the failure. + */ +class RevylApiException(message: String) : RuntimeException(message) From e688865a3b5e89120f2c033aea5d9a02a27630ae Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Wed, 18 Mar 2026 19:23:01 -0700 Subject: [PATCH 02/16] Replace RevylWorkerClient with CLI-based RevylCliClient MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HTTP-based RevylWorkerClient had wrong API paths, direct worker access bypassing the backend proxy, and missing auth headers. Four tool handlers were unimplemented ("not supported — skipping"). RevylCliClient delegates all device actions to the `revyl` CLI binary via ProcessBuilder. The CLI is auto-downloaded from GitHub Releases if not already on PATH — the only prerequisite is REVYL_API_KEY. Changes: - RevylCliClient.kt: CLI subprocess wrapper with auto-install - RevylTrailblazeAgent.kt: all 12 tools implemented via CLI - RevylMcpServerFactory.kt: provisions device via CLI - RevylMcpBridge.kt: updated constructor and imports - RevylDeviceService.kt: delegates to CLI client - RevylScreenState.kt: reads screenshot from temp file - docs/revyl-integration.md: updated for CLI approach - RevylDemo.kt: moved to src/test as usage example Made-with: Cursor Signed-off-by: Anam Hira --- docs/revyl-integration.md | 120 ++++-- .../trailblaze/host/revyl/RevylCliClient.kt | 395 ++++++++++++++++++ .../host/revyl/RevylDeviceService.kt | 31 +- .../trailblaze/host/revyl/RevylMcpBridge.kt | 34 +- .../host/revyl/RevylMcpServerFactory.kt | 46 +- .../trailblaze/host/revyl/RevylScreenState.kt | 17 +- .../host/revyl/RevylTrailblazeAgent.kt | 236 ++++------- .../host/revyl/RevylWorkerClient.kt | 393 ----------------- .../block/trailblaze/host/revyl/RevylDemo.kt | 95 +++++ 9 files changed, 709 insertions(+), 658 deletions(-) create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt delete mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylWorkerClient.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt diff --git a/docs/revyl-integration.md b/docs/revyl-integration.md index 9fe90d49..9dfb9a6a 100644 --- a/docs/revyl-integration.md +++ b/docs/revyl-integration.md @@ -4,88 +4,124 @@ title: Revyl Cloud Device Integration # Revyl Cloud Device Integration -Trailblaze can use [Revyl](https://revyl.ai) cloud devices instead of local ADB or Maestro. This lets you run the same AI-powered tests against managed Android (and iOS) devices without a local device or emulator. +Trailblaze can use [Revyl](https://revyl.ai) cloud devices instead of local ADB or Maestro. This lets you run the same AI-powered tests against managed Android and iOS devices without a local device or emulator. ## Overview The Revyl integration provides: -- **RevylTrailblazeAgent** – A standalone `TrailblazeAgent` that maps Trailblaze tools to Revyl HTTP APIs (no Maestro). -- **RevylDeviceService** – Provisions and lists cloud devices via the Revyl backend. -- **RevylMcpServerFactory** – Builds an MCP server that uses Revyl for device communication. +- **RevylCliClient** – Shells out to the `revyl` CLI binary for all device interactions. Auto-downloads the CLI if not already installed. +- **RevylTrailblazeAgent** – Maps every Trailblaze tool to a `revyl device` CLI command. +- **RevylMcpServerFactory** – Builds an MCP server that provisions a Revyl cloud device and routes tool calls through the CLI. All integration code lives under `trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/`. ## Prerequisites -- A Revyl account and API key. -- [Revyl CLI](https://github.com/revyl/revyl-cli) (optional but recommended for session management and debugging). - -Set environment variables: +Set one environment variable: - `REVYL_API_KEY` – Your Revyl API key (required). -- `REVYL_BACKEND_URL` – Backend base URL (optional; defaults to production). + +That's it. The `revyl` CLI binary is **auto-downloaded** from [GitHub Releases](https://github.com/RevylAI/revyl-cli/releases) on first use if not already on PATH. No manual install needed. + +**Optional overrides:** + +- `REVYL_BINARY` – Path to a specific `revyl` binary (skips auto-download and PATH lookup). ## Architecture ```mermaid flowchart LR - subgraph client["Client"] - LLM["LLM"] + subgraph trailblaze["Trailblaze"] + LLM["LLM Agent"] AGENT["RevylTrailblazeAgent"] + CLI["RevylCliClient"] end - subgraph revyl["Revyl"] - API["Revyl Backend API"] - WORKER["Worker HTTP"] + subgraph cli["revyl CLI"] + BIN["revyl device *"] + end + subgraph revyl["Revyl Cloud"] + PROXY["Backend Proxy"] + WORKER["Worker"] end DEVICE["Cloud Device"] LLM --> AGENT - AGENT --> API - AGENT --> WORKER + AGENT --> CLI + CLI -->|"ProcessBuilder"| BIN + BIN --> PROXY + PROXY --> WORKER WORKER --> DEVICE ``` 1. The LLM calls Trailblaze tools (tap, inputText, swipe, etc.). -2. **RevylTrailblazeAgent** maps each tool to Revyl operations. -3. **RevylWorkerClient** sends HTTP requests to the Revyl backend (session, device) and to the worker (screenshot, tap, type, swipe, etc.). -4. The worker drives the cloud device (Android/iOS). +2. **RevylTrailblazeAgent** dispatches each tool to **RevylCliClient**. +3. **RevylCliClient** runs the corresponding `revyl device` command via `ProcessBuilder` and parses the JSON output. +4. The `revyl` CLI handles auth, backend proxy routing, and AI-powered target grounding transparently. +5. The cloud device executes the action and returns results. + +## Quick start + +```kotlin +// Only prerequisite: set REVYL_API_KEY in your environment +val client = RevylCliClient() // auto-downloads revyl if not on PATH + +// Start a cloud device with an app installed +val session = client.startSession( + platform = "android", + appUrl = "https://example.com/my-app.apk", +) +println("Viewer: ${session.viewerUrl}") + +// Interact using natural language targets +client.tapTarget("Sign In button") +client.typeText("user@example.com", target = "email field") +client.tapTarget("Log In") + +// Screenshot +client.screenshot("after-login.png") + +// Clean up +client.stopSession() +``` ## MCP server usage -Use **RevylMcpServerFactory** to create an MCP server that provisions a Revyl device and runs the agent: +Use **RevylMcpServerFactory** to create an MCP server backed by Revyl: ```kotlin -val server = RevylMcpServerFactory.create( - backendBaseUrl = System.getenv("REVYL_BACKEND_URL") ?: "https://backend.revyl.ai", - apiKey = System.getenv("REVYL_API_KEY") ?: error("REVYL_API_KEY required"), -) -// Use server with your MCP client +val server = RevylMcpServerFactory.create(platform = "android") +server.startStreamableHttpMcpServer(port = 8080, wait = true) ``` -The factory starts a device session, builds a **RevylMcpBridge** with **RevylTrailblazeAgent**, and returns a **TrailblazeMcpServer** that speaks MCP. +The factory auto-downloads the CLI, provisions a cloud device, and returns a **TrailblazeMcpServer** that speaks MCP. ## Supported operations -| Trailblaze tool | Revyl implementation | -|-------------------|-----------------------------------------------| -| tap | POST /input (tap at coordinates) | -| inputText | POST /input (typeText; optional clear_first) | -| swipe | POST /input (swipe) | -| longPress | POST /input (longPress) | -| launchApp | POST /session/launch_app | -| installApp | POST /session/install_app | -| eraseText | typeText with clear_first + space | -| getScreenState | GET screenshot + minimal hierarchy | - -Screenshots and view hierarchy are provided by the Revyl worker; hierarchy may be minimal compared to a full Maestro tree. +All 12 Trailblaze tools are fully implemented: + +| Trailblaze tool | CLI command | +|-----------------|-------------| +| tap (coordinates) | `revyl device tap --x N --y N` | +| tap (grounded) | `revyl device tap --target "..."` | +| inputText | `revyl device type --text "..." [--target "..."]` | +| swipe | `revyl device swipe --direction ` | +| longPress | `revyl device long-press --target "..."` | +| launchApp | `revyl device launch --bundle-id ` | +| installApp | `revyl device install --app-url ` | +| eraseText | `revyl device clear-text` | +| pressBack | `revyl device back` | +| pressKey | `revyl device key --key ENTER` | +| openUrl | `revyl device navigate --url "..."` | +| screenshot | `revyl device screenshot --out ` | ## Limitations -- No local ADB or Maestro; all device interaction goes through Revyl. -- View hierarchy from Revyl may be reduced (e.g. dimensions only, empty tree). -- Requires network access to Revyl backend and worker. +- No local ADB or Maestro; all device interaction goes through Revyl cloud devices. +- View hierarchy from Revyl is minimal (screenshot-based AI grounding is used instead). +- Requires network access to Revyl backend and GitHub (for auto-download on first use). ## See also - [Architecture](architecture.md) – Revyl as an alternative to HostMaestroTrailblazeAgent. -- [Revyl CLI](https://github.com/revyl/revyl-cli) – Command-line tool for devices and tests. +- [Revyl CLI](https://github.com/RevylAI/revyl-cli) – Command-line tool for devices and tests. +- [Revyl docs](https://docs.revyl.ai) – Full CLI and SDK documentation. diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt new file mode 100644 index 00000000..68241275 --- /dev/null +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt @@ -0,0 +1,395 @@ +package xyz.block.trailblaze.host.revyl + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.int +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import xyz.block.trailblaze.util.Console +import java.io.File +import java.net.URL + +/** + * Device interaction client that delegates to the `revyl` CLI binary. + * + * Every method builds a `revyl device --json` process, + * executes it, and parses the structured JSON output. The CLI handles + * auth, backend proxy routing, and AI-powered target grounding. + * + * If the `revyl` binary is not found on PATH, it is automatically + * downloaded from GitHub Releases to `~/.revyl/bin/revyl`. The only + * prerequisite is setting the `REVYL_API_KEY` environment variable. + * + * @property revylBinaryOverride Explicit path to the revyl binary. + * Defaults to `REVYL_BINARY` env var, then PATH lookup, then + * auto-download. + * @property workingDirectory Optional working directory for CLI + * invocations. Defaults to the JVM's current directory. + */ +class RevylCliClient( + private val revylBinaryOverride: String? = System.getenv("REVYL_BINARY"), + private val workingDirectory: File? = null, +) { + + private val json = Json { ignoreUnknownKeys = true } + private var currentSession: RevylSession? = null + + private var resolvedBinary: String = revylBinaryOverride ?: "revyl" + + init { + ensureRevylInstalled() + } + + /** + * Returns the currently active session, or null if none has been started. + */ + fun getSession(): RevylSession? = currentSession + + // --------------------------------------------------------------------------- + // Auto-install + // --------------------------------------------------------------------------- + + /** + * Ensures the revyl CLI binary is available. If not found on PATH, + * downloads the correct platform binary from GitHub Releases to + * `~/.revyl/bin/revyl` and uses that path for all subsequent calls. + * + * @throws RevylCliException If the platform is unsupported or download fails. + */ + private fun ensureRevylInstalled() { + if (isRevylAvailable()) return + + Console.log("RevylCli: 'revyl' not found on PATH — downloading automatically...") + val (os, arch) = detectPlatform() + val assetName = "revyl-$os-$arch" + if (os == "windows") ".exe" else "" + val downloadUrl = + "https://github.com/RevylAI/revyl-cli/releases/latest/download/$assetName" + + val installDir = File(System.getProperty("user.home"), ".revyl/bin") + installDir.mkdirs() + val binaryName = if (os == "windows") "revyl.exe" else "revyl" + val binaryFile = File(installDir, binaryName) + + try { + Console.log("RevylCli: downloading $downloadUrl") + URL(downloadUrl).openStream().use { input -> + binaryFile.outputStream().use { output -> input.copyTo(output) } + } + binaryFile.setExecutable(true) + resolvedBinary = binaryFile.absolutePath + Console.log("RevylCli: installed to ${binaryFile.absolutePath}") + } catch (e: Exception) { + throw RevylCliException( + "Auto-download failed: ${e.message}. " + + "Install manually: brew install RevylAI/tap/revyl " + + "or download from https://github.com/RevylAI/revyl-cli/releases" + ) + } + + if (!isRevylAvailable()) { + throw RevylCliException( + "Downloaded binary at ${binaryFile.absolutePath} is not executable. " + + "Install manually: brew install RevylAI/tap/revyl" + ) + } + } + + /** + * Checks whether the resolved revyl binary is callable. + * + * @return true if `revyl --version` exits successfully. + */ + private fun isRevylAvailable(): Boolean { + return try { + val process = ProcessBuilder(resolvedBinary, "--version") + .redirectErrorStream(true) + .start() + process.inputStream.bufferedReader().readText() + process.waitFor() == 0 + } catch (_: Exception) { + false + } + } + + /** + * Detects the current OS and CPU architecture for binary selection. + * + * @return Pair of (os, arch) matching GitHub Release asset names. + * @throws RevylCliException If the platform is not supported. + */ + private fun detectPlatform(): Pair { + val osName = System.getProperty("os.name").lowercase() + val os = when { + "mac" in osName || "darwin" in osName -> "darwin" + "linux" in osName -> "linux" + "windows" in osName -> "windows" + else -> throw RevylCliException("Unsupported OS: $osName") + } + val archName = System.getProperty("os.arch").lowercase() + val arch = when (archName) { + "aarch64", "arm64" -> "arm64" + "amd64", "x86_64" -> "amd64" + else -> throw RevylCliException("Unsupported architecture: $archName") + } + return Pair(os, arch) + } + + // --------------------------------------------------------------------------- + // Session lifecycle + // --------------------------------------------------------------------------- + + /** + * Provisions a cloud device by running `revyl device start`. + * + * @param platform "ios" or "android". + * @param appUrl Optional public URL to an .apk/.ipa to install on start. + * @param appLink Optional deep-link to open after launch. + * @return The newly created [RevylSession] parsed from CLI JSON output. + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun startSession( + platform: String, + appUrl: String? = null, + appLink: String? = null, + ): RevylSession { + val args = mutableListOf("device", "start", "--platform", platform.lowercase()) + if (!appUrl.isNullOrBlank()) { + args += listOf("--app-url", appUrl) + } + if (!appLink.isNullOrBlank()) { + args += listOf("--app-link", appLink) + } + + val result = runCli(args) + val obj = json.parseToJsonElement(result).jsonObject + + val session = RevylSession( + index = obj["index"]?.jsonPrimitive?.int ?: 0, + sessionId = obj["session_id"]?.jsonPrimitive?.content ?: "", + workflowRunId = obj["workflow_run_id"]?.jsonPrimitive?.content ?: "", + workerBaseUrl = obj["worker_base_url"]?.jsonPrimitive?.content ?: "", + viewerUrl = obj["viewer_url"]?.jsonPrimitive?.content ?: "", + platform = platform.lowercase(), + ) + currentSession = session + Console.log("RevylCli: device ready (session ${session.index}, ${session.platform})") + Console.log(" Viewer: ${session.viewerUrl}") + return session + } + + /** + * Stops the active device session. + * + * @param index Session index to stop. Defaults to -1 (active session). + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun stopSession(index: Int = -1) { + val args = mutableListOf("device", "stop") + if (index >= 0) args += listOf("-s", index.toString()) + runCli(args) + currentSession = null + } + + // --------------------------------------------------------------------------- + // Device actions + // --------------------------------------------------------------------------- + + /** + * Captures a PNG screenshot and returns the raw bytes. + * + * @param outPath File path to write the screenshot to. + * @return Raw PNG bytes read from the output file. + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun screenshot(outPath: String = createTempScreenshotPath()): ByteArray { + runCli(listOf("device", "screenshot", "--out", outPath)) + val file = File(outPath) + if (!file.exists()) { + throw RevylCliException("Screenshot file not found at $outPath") + } + return file.readBytes() + } + + /** + * Taps at exact pixel coordinates on the device screen. + * + * @param x Horizontal pixel coordinate. + * @param y Vertical pixel coordinate. + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun tap(x: Int, y: Int) { + runCli(listOf("device", "tap", "--x", x.toString(), "--y", y.toString())) + } + + /** + * Taps a UI element identified by natural language description. + * The CLI handles AI-powered grounding to resolve the target to + * exact coordinates transparently. + * + * @param target Natural language description (e.g. "Sign In button"). + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun tapTarget(target: String) { + runCli(listOf("device", "tap", "--target", target)) + } + + /** + * Types text into an input field, optionally targeting a specific element. + * + * @param text The text to type. + * @param target Optional natural language element description to tap first. + * @param clearFirst If true, clears the field before typing. + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun typeText(text: String, target: String? = null, clearFirst: Boolean = false) { + val args = mutableListOf("device", "type", "--text", text) + if (!target.isNullOrBlank()) args += listOf("--target", target) + if (clearFirst) args += "--clear-first" + runCli(args) + } + + /** + * Swipes in the given direction, optionally from a targeted element. + * + * @param direction One of "up", "down", "left", "right". + * @param target Optional natural language element description for swipe origin. + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun swipe(direction: String, target: String? = null) { + val args = mutableListOf("device", "swipe", "--direction", direction) + if (!target.isNullOrBlank()) args += listOf("--target", target) + runCli(args) + } + + /** + * Long-presses a UI element identified by natural language description. + * + * @param target Natural language description of the element. + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun longPress(target: String) { + runCli(listOf("device", "long-press", "--target", target)) + } + + /** + * Presses the Android back button. + * + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun back() { + runCli(listOf("device", "back")) + } + + /** + * Sends a key press event (ENTER or BACKSPACE). + * + * @param key Key name: "ENTER" or "BACKSPACE". + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun pressKey(key: String) { + runCli(listOf("device", "key", "--key", key.uppercase())) + } + + /** + * Opens a URL or deep link on the device. + * + * @param url The URL or deep link to open. + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun navigate(url: String) { + runCli(listOf("device", "navigate", "--url", url)) + } + + /** + * Clears text from the currently focused input field. + * + * @param target Optional natural language element description. + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun clearText(target: String? = null) { + val args = mutableListOf("device", "clear-text") + if (!target.isNullOrBlank()) args += listOf("--target", target) + runCli(args) + } + + /** + * Installs an app on the device from a public URL. + * + * @param appUrl Direct download URL for the .apk or .ipa. + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun installApp(appUrl: String) { + runCli(listOf("device", "install", "--app-url", appUrl)) + } + + /** + * Launches an installed app by its bundle/package ID. + * + * @param bundleId The app's bundle identifier (iOS) or package name (Android). + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun launchApp(bundleId: String) { + runCli(listOf("device", "launch", "--bundle-id", bundleId)) + } + + /** + * Navigates to the device home screen. + * + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun home() { + runCli(listOf("device", "home")) + } + + // --------------------------------------------------------------------------- + // CLI execution + // --------------------------------------------------------------------------- + + /** + * Executes a revyl CLI command with `--json` and returns stdout. + * + * Inherits `REVYL_API_KEY` and other env vars from the parent process. + * On non-zero exit, throws [RevylCliException] with stderr content. + * + * @param args Command arguments after the binary name. + * @return Stdout content from the CLI process. + * @throws RevylCliException If the process exits with a non-zero code. + */ + private fun runCli(args: List): String { + val command = listOf(resolvedBinary) + args + "--json" + Console.log("RevylCli: ${command.joinToString(" ")}") + + val processBuilder = ProcessBuilder(command) + .redirectErrorStream(false) + + if (workingDirectory != null) { + processBuilder.directory(workingDirectory) + } + + val process = processBuilder.start() + val stdout = process.inputStream.bufferedReader().readText() + val stderr = process.errorStream.bufferedReader().readText() + val exitCode = process.waitFor() + + if (exitCode != 0) { + val errorDetail = stderr.ifBlank { stdout } + throw RevylCliException( + "revyl ${args.firstOrNull() ?: ""} ${args.getOrNull(1) ?: ""} " + + "failed (exit $exitCode): ${errorDetail.take(500)}" + ) + } + + return stdout.trim() + } + + private fun createTempScreenshotPath(): String { + val tmpDir = System.getProperty("java.io.tmpdir") + return "$tmpDir/revyl-screenshot-${System.currentTimeMillis()}.png" + } +} + +/** + * Exception thrown when a revyl CLI command fails. + * + * @property message Human-readable description including the exit code and stderr. + */ +class RevylCliException(message: String) : RuntimeException(message) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt index a6a9ce57..b4e9ee6f 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt @@ -6,31 +6,30 @@ import xyz.block.trailblaze.devices.TrailblazeDevicePlatform import xyz.block.trailblaze.devices.TrailblazeDriverType /** - * Provisions and manages Revyl cloud device sessions as an alternative - * to [xyz.block.trailblaze.host.devices.TrailblazeDeviceService] which - * discovers local ADB/iOS devices. + * Provisions and manages Revyl cloud device sessions via the CLI, + * serving as the device-listing layer for [RevylMcpBridge]. * - * @property revylClient The HTTP client used for Revyl API and worker communication. + * @property cliClient CLI-based client for Revyl device interactions. */ class RevylDeviceService( - private val revylClient: RevylWorkerClient, + private val cliClient: RevylCliClient, ) { /** - * Provisions a new cloud device via the Revyl backend. + * Provisions a new cloud device via the Revyl CLI. * * @param platform "ios" or "android". - * @param appUrl Optional direct download URL for an .apk/.ipa. + * @param appUrl Optional public download URL for an .apk/.ipa. * @param appLink Optional deep-link to open after launch. - * @return A summary of the connected device for use with [DeviceManagerToolSet]. - * @throws RevylApiException If provisioning fails. + * @return A summary of the connected device. + * @throws RevylCliException If provisioning fails. */ fun startDevice( platform: String, appUrl: String? = null, appLink: String? = null, ): TrailblazeConnectedDeviceSummary { - val session = revylClient.startSession( + val session = cliClient.startSession( platform = platform, appUrl = appUrl, appLink = appLink, @@ -49,17 +48,17 @@ class RevylDeviceService( } /** - * Stops all active Revyl device sessions managed by this service. + * Stops the active Revyl device session. */ fun stopDevice() { - revylClient.stopSession() + cliClient.stopSession() } /** - * Returns the [TrailblazeDeviceId] for the currently active session, or null if none. + * Returns the [TrailblazeDeviceId] for the currently active session, or null. */ fun getCurrentDeviceId(): TrailblazeDeviceId? { - val session = revylClient.getSession() ?: return null + val session = cliClient.getSession() ?: return null val platform = when (session.platform) { "ios" -> TrailblazeDevicePlatform.IOS else -> TrailblazeDevicePlatform.ANDROID @@ -71,10 +70,10 @@ class RevylDeviceService( } /** - * Returns the set of connected device summaries (at most one for Revyl sessions). + * Returns the set of connected device summaries (at most one for CLI sessions). */ fun listDevices(): Set { - val session = revylClient.getSession() ?: return emptySet() + val session = cliClient.getSession() ?: return emptySet() val driverType = when (session.platform) { "ios" -> TrailblazeDriverType.IOS_HOST else -> TrailblazeDriverType.ANDROID_HOST diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt index 1b63699b..d9863723 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt @@ -3,9 +3,11 @@ package xyz.block.trailblaze.host.revyl import xyz.block.trailblaze.api.ScreenState import xyz.block.trailblaze.devices.TrailblazeConnectedDeviceSummary import xyz.block.trailblaze.devices.TrailblazeDeviceId +import xyz.block.trailblaze.mcp.AgentImplementation import xyz.block.trailblaze.mcp.TrailblazeMcpBridge import xyz.block.trailblaze.model.TrailblazeHostAppTarget import xyz.block.trailblaze.toolcalls.TrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult import xyz.block.trailblaze.toolcalls.getToolNameFromAnnotation import xyz.block.trailblaze.toolcalls.commands.BooleanAssertionTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.StringEvaluationTrailblazeTool @@ -13,19 +15,17 @@ import xyz.block.trailblaze.util.Console import xyz.block.trailblaze.utils.ElementComparator /** - * [TrailblazeMcpBridge] implementation backed by the Revyl cloud device infrastructure. + * [TrailblazeMcpBridge] backed by the Revyl CLI for cloud device interactions. * * Routes MCP tool calls through [RevylTrailblazeAgent] and provides device - * listing / selection via [RevylDeviceService]. This bridge is plugged into - * [xyz.block.trailblaze.logs.server.TrailblazeMcpServer] as a drop-in - * replacement for the local-device [xyz.block.trailblaze.mcp.TrailblazeMcpBridgeImpl]. + * listing/selection via [RevylDeviceService]. * - * @property revylClient The HTTP client for Revyl worker communication. + * @property cliClient CLI-based client for Revyl device interactions. * @property revylDeviceService Handles session provisioning and listing. - * @property agent The standalone Trailblaze agent that dispatches tools to Revyl. + * @property agent The Trailblaze agent that dispatches tools via CLI. */ class RevylMcpBridge( - private val revylClient: RevylWorkerClient, + private val cliClient: RevylCliClient, private val revylDeviceService: RevylDeviceService, private val agent: RevylTrailblazeAgent, ) : TrailblazeMcpBridge { @@ -41,7 +41,6 @@ class RevylMcpBridge( } override suspend fun getInstalledAppIds(): Set { - // Revyl doesn't expose an installed-apps endpoint — return empty for PoC return emptySet() } @@ -49,8 +48,8 @@ class RevylMcpBridge( return setOf(TrailblazeHostAppTarget.DefaultTrailblazeHostAppTarget) } - override suspend fun runYaml(yaml: String, startNewSession: Boolean): String { - Console.log("RevylMcpBridge: runYaml not supported for Revyl cloud devices (use tool calls instead)") + override suspend fun runYaml(yaml: String, startNewSession: Boolean, agentImplementation: AgentImplementation): String { + Console.log("RevylMcpBridge: runYaml not supported for CLI-based Revyl (use tool calls instead)") return "unsupported" } @@ -59,8 +58,8 @@ class RevylMcpBridge( } override suspend fun getCurrentScreenState(): ScreenState? { - val session = revylClient.getSession() ?: return null - return RevylScreenState(revylClient, session.platform) + val session = cliClient.getSession() ?: return null + return RevylScreenState(cliClient, session.platform) } /** @@ -79,16 +78,16 @@ class RevylMcpBridge( ) return when (result.result) { - is xyz.block.trailblaze.toolcalls.TrailblazeToolResult.Success -> + is TrailblazeToolResult.Success -> "Successfully executed $toolName on Revyl cloud device." - is xyz.block.trailblaze.toolcalls.TrailblazeToolResult.Error -> - "Error executing $toolName: ${(result.result as xyz.block.trailblaze.toolcalls.TrailblazeToolResult.Error).errorMessage}" + is TrailblazeToolResult.Error -> + "Error executing $toolName: ${(result.result as TrailblazeToolResult.Error).errorMessage}" } } override suspend fun endSession(): Boolean { return try { - revylClient.stopSession() + cliClient.stopSession() true } catch (_: Exception) { false @@ -104,9 +103,6 @@ class RevylMcpBridge( } } -/** - * No-op [ElementComparator] used when memory-based assertions are not needed. - */ private object NoOpElementComparator : ElementComparator { override fun getElementValue(prompt: String): String? = null override fun evaluateBoolean(statement: String): BooleanAssertionTrailblazeTool = diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt index df0c7741..2cde7e92 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt @@ -7,65 +7,56 @@ import xyz.block.trailblaze.util.Console import java.io.File /** - * Factory for constructing a fully wired [TrailblazeMcpServer] backed by - * the Revyl cloud device infrastructure. + * Factory for constructing a [TrailblazeMcpServer] backed by + * the Revyl CLI for cloud device interactions. * * Usage: * ``` - * val server = RevylMcpServerFactory.create( - * apiKey = System.getenv("REVYL_API_KEY"), - * platform = "android", - * ) + * val server = RevylMcpServerFactory.create(platform = "android") * server.startStreamableHttpMcpServer(port = 8080, wait = true) * ``` + * + * The CLI binary must be on PATH (or set via REVYL_BINARY env var) + * and authenticated via REVYL_API_KEY or `revyl auth login`. */ object RevylMcpServerFactory { /** * Creates a [TrailblazeMcpServer] that provisions a Revyl cloud device - * and routes all MCP tool calls through [RevylTrailblazeAgent]. + * via the CLI and routes all MCP tool calls through [RevylTrailblazeAgent]. * - * @param apiKey Revyl API key (typically from REVYL_API_KEY env var). * @param platform "ios" or "android". - * @param backendUrl Override for the Revyl backend URL. - * @param appUrl Optional direct URL to an .apk/.ipa to install on the device. + * @param appUrl Optional public URL to an .apk/.ipa to install on the device. * @param appLink Optional deep-link to open after launch. * @param trailsDir Directory containing .trail YAML files. * @return A configured [TrailblazeMcpServer] ready to start. - * @throws RevylApiException If device provisioning fails. + * @throws RevylCliException If device provisioning fails. */ fun create( - apiKey: String, platform: String = "android", - backendUrl: String = RevylWorkerClient.DEFAULT_BACKEND_URL, appUrl: String? = null, appLink: String? = null, trailsDir: File = File(System.getProperty("user.dir"), "trails"), ): TrailblazeMcpServer { - Console.log("RevylMcpServerFactory: creating Revyl-backed MCP server") + Console.log("RevylMcpServerFactory: creating CLI-backed MCP server") Console.log(" Platform: $platform") - Console.log(" Backend: $backendUrl") - val revylClient = RevylWorkerClient( - apiKey = apiKey, - backendBaseUrl = backendUrl, - ) + val cliClient = RevylCliClient() - Console.log("RevylMcpServerFactory: provisioning cloud device...") - revylClient.startSession( + Console.log("RevylMcpServerFactory: provisioning cloud device via CLI...") + cliClient.startSession( platform = platform, appUrl = appUrl, appLink = appLink, ) - val session = revylClient.getSession()!! + val session = cliClient.getSession()!! Console.log("RevylMcpServerFactory: device ready") - Console.log(" Worker URL: ${session.workerBaseUrl}") - Console.log(" Viewer URL: ${session.viewerUrl}") + Console.log(" Viewer: ${session.viewerUrl}") - val revylDeviceService = RevylDeviceService(revylClient) - val agent = RevylTrailblazeAgent(revylClient, platform) - val bridge = RevylMcpBridge(revylClient, revylDeviceService, agent) + val revylDeviceService = RevylDeviceService(cliClient) + val agent = RevylTrailblazeAgent(cliClient, platform) + val bridge = RevylMcpBridge(cliClient, revylDeviceService, agent) val logsDir = File(System.getProperty("user.dir"), ".trailblaze/logs") logsDir.mkdirs() @@ -76,6 +67,7 @@ object RevylMcpServerFactory { mcpBridge = bridge, trailsDirProvider = { trailsDir }, targetTestAppProvider = { TrailblazeHostAppTarget.DefaultTrailblazeHostAppTarget }, + llmModelListsProvider = { emptySet() }, ) } } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt index fae329c5..48693ce6 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt @@ -8,22 +8,22 @@ import java.nio.ByteBuffer import java.nio.ByteOrder /** - * [ScreenState] implementation backed by Revyl cloud device screenshots. + * [ScreenState] backed by Revyl CLI screenshots. * * Since Revyl uses AI-powered visual grounding (not accessibility trees), - * the view hierarchy is returned as a minimal empty root node. The LLM agent - * relies on screenshot-based reasoning instead of element trees. + * the view hierarchy is a minimal root node. The LLM agent relies on + * screenshot-based reasoning instead of element trees. * - * @property revylClient Client used to capture screenshots from the Revyl worker. + * @property cliClient CLI client used to capture screenshots. * @property platform The device platform ("ios" or "android"). */ class RevylScreenState( - private val revylClient: RevylWorkerClient, + private val cliClient: RevylCliClient, private val platform: String, ) : ScreenState { private val capturedScreenshot: ByteArray? = try { - revylClient.screenshot() + cliClient.screenshot() } catch (_: Exception) { null } @@ -37,10 +37,6 @@ class RevylScreenState( override val deviceHeight: Int = dimensions.second - /** - * Returns a minimal root node — Revyl does not provide a view hierarchy tree. - * The AI agent should rely on screenshot-based visual grounding instead. - */ override val viewHierarchyOriginal: ViewHierarchyTreeNode = ViewHierarchyTreeNode( nodeId = 1, text = "RevylRootNode", @@ -74,7 +70,6 @@ class RevylScreenState( * @return (width, height) pair, or null if the data is not valid PNG. */ private fun extractPngDimensions(data: ByteArray): Pair? { - // PNG: 8-byte signature + 4-byte IHDR length + 4-byte "IHDR" + 4-byte width + 4-byte height = 24 bytes minimum if (data.size < 24) return null val pngSignature = byteArrayOf( 0x89.toByte(), 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt index 075058be..7307b9e1 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt @@ -29,31 +29,24 @@ import xyz.block.trailblaze.utils.ElementComparator import xyz.block.trailblaze.util.Console /** - * A standalone [TrailblazeAgent] implementation that routes device actions - * directly to the Revyl cloud device worker HTTP API. + * [TrailblazeAgent] implementation that routes all device actions through + * the Revyl CLI binary via [RevylCliClient]. * - * Unlike [xyz.block.trailblaze.MaestroTrailblazeAgent], this agent does NOT - * depend on the Maestro driver stack at all. Each [TrailblazeTool] is - * pattern-matched by type and translated into the corresponding Revyl HTTP - * call in a single hop — no intermediate Maestro Command objects. + * Each [TrailblazeTool] is mapped to the corresponding `revyl device` + * subcommand. The CLI handles auth, backend proxying, and AI-powered + * target grounding transparently. * - * This makes it a clean, minimal integration layer between Trailblaze's - * LLM agent loop and Revyl's cloud device infrastructure. - * - * @property revylClient HTTP client for the Revyl device worker. + * @property cliClient CLI-based client for Revyl device interactions. * @property platform "ios" or "android" — used for ScreenState construction. */ class RevylTrailblazeAgent( - private val revylClient: RevylWorkerClient, + private val cliClient: RevylCliClient, private val platform: String, ) : TrailblazeAgent { /** - * Dispatches a list of [TrailblazeTool]s by mapping each tool's type - * directly to Revyl worker HTTP calls. - * - * Memory tools (assert/remember) are handled in-process via the - * [ElementComparator] — they don't require device interaction. + * Dispatches a list of [TrailblazeTool]s by mapping each tool to + * a `revyl device` CLI command via [RevylCliClient]. * * @param tools Ordered list of tools to execute sequentially. * @param traceId Optional trace ID for log correlation. @@ -73,8 +66,8 @@ class RevylTrailblazeAgent( for (tool in tools) { executed.add(tool) - val result = executeTool(tool, elementComparator, screenStateProvider) - if (result != TrailblazeToolResult.Success) { + val result = executeTool(tool, screenStateProvider) + if (result !is TrailblazeToolResult.Success) { return RunTrailblazeToolsResult( inputTools = tools, executedTools = executed, @@ -86,17 +79,12 @@ class RevylTrailblazeAgent( return RunTrailblazeToolsResult( inputTools = tools, executedTools = executed, - result = TrailblazeToolResult.Success, + result = TrailblazeToolResult.Success(), ) } - // --------------------------------------------------------------------------- - // Tool dispatch — maps each TrailblazeTool to the equivalent Revyl call. - // --------------------------------------------------------------------------- - private fun executeTool( tool: TrailblazeTool, - elementComparator: ElementComparator, screenStateProvider: (() -> ScreenState)?, ): TrailblazeToolResult { val toolName = tool.getToolNameFromAnnotation() @@ -104,142 +92,90 @@ class RevylTrailblazeAgent( return try { when (tool) { - is TapOnPointTrailblazeTool -> handleTapOnPoint(tool) - is InputTextTrailblazeTool -> handleInputText(tool) - is SwipeTrailblazeTool -> handleSwipe(tool) - is LaunchAppTrailblazeTool -> handleLaunchApp(tool) - is EraseTextTrailblazeTool -> handleEraseText() - is HideKeyboardTrailblazeTool -> handleHideKeyboard() - is PressBackTrailblazeTool -> handlePressBack() - is PressKeyTrailblazeTool -> handlePressKey(tool) - is OpenUrlTrailblazeTool -> handleOpenUrl(tool) - is TakeSnapshotTool -> handleTakeSnapshot(tool, screenStateProvider) - is WaitForIdleSyncTrailblazeTool -> handleWaitForIdle(tool) - is ScrollUntilTextIsVisibleTrailblazeTool -> handleScroll(tool) - is NetworkConnectionTrailblazeTool -> handleNetworkConnection(tool) - is TapOnElementByNodeIdTrailblazeTool -> handleTapByNodeId(tool) - is LongPressOnElementWithTextTrailblazeTool -> handleLongPressText(tool) - is ObjectiveStatusTrailblazeTool -> TrailblazeToolResult.Success - is MemoryTrailblazeTool -> { - // Memory tools don't need device interaction - TrailblazeToolResult.Success + is TapOnPointTrailblazeTool -> { + if (tool.longPress) { + cliClient.longPress("element at (${tool.x}, ${tool.y})") + } else { + cliClient.tap(tool.x, tool.y) + } + TrailblazeToolResult.Success() + } + is InputTextTrailblazeTool -> { + cliClient.typeText(tool.text) + TrailblazeToolResult.Success() + } + is SwipeTrailblazeTool -> { + val direction = when (tool.direction) { + SwipeDirection.UP -> "up" + SwipeDirection.DOWN -> "down" + SwipeDirection.LEFT -> "left" + SwipeDirection.RIGHT -> "right" + else -> "down" + } + cliClient.swipe(direction) + TrailblazeToolResult.Success() + } + is LaunchAppTrailblazeTool -> { + cliClient.launchApp(tool.appId) + TrailblazeToolResult.Success() + } + is EraseTextTrailblazeTool -> { + cliClient.clearText() + TrailblazeToolResult.Success() + } + is HideKeyboardTrailblazeTool -> { + TrailblazeToolResult.Success() + } + is PressBackTrailblazeTool -> { + cliClient.back() + TrailblazeToolResult.Success() + } + is PressKeyTrailblazeTool -> { + cliClient.pressKey(tool.keyCode.name) + TrailblazeToolResult.Success() + } + is OpenUrlTrailblazeTool -> { + cliClient.navigate(tool.url) + TrailblazeToolResult.Success() + } + is TakeSnapshotTool -> { + cliClient.screenshot() + TrailblazeToolResult.Success() + } + is WaitForIdleSyncTrailblazeTool -> { + Thread.sleep(1000) + TrailblazeToolResult.Success() } + is ScrollUntilTextIsVisibleTrailblazeTool -> { + cliClient.swipe("down") + TrailblazeToolResult.Success() + } + is NetworkConnectionTrailblazeTool -> { + Console.log("RevylAgent: network toggle not supported on cloud devices") + TrailblazeToolResult.Success() + } + is TapOnElementByNodeIdTrailblazeTool -> { + cliClient.tapTarget("element with node id ${tool.nodeId}") + TrailblazeToolResult.Success() + } + is LongPressOnElementWithTextTrailblazeTool -> { + cliClient.longPress(tool.text) + TrailblazeToolResult.Success() + } + is ObjectiveStatusTrailblazeTool -> TrailblazeToolResult.Success() + is MemoryTrailblazeTool -> TrailblazeToolResult.Success() else -> { - Console.log("RevylAgent: unsupported tool type ${tool::class.simpleName} — skipping") - TrailblazeToolResult.Success + Console.log("RevylAgent: unsupported tool ${tool::class.simpleName}") + TrailblazeToolResult.Success() } } } catch (e: Exception) { Console.error("RevylAgent: tool '$toolName' failed: ${e.message}") TrailblazeToolResult.Error.ExceptionThrown( - errorMessage = "Revyl execution failed for '$toolName': ${e.message}", + errorMessage = "CLI execution failed for '$toolName': ${e.message}", command = tool, stackTrace = e.stackTraceToString(), ) } } - - // --------------------------------------------------------------------------- - // Individual tool handlers - // --------------------------------------------------------------------------- - - private fun handleTapOnPoint(tool: TapOnPointTrailblazeTool): TrailblazeToolResult { - if (tool.longPress) { - Console.log("RevylAgent: long-press at (${tool.x}, ${tool.y})") - revylClient.longPress(tool.x, tool.y) - } else { - Console.log("RevylAgent: tap at (${tool.x}, ${tool.y})") - revylClient.tap(tool.x, tool.y) - } - return TrailblazeToolResult.Success - } - - private fun handleInputText(tool: InputTextTrailblazeTool): TrailblazeToolResult { - Console.log("RevylAgent: type text '${tool.text}'") - revylClient.typeText(tool.text) - return TrailblazeToolResult.Success - } - - private fun handleSwipe(tool: SwipeTrailblazeTool): TrailblazeToolResult { - val direction = when (tool.direction) { - SwipeDirection.UP -> "up" - SwipeDirection.DOWN -> "down" - SwipeDirection.LEFT -> "left" - SwipeDirection.RIGHT -> "right" - else -> "down" - } - Console.log("RevylAgent: swipe $direction") - revylClient.swipe(direction) - return TrailblazeToolResult.Success - } - - private fun handleLaunchApp(tool: LaunchAppTrailblazeTool): TrailblazeToolResult { - Console.log("RevylAgent: launch app '${tool.appId}'") - revylClient.launchApp(tool.appId) - return TrailblazeToolResult.Success - } - - private fun handleEraseText(): TrailblazeToolResult { - Console.log("RevylAgent: erase text (clear_first + space via /input)") - revylClient.typeText(text = " ", clearFirst = true) - return TrailblazeToolResult.Success - } - - private fun handleHideKeyboard(): TrailblazeToolResult { - Console.log("RevylAgent: hide keyboard (no-op for cloud device)") - return TrailblazeToolResult.Success - } - - private fun handlePressBack(): TrailblazeToolResult { - Console.log("RevylAgent: press back (not directly supported — skipping)") - return TrailblazeToolResult.Success - } - - private fun handlePressKey(tool: PressKeyTrailblazeTool): TrailblazeToolResult { - Console.log("RevylAgent: press key '${tool.keyCode}' (not directly supported — skipping)") - return TrailblazeToolResult.Success - } - - private fun handleOpenUrl(tool: OpenUrlTrailblazeTool): TrailblazeToolResult { - Console.log("RevylAgent: open URL '${tool.url}' (not directly supported — skipping)") - return TrailblazeToolResult.Success - } - - private fun handleTakeSnapshot( - tool: TakeSnapshotTool, - screenStateProvider: (() -> ScreenState)?, - ): TrailblazeToolResult { - Console.log("RevylAgent: take snapshot '${tool.screenName}'") - // Capture screenshot from Revyl for the snapshot - revylClient.screenshot() - return TrailblazeToolResult.Success - } - - private fun handleWaitForIdle(tool: WaitForIdleSyncTrailblazeTool): TrailblazeToolResult { - Console.log("RevylAgent: wait for idle — sleeping briefly") - Thread.sleep(1000) - return TrailblazeToolResult.Success - } - - private fun handleScroll(tool: ScrollUntilTextIsVisibleTrailblazeTool): TrailblazeToolResult { - Console.log("RevylAgent: scroll until text visible — performing swipe down") - revylClient.swipe("down") - return TrailblazeToolResult.Success - } - - private fun handleNetworkConnection(tool: NetworkConnectionTrailblazeTool): TrailblazeToolResult { - Console.log("RevylAgent: network connection toggle (not supported — skipping)") - return TrailblazeToolResult.Success - } - - private fun handleTapByNodeId(tool: TapOnElementByNodeIdTrailblazeTool): TrailblazeToolResult { - Console.log("RevylAgent: tap by nodeId ${tool.nodeId} (not directly supported — skipping)") - return TrailblazeToolResult.Success - } - - private fun handleLongPressText(tool: LongPressOnElementWithTextTrailblazeTool): TrailblazeToolResult { - Console.log("RevylAgent: long press on element with text '${tool.text}'") - revylClient.tapTarget(tool.text) - return TrailblazeToolResult.Success - } } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylWorkerClient.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylWorkerClient.kt deleted file mode 100644 index 38188f40..00000000 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylWorkerClient.kt +++ /dev/null @@ -1,393 +0,0 @@ -package xyz.block.trailblaze.host.revyl - -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.buildJsonObject -import kotlinx.serialization.json.int -import kotlinx.serialization.json.jsonObject -import kotlinx.serialization.json.jsonPrimitive -import kotlinx.serialization.json.put -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.RequestBody.Companion.toRequestBody -import java.io.IOException -import java.util.concurrent.TimeUnit - -/** - * HTTP client that communicates with a Revyl cloud device worker - * and the Revyl backend for session provisioning and AI-powered target resolution. - * - * Uses the same HTTP endpoints as the revyl-cli Go implementation - * (see revyl-cli/internal/mcp/device_session.go). - * - * @property apiKey Revyl API key for backend authentication. - * @property backendBaseUrl Base URL for the Revyl backend API. - */ -class RevylWorkerClient( - private val apiKey: String, - private val backendBaseUrl: String = DEFAULT_BACKEND_URL, -) { - - private val json = Json { ignoreUnknownKeys = true } - - private val httpClient = OkHttpClient.Builder() - .connectTimeout(30, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.SECONDS) - .writeTimeout(30, TimeUnit.SECONDS) - .build() - - private var currentSession: RevylSession? = null - - /** - * Returns the currently active session, or null if none is provisioned. - */ - fun getSession(): RevylSession? = currentSession - - // --------------------------------------------------------------------------- - // Session lifecycle - // --------------------------------------------------------------------------- - - /** - * Provisions a new cloud-hosted device and polls until the worker is reachable. - * - * @param platform "ios" or "android". - * @param appUrl Optional direct URL to an .apk/.ipa to install. - * @param appLink Optional deep-link to open after launch. - * @return The newly created [RevylSession]. - * @throws RevylApiException If provisioning fails or the worker never becomes ready. - */ - fun startSession( - platform: String, - appUrl: String? = null, - appLink: String? = null, - ): RevylSession { - val body = buildJsonObject { - put("platform", platform.lowercase()) - put("is_simulation", true) - if (!appUrl.isNullOrBlank()) put("app_url", appUrl) - if (!appLink.isNullOrBlank()) put("app_link", appLink) - } - - val response = backendPost("/api/v1/execution/start-device", body) - val workflowRunId = response["workflow_run_id"]?.jsonPrimitive?.content ?: "" - if (workflowRunId.isBlank()) { - val error = response["error"]?.jsonPrimitive?.content ?: "unknown error" - throw RevylApiException("Failed to start device: $error") - } - - val workerBaseUrl = pollForWorkerUrl(workflowRunId, maxWaitSeconds = 120) - waitForDeviceReady(workerBaseUrl, maxWaitSeconds = 30) - - val viewerUrl = "$backendBaseUrl/tests/execute?workflowRunId=$workflowRunId&platform=$platform" - - val session = RevylSession( - index = 0, - sessionId = workflowRunId, - workflowRunId = workflowRunId, - workerBaseUrl = workerBaseUrl, - viewerUrl = viewerUrl, - platform = platform.lowercase(), - ) - currentSession = session - return session - } - - /** - * Stops (cancels) the current device session and releases cloud resources. - * - * @throws RevylApiException If the cancellation request fails. - */ - fun stopSession() { - val session = currentSession ?: return - backendPost( - "/cancel-device", - buildJsonObject { put("workflow_run_id", session.workflowRunId) }, - ) - currentSession = null - } - - // --------------------------------------------------------------------------- - // Device actions — delegated to the worker HTTP API - // --------------------------------------------------------------------------- - - /** - * Captures a PNG screenshot of the current device screen. - * - * @return Raw PNG bytes. - * @throws RevylApiException If no session is active or the request fails. - */ - fun screenshot(): ByteArray { - val session = requireSession() - val request = Request.Builder() - .url("${session.workerBaseUrl}/screenshot") - .get() - .build() - - val response = httpClient.newCall(request).execute() - if (!response.isSuccessful) { - throw RevylApiException("Screenshot failed: HTTP ${response.code}") - } - return response.body?.bytes() ?: throw RevylApiException("Screenshot returned empty body") - } - - /** - * Taps at the given coordinates on the device screen. - * - * @param x Horizontal pixel coordinate. - * @param y Vertical pixel coordinate. - * @throws RevylApiException If the request fails. - */ - fun tap(x: Int, y: Int) { - workerPost("/tap", buildJsonObject { put("x", x); put("y", y) }) - } - - /** - * Taps a UI element identified by a natural language description - * using the Revyl AI grounding model. - * - * @param target Natural language description (e.g. "Sign In button"). - * @throws RevylApiException If grounding or the tap fails. - */ - fun tapTarget(target: String) { - val (x, y) = resolveTarget(target) - tap(x, y) - } - - /** - * Types text into the currently focused input field or a targeted element. - * - * @param text The text to type. - * @param targetX Optional x coordinate to tap before typing. - * @param targetY Optional y coordinate to tap before typing. - * @param clearFirst If true, clears the field content before typing. - * @throws RevylApiException If the request fails. - */ - fun typeText(text: String, targetX: Int? = null, targetY: Int? = null, clearFirst: Boolean = false) { - val body = buildJsonObject { - put("text", text) - if (targetX != null && targetY != null) { - put("x", targetX) - put("y", targetY) - } - if (clearFirst) { - put("clear_first", true) - } - } - workerPost("/input", body) - } - - /** - * Swipes in the given direction from the center or a specific point. - * - * @param direction One of "up", "down", "left", "right". - * @param startX Optional starting x coordinate. - * @param startY Optional starting y coordinate. - * @throws RevylApiException If the request fails. - */ - fun swipe(direction: String, startX: Int? = null, startY: Int? = null) { - val body = buildJsonObject { - put("direction", direction) - if (startX != null) put("x", startX) - if (startY != null) put("y", startY) - } - workerPost("/swipe", body) - } - - /** - * Long-presses at the given coordinates. - * - * @param x Horizontal pixel coordinate. - * @param y Vertical pixel coordinate. - * @param durationMs Duration of the press in milliseconds. - * @throws RevylApiException If the request fails. - */ - fun longPress(x: Int, y: Int, durationMs: Int = 1500) { - val body = buildJsonObject { - put("x", x) - put("y", y) - put("duration", durationMs) - } - workerPost("/longpress", body) - } - - /** - * Installs an app from a URL onto the device. - * - * @param appUrl Direct download URL for the .apk or .ipa. - * @throws RevylApiException If the request fails. - */ - fun installApp(appUrl: String) { - workerPost("/install", buildJsonObject { put("app_url", appUrl) }) - } - - /** - * Launches an installed app by its bundle/package ID. - * - * @param bundleId The app's bundle identifier (iOS) or package name (Android). - * @throws RevylApiException If the request fails. - */ - fun launchApp(bundleId: String) { - workerPost("/launch", buildJsonObject { put("bundle_id", bundleId) }) - } - - /** - * Resolves a natural language target description to screen coordinates - * using the Revyl AI grounding model on the worker. - * - * Falls back to the backend grounding endpoint if the worker doesn't support - * native target resolution. - * - * @param target Natural language element description (e.g. "the search bar"). - * @return Pair of (x, y) pixel coordinates. - * @throws RevylApiException If grounding fails. - */ - fun resolveTarget(target: String): Pair { - val session = requireSession() - - // Try worker-native grounding first - try { - val body = buildJsonObject { put("target", target) } - val responseBody = workerPostRaw("/resolve_target", body) - val parsed = json.parseToJsonElement(responseBody).jsonObject - val x = parsed["x"]!!.jsonPrimitive.int - val y = parsed["y"]!!.jsonPrimitive.int - return Pair(x, y) - } catch (_: Exception) { - // Fall back to backend grounding - } - - // Backend grounding fallback: send screenshot + target to the backend - val screenshotBytes = screenshot() - val base64Screenshot = java.util.Base64.getEncoder().encodeToString(screenshotBytes) - val groundBody = buildJsonObject { - put("screenshot_base64", base64Screenshot) - put("target", target) - put("workflow_run_id", session.workflowRunId) - } - - val response = backendPost("/api/v1/execution/ground-element", groundBody) - val x = response["x"]!!.jsonPrimitive.int - val y = response["y"]!!.jsonPrimitive.int - return Pair(x, y) - } - - // --------------------------------------------------------------------------- - // Internal helpers - // --------------------------------------------------------------------------- - - private fun requireSession(): RevylSession = - currentSession ?: throw RevylApiException("No active device session. Call startSession() first.") - - private fun backendPost(path: String, body: JsonObject): JsonObject { - val mediaType = "application/json; charset=utf-8".toMediaType() - val request = Request.Builder() - .url("$backendBaseUrl$path") - .addHeader("X-API-Key", apiKey) - .addHeader("Content-Type", "application/json") - .post(body.toString().toRequestBody(mediaType)) - .build() - - val response = httpClient.newCall(request).execute() - val responseBody = response.body?.string() ?: "{}" - if (!response.isSuccessful) { - throw RevylApiException("Backend request to $path failed (HTTP ${response.code}): $responseBody") - } - return json.parseToJsonElement(responseBody).jsonObject - } - - private fun workerPost(path: String, body: JsonObject) { - val responseBody = workerPostRaw(path, body) - if (responseBody.isNotBlank()) { - try { - val parsed = json.parseToJsonElement(responseBody).jsonObject - if (parsed.containsKey("error")) { - throw RevylApiException("Worker action $path failed: ${parsed["error"]!!.jsonPrimitive.content}") - } - } catch (_: kotlinx.serialization.SerializationException) { - // Non-JSON response is fine for success - } - } - } - - private fun workerPostRaw(path: String, body: JsonObject): String { - val session = requireSession() - val mediaType = "application/json; charset=utf-8".toMediaType() - val request = Request.Builder() - .url("${session.workerBaseUrl}$path") - .addHeader("Content-Type", "application/json") - .post(body.toString().toRequestBody(mediaType)) - .build() - - val response = httpClient.newCall(request).execute() - val responseBody = response.body?.string() ?: "" - if (!response.isSuccessful) { - throw RevylApiException("Worker request to $path failed (HTTP ${response.code}): $responseBody") - } - return responseBody - } - - /** - * Polls the backend until a worker URL is available for the given workflow run. - */ - private fun pollForWorkerUrl(workflowRunId: String, maxWaitSeconds: Int): String { - val deadline = System.currentTimeMillis() + (maxWaitSeconds * 1000L) - while (System.currentTimeMillis() < deadline) { - try { - val request = Request.Builder() - .url("$backendBaseUrl/api/v1/execution/worker-ws-url/$workflowRunId") - .addHeader("X-API-Key", apiKey) - .get() - .build() - - val response = httpClient.newCall(request).execute() - if (response.isSuccessful) { - val body = response.body?.string() ?: "" - val parsed = json.parseToJsonElement(body).jsonObject - val workerUrl = parsed["worker_url"]?.jsonPrimitive?.content ?: "" - if (workerUrl.isNotBlank()) { - return workerUrl.trimEnd('/') - } - } - } catch (_: IOException) { - // Retry - } - Thread.sleep(2000) - } - throw RevylApiException("Worker URL not available after ${maxWaitSeconds}s for workflow $workflowRunId") - } - - /** - * Waits until the worker's health endpoint reports a connected device. - */ - private fun waitForDeviceReady(workerBaseUrl: String, maxWaitSeconds: Int) { - val deadline = System.currentTimeMillis() + (maxWaitSeconds * 1000L) - while (System.currentTimeMillis() < deadline) { - try { - val request = Request.Builder() - .url("$workerBaseUrl/health") - .get() - .build() - - val response = httpClient.newCall(request).execute() - if (response.isSuccessful) { - return - } - } catch (_: IOException) { - // Retry - } - Thread.sleep(2000) - } - } - - companion object { - const val DEFAULT_BACKEND_URL = "https://backend.revyl.ai" - } -} - -/** - * Exception thrown when a Revyl API or worker request fails. - * - * @property message Human-readable description of the failure. - */ -class RevylApiException(message: String) : RuntimeException(message) diff --git a/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt b/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt new file mode 100644 index 00000000..e066379d --- /dev/null +++ b/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt @@ -0,0 +1,95 @@ +package xyz.block.trailblaze.host.revyl + +/** + * Standalone demo that drives a Revyl cloud device through a Bug Bazaar + * e-commerce flow using [RevylCliClient] — the same integration layer + * that [RevylTrailblazeAgent] uses for all tool execution. + * + * Each step prints the equivalent `revyl device` CLI command so the + * mapping between Trailblaze Kotlin code and the Revyl CLI is visible. + * + * Usage: + * ./gradlew :trailblaze-host:run -PmainClass=xyz.block.trailblaze.host.revyl.RevylDemoKt + */ + +private const val BUG_BAZAAR_APK = + "https://pub-b03f222a53c447c18ef5f8d365a2f00e.r2.dev/bug-bazaar/bug-bazaar-preview.apk" + +private const val BUG_BAZAAR_IOS = + "https://pub-b03f222a53c447c18ef5f8d365a2f00e.r2.dev/bug-bazaar/bug-bazaar-preview-simulator.tar.gz" + +private const val BUNDLE_ID = "com.bugbazaar.app" + +fun main() { + val client = RevylCliClient() + + println("\n=== Trailblaze x Revyl Demo ===") + println("Each step shows the Kotlin call AND the equivalent CLI command.\n") + + // ── Step 0: Provision device + install app ───────────────────────── + // CLI: revyl device start --platform android --app-url --open --json + println("Step 0: Start device + install Bug Bazaar") + val session = client.startSession( + platform = "android", + appUrl = BUG_BAZAAR_APK, + ) + println(" Viewer: ${session.viewerUrl}") + + // CLI: revyl device launch --bundle-id com.bugbazaar.app --json + println("\nStep 0b: Launch app") + client.launchApp(BUNDLE_ID) + Thread.sleep(2000) + + // ── Step 1: Screenshot home ──────────────────────────────────────── + // CLI: revyl device screenshot --out flow-01-home.png --json + println("\nStep 1: Screenshot home screen") + client.screenshot("flow-01-home.png") + + // ── Step 2: Navigate to search ───────────────────────────────────── + // CLI: revyl device tap --target "Search tab" --json + println("\nStep 2: Tap Search tab") + client.tapTarget("Search tab") + Thread.sleep(1000) + + // ── Step 3: Search for "beetle" ──────────────────────────────────── + // CLI: revyl device type --target "search input field" --text "beetle" --json + println("\nStep 3: Type 'beetle' in search field") + client.typeText("beetle", target = "search input field") + Thread.sleep(1000) + // CLI: revyl device screenshot --out flow-02-search.png --json + client.screenshot("flow-02-search.png") + + // ── Step 4: Open product detail ──────────────────────────────────── + // CLI: revyl device tap --target "Hercules Beetle" --json + println("\nStep 4: Tap Hercules Beetle result") + client.tapTarget("Hercules Beetle") + Thread.sleep(1000) + // CLI: revyl device screenshot --out flow-03-product.png --json + client.screenshot("flow-03-product.png") + + // ── Step 5: Add to cart ──────────────────────────────────────────── + // CLI: revyl device tap --target "Add to Cart button" --json + println("\nStep 5: Tap Add to Cart") + client.tapTarget("Add to Cart button") + Thread.sleep(1000) + + // ── Step 6: Back to home ─────────────────────────────────────────── + // CLI: revyl device back --json + println("\nStep 6: Navigate back to home") + client.back() + client.back() + Thread.sleep(1000) + // CLI: revyl device screenshot --out flow-04-done.png --json + client.screenshot("flow-04-done.png") + + // ── Done ─────────────────────────────────────────────────────────── + println("\n=== Demo complete ===") + println("Session viewer: ${session.viewerUrl}") + println("Screenshots: flow-01-home.png … flow-04-done.png") + println("\nGet session report:") + println(" CLI: revyl device report --json") + println(" Kotlin: // report data available via session.viewerUrl") + println("\nStop device:") + println(" CLI: revyl device stop") + println(" Kotlin: client.stopSession()") +} From 07a0d149eca0fc8cd14ee4ba4d76b824cf049a0b Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Sun, 22 Mar 2026 18:27:13 -0700 Subject: [PATCH 03/16] =?UTF-8?q?feat:=20Revyl=20integration=20parity=20?= =?UTF-8?q?=E2=80=94=20runYaml,=20blaze=20mode,=20network=20toggle,=20mult?= =?UTF-8?q?i-session?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - RevylMcpBridge: implement runYaml() to parse trail YAML and replay tools; add blazeExecute() for V3 AI exploration via BlazeGoalPlanner - RevylCliClient: refactor from single session to multi-session map with activeSessionIndex; pass -s flag on all device commands; add setNetworkConnected() for airplane mode toggle - RevylTrailblazeAgent: wire network toggle to CLI client - RevylDeviceService: support listing/stopping multiple sessions - RevylMcpServerFactory: add multi-platform create() overload - Fix selectDevice to fail fast on missing session instead of silently continuing on wrong device Made-with: Cursor Signed-off-by: Anam Hira --- .../trailblaze/host/revyl/RevylCliClient.kt | 113 ++++++++++++++---- .../host/revyl/RevylDeviceService.kt | 68 ++++++----- .../trailblaze/host/revyl/RevylMcpBridge.kt | 110 ++++++++++++++++- .../host/revyl/RevylMcpServerFactory.kt | 64 +++++++--- .../host/revyl/RevylTrailblazeAgent.kt | 2 +- 5 files changed, 287 insertions(+), 70 deletions(-) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt index 68241275..6bbd4bda 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt @@ -31,7 +31,8 @@ class RevylCliClient( ) { private val json = Json { ignoreUnknownKeys = true } - private var currentSession: RevylSession? = null + private val sessions = mutableMapOf() + private var activeSessionIndex: Int = 0 private var resolvedBinary: String = revylBinaryOverride ?: "revyl" @@ -42,7 +43,40 @@ class RevylCliClient( /** * Returns the currently active session, or null if none has been started. */ - fun getSession(): RevylSession? = currentSession + fun getActiveSession(): RevylSession? = sessions[activeSessionIndex] + + /** + * Returns the session at the given index, or null if not found. + * + * @param index Session index to retrieve. + */ + fun getSession(index: Int): RevylSession? = sessions[index] + + /** + * Finds a session by its workflow run ID. + * + * @param workflowRunId The Hatchet workflow run identifier. + * @return The matching session, or null if not found. + */ + fun getSession(workflowRunId: String): RevylSession? = + sessions.values.firstOrNull { it.workflowRunId == workflowRunId } + + /** + * Returns all active sessions. + */ + fun getAllSessions(): Map = sessions.toMap() + + /** + * Switches the active session to the given index. + * + * @param index Session index to make active. + * @throws IllegalArgumentException If no session exists at the given index. + */ + fun useSession(index: Int) { + require(sessions.containsKey(index)) { "No session at index $index. Active sessions: ${sessions.keys}" } + activeSessionIndex = index + Console.log("RevylCli: switched to session $index (${sessions[index]!!.platform})") + } // --------------------------------------------------------------------------- // Auto-install @@ -170,29 +204,57 @@ class RevylCliClient( viewerUrl = obj["viewer_url"]?.jsonPrimitive?.content ?: "", platform = platform.lowercase(), ) - currentSession = session + sessions[session.index] = session + activeSessionIndex = session.index Console.log("RevylCli: device ready (session ${session.index}, ${session.platform})") Console.log(" Viewer: ${session.viewerUrl}") return session } /** - * Stops the active device session. + * Stops a device session and removes it from the local session map. * * @param index Session index to stop. Defaults to -1 (active session). * @throws RevylCliException If the CLI exits with a non-zero code. */ fun stopSession(index: Int = -1) { + val targetIndex = if (index >= 0) index else activeSessionIndex val args = mutableListOf("device", "stop") - if (index >= 0) args += listOf("-s", index.toString()) + if (targetIndex >= 0) args += listOf("-s", targetIndex.toString()) runCli(args) - currentSession = null + sessions.remove(targetIndex) + if (activeSessionIndex == targetIndex && sessions.isNotEmpty()) { + activeSessionIndex = sessions.keys.first() + } + } + + /** + * Stops all active device sessions. + * + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun stopAllSessions() { + runCli(listOf("device", "stop", "--all")) + sessions.clear() } // --------------------------------------------------------------------------- // Device actions // --------------------------------------------------------------------------- + /** + * Builds session-scoped CLI args by prepending `-s ` + * to device commands so each action targets the correct session. + */ + private fun deviceArgs(vararg args: String): List { + val base = mutableListOf("device") + if (sessions.size > 1) { + base += listOf("-s", activeSessionIndex.toString()) + } + base += args.toList() + return base + } + /** * Captures a PNG screenshot and returns the raw bytes. * @@ -201,7 +263,7 @@ class RevylCliClient( * @throws RevylCliException If the CLI exits with a non-zero code. */ fun screenshot(outPath: String = createTempScreenshotPath()): ByteArray { - runCli(listOf("device", "screenshot", "--out", outPath)) + runCli(deviceArgs("screenshot", "--out", outPath)) val file = File(outPath) if (!file.exists()) { throw RevylCliException("Screenshot file not found at $outPath") @@ -217,19 +279,17 @@ class RevylCliClient( * @throws RevylCliException If the CLI exits with a non-zero code. */ fun tap(x: Int, y: Int) { - runCli(listOf("device", "tap", "--x", x.toString(), "--y", y.toString())) + runCli(deviceArgs("tap", "--x", x.toString(), "--y", y.toString())) } /** * Taps a UI element identified by natural language description. - * The CLI handles AI-powered grounding to resolve the target to - * exact coordinates transparently. * * @param target Natural language description (e.g. "Sign In button"). * @throws RevylCliException If the CLI exits with a non-zero code. */ fun tapTarget(target: String) { - runCli(listOf("device", "tap", "--target", target)) + runCli(deviceArgs("tap", "--target", target)) } /** @@ -241,7 +301,7 @@ class RevylCliClient( * @throws RevylCliException If the CLI exits with a non-zero code. */ fun typeText(text: String, target: String? = null, clearFirst: Boolean = false) { - val args = mutableListOf("device", "type", "--text", text) + val args = deviceArgs("type", "--text", text).toMutableList() if (!target.isNullOrBlank()) args += listOf("--target", target) if (clearFirst) args += "--clear-first" runCli(args) @@ -255,7 +315,7 @@ class RevylCliClient( * @throws RevylCliException If the CLI exits with a non-zero code. */ fun swipe(direction: String, target: String? = null) { - val args = mutableListOf("device", "swipe", "--direction", direction) + val args = deviceArgs("swipe", "--direction", direction).toMutableList() if (!target.isNullOrBlank()) args += listOf("--target", target) runCli(args) } @@ -267,7 +327,7 @@ class RevylCliClient( * @throws RevylCliException If the CLI exits with a non-zero code. */ fun longPress(target: String) { - runCli(listOf("device", "long-press", "--target", target)) + runCli(deviceArgs("long-press", "--target", target)) } /** @@ -276,7 +336,7 @@ class RevylCliClient( * @throws RevylCliException If the CLI exits with a non-zero code. */ fun back() { - runCli(listOf("device", "back")) + runCli(deviceArgs("back")) } /** @@ -286,7 +346,7 @@ class RevylCliClient( * @throws RevylCliException If the CLI exits with a non-zero code. */ fun pressKey(key: String) { - runCli(listOf("device", "key", "--key", key.uppercase())) + runCli(deviceArgs("key", "--key", key.uppercase())) } /** @@ -296,7 +356,7 @@ class RevylCliClient( * @throws RevylCliException If the CLI exits with a non-zero code. */ fun navigate(url: String) { - runCli(listOf("device", "navigate", "--url", url)) + runCli(deviceArgs("navigate", "--url", url)) } /** @@ -306,7 +366,7 @@ class RevylCliClient( * @throws RevylCliException If the CLI exits with a non-zero code. */ fun clearText(target: String? = null) { - val args = mutableListOf("device", "clear-text") + val args = deviceArgs("clear-text").toMutableList() if (!target.isNullOrBlank()) args += listOf("--target", target) runCli(args) } @@ -318,7 +378,7 @@ class RevylCliClient( * @throws RevylCliException If the CLI exits with a non-zero code. */ fun installApp(appUrl: String) { - runCli(listOf("device", "install", "--app-url", appUrl)) + runCli(deviceArgs("install", "--app-url", appUrl)) } /** @@ -328,7 +388,7 @@ class RevylCliClient( * @throws RevylCliException If the CLI exits with a non-zero code. */ fun launchApp(bundleId: String) { - runCli(listOf("device", "launch", "--bundle-id", bundleId)) + runCli(deviceArgs("launch", "--bundle-id", bundleId)) } /** @@ -337,7 +397,18 @@ class RevylCliClient( * @throws RevylCliException If the CLI exits with a non-zero code. */ fun home() { - runCli(listOf("device", "home")) + runCli(deviceArgs("home")) + } + + /** + * Toggles device network connectivity (airplane mode). + * + * @param connected true to enable network (disable airplane mode), + * false to disable network (enable airplane mode). + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun setNetworkConnected(connected: Boolean) { + runCli(deviceArgs("network", if (connected) "--connected" else "--disconnected")) } // --------------------------------------------------------------------------- diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt index b4e9ee6f..68e45ea8 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt @@ -9,6 +9,9 @@ import xyz.block.trailblaze.devices.TrailblazeDriverType * Provisions and manages Revyl cloud device sessions via the CLI, * serving as the device-listing layer for [RevylMcpBridge]. * + * Supports multiple concurrent sessions -- each session is tracked + * in the underlying [RevylCliClient] session map. + * * @property cliClient CLI-based client for Revyl device interactions. */ class RevylDeviceService( @@ -17,6 +20,8 @@ class RevylDeviceService( /** * Provisions a new cloud device via the Revyl CLI. + * The new session is added to the client's session map without + * replacing any existing sessions. * * @param platform "ios" or "android". * @param appUrl Optional public download URL for an .apk/.ipa. @@ -35,55 +40,60 @@ class RevylDeviceService( appLink = appLink, ) - val driverType = when (session.platform) { - "ios" -> TrailblazeDriverType.IOS_HOST - else -> TrailblazeDriverType.ANDROID_HOST - } + return sessionToSummary(session) + } - return TrailblazeConnectedDeviceSummary( - trailblazeDriverType = driverType, - instanceId = session.workflowRunId, - description = "Revyl cloud ${session.platform} device (${session.viewerUrl})", - ) + /** + * Stops a device session by index. + * + * @param index Session index to stop. Defaults to -1 (active session). + */ + fun stopDevice(index: Int = -1) { + cliClient.stopSession(index) } /** - * Stops the active Revyl device session. + * Stops all active device sessions. */ - fun stopDevice() { - cliClient.stopSession() + fun stopAllDevices() { + cliClient.stopAllSessions() } /** * Returns the [TrailblazeDeviceId] for the currently active session, or null. */ fun getCurrentDeviceId(): TrailblazeDeviceId? { - val session = cliClient.getSession() ?: return null - val platform = when (session.platform) { - "ios" -> TrailblazeDevicePlatform.IOS - else -> TrailblazeDevicePlatform.ANDROID - } - return TrailblazeDeviceId( - instanceId = session.workflowRunId, - trailblazeDevicePlatform = platform, - ) + val session = cliClient.getActiveSession() ?: return null + return sessionToDeviceId(session) } /** - * Returns the set of connected device summaries (at most one for CLI sessions). + * Returns summaries for all active device sessions. */ fun listDevices(): Set { - val session = cliClient.getSession() ?: return emptySet() + return cliClient.getAllSessions().values.map { sessionToSummary(it) }.toSet() + } + + private fun sessionToSummary(session: RevylSession): TrailblazeConnectedDeviceSummary { val driverType = when (session.platform) { "ios" -> TrailblazeDriverType.IOS_HOST else -> TrailblazeDriverType.ANDROID_HOST } - return setOf( - TrailblazeConnectedDeviceSummary( - trailblazeDriverType = driverType, - instanceId = session.workflowRunId, - description = "Revyl cloud ${session.platform} device", - ), + return TrailblazeConnectedDeviceSummary( + trailblazeDriverType = driverType, + instanceId = session.workflowRunId, + description = "Revyl cloud ${session.platform} device (session ${session.index})", + ) + } + + private fun sessionToDeviceId(session: RevylSession): TrailblazeDeviceId { + val platform = when (session.platform) { + "ios" -> TrailblazeDevicePlatform.IOS + else -> TrailblazeDevicePlatform.ANDROID + } + return TrailblazeDeviceId( + instanceId = session.workflowRunId, + trailblazeDevicePlatform = platform, ) } } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt index d9863723..3d8e511e 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt @@ -1,5 +1,10 @@ package xyz.block.trailblaze.host.revyl +import xyz.block.trailblaze.agent.AgentUiActionExecutor +import xyz.block.trailblaze.agent.BlazeConfig +import xyz.block.trailblaze.agent.blaze.BlazeGoalPlanner +import xyz.block.trailblaze.agent.blaze.BlazeState +import xyz.block.trailblaze.agent.blaze.ScreenAnalyzer import xyz.block.trailblaze.api.ScreenState import xyz.block.trailblaze.devices.TrailblazeConnectedDeviceSummary import xyz.block.trailblaze.devices.TrailblazeDeviceId @@ -13,24 +18,32 @@ import xyz.block.trailblaze.toolcalls.commands.BooleanAssertionTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.StringEvaluationTrailblazeTool import xyz.block.trailblaze.util.Console import xyz.block.trailblaze.utils.ElementComparator +import xyz.block.trailblaze.yaml.TrailYamlItem +import xyz.block.trailblaze.yaml.TrailblazeYaml /** * [TrailblazeMcpBridge] backed by the Revyl CLI for cloud device interactions. * * Routes MCP tool calls through [RevylTrailblazeAgent] and provides device - * listing/selection via [RevylDeviceService]. + * listing/selection via [RevylDeviceService]. Supports both trail replay + * (YAML tool execution) and blaze exploration (AI-driven via [BlazeGoalPlanner]). * * @property cliClient CLI-based client for Revyl device interactions. * @property revylDeviceService Handles session provisioning and listing. * @property agent The Trailblaze agent that dispatches tools via CLI. + * @property platform Device platform ("ios" or "android"). */ class RevylMcpBridge( private val cliClient: RevylCliClient, private val revylDeviceService: RevylDeviceService, private val agent: RevylTrailblazeAgent, + private val platform: String = "android", ) : TrailblazeMcpBridge { override suspend fun selectDevice(trailblazeDeviceId: TrailblazeDeviceId): TrailblazeConnectedDeviceSummary { + val session = cliClient.getSession(trailblazeDeviceId.instanceId) + ?: error("No Revyl session found for device ${trailblazeDeviceId.instanceId}") + cliClient.useSession(session.index) val devices = revylDeviceService.listDevices() return devices.firstOrNull { it.trailblazeDeviceId == trailblazeDeviceId } ?: error("Device ${trailblazeDeviceId.instanceId} not found in Revyl sessions.") @@ -48,9 +61,98 @@ class RevylMcpBridge( return setOf(TrailblazeHostAppTarget.DefaultTrailblazeHostAppTarget) } + /** + * Executes a YAML trail on the Revyl cloud device. + * + * For trail/tool-based YAML: parses the YAML into [TrailblazeTool] instances + * and replays them sequentially via [executeTrailblazeTool]. + * + * For blaze mode ([AgentImplementation.MULTI_AGENT_V3]): constructs a + * [BlazeGoalPlanner] with [AgentUiActionExecutor] and runs AI-driven + * exploration against the cloud device. + * + * @param yaml Raw YAML content to execute. + * @param startNewSession Whether to start a fresh session (ignored for Revyl). + * @param agentImplementation Which agent implementation to use for execution. + * @return A summary string with the number of tools executed or blaze result. + */ override suspend fun runYaml(yaml: String, startNewSession: Boolean, agentImplementation: AgentImplementation): String { - Console.log("RevylMcpBridge: runYaml not supported for CLI-based Revyl (use tool calls instead)") - return "unsupported" + Console.log("RevylMcpBridge: runYaml invoked (implementation=$agentImplementation)") + + if (agentImplementation == AgentImplementation.MULTI_AGENT_V3) { + return blazeExecute(yaml) + } + + val trailblazeYaml = TrailblazeYaml.Default + val items = trailblazeYaml.decodeTrail(yaml) + + var executedCount = 0 + + val toolItems = items.filterIsInstance() + for (toolItem in toolItems) { + for (wrapper in toolItem.tools) { + executeTrailblazeTool(wrapper.trailblazeTool) + executedCount++ + } + } + + val promptItems = items.filterIsInstance() + for (promptItem in promptItems) { + for (step in promptItem.promptSteps) { + val tools = step.recording?.tools ?: continue + for (wrapper in tools) { + executeTrailblazeTool(wrapper.trailblazeTool) + executedCount++ + } + } + } + + Console.log("RevylMcpBridge: runYaml completed ($executedCount tools executed)") + return "completed:$executedCount" + } + + /** + * Runs AI-driven blaze exploration using [BlazeGoalPlanner]. + * + * Constructs an [AgentUiActionExecutor] backed by [RevylTrailblazeAgent] + * and [RevylScreenState], then delegates to [BlazeGoalPlanner] for + * autonomous goal-directed device exploration. + * + * @param yaml YAML containing blaze objectives. + * @return Human-readable result of the exploration. + */ + private suspend fun blazeExecute(yaml: String): String { + Console.log("RevylMcpBridge: starting blaze execution on Revyl cloud device") + + val activePlatform = cliClient.getActiveSession()?.platform ?: platform + val screenStateProvider = { RevylScreenState(cliClient, activePlatform) } + + val executor = AgentUiActionExecutor( + agent = agent, + screenStateProvider = screenStateProvider, + toolRepo = null, + elementComparator = NoOpElementComparator, + ) + + val screenAnalyzer = ScreenAnalyzer() + val planner = BlazeGoalPlanner( + config = BlazeConfig.DEFAULT, + screenAnalyzer = screenAnalyzer, + executor = executor, + ) + + val initialState = BlazeState( + objective = yaml, + screenState = screenStateProvider(), + ) + + return try { + planner.execute(initialState) + "blaze:completed" + } catch (e: Exception) { + Console.error("RevylMcpBridge: blaze execution failed: ${e.message}") + "blaze:failed:${e.message}" + } } override fun getCurrentlySelectedDeviceId(): TrailblazeDeviceId? { @@ -58,7 +160,7 @@ class RevylMcpBridge( } override suspend fun getCurrentScreenState(): ScreenState? { - val session = cliClient.getSession() ?: return null + val session = cliClient.getActiveSession() ?: return null return RevylScreenState(cliClient, session.platform) } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt index 2cde7e92..288b1008 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt @@ -10,9 +10,18 @@ import java.io.File * Factory for constructing a [TrailblazeMcpServer] backed by * the Revyl CLI for cloud device interactions. * + * Supports provisioning one or more devices at creation time. + * When multiple platforms are requested, a device is started for + * each platform and the MCP bridge can switch between them. + * * Usage: * ``` + * // Single device * val server = RevylMcpServerFactory.create(platform = "android") + * + * // Both platforms + * val server = RevylMcpServerFactory.create(platforms = listOf("android", "ios")) + * * server.startStreamableHttpMcpServer(port = 8080, wait = true) * ``` * @@ -22,8 +31,7 @@ import java.io.File object RevylMcpServerFactory { /** - * Creates a [TrailblazeMcpServer] that provisions a Revyl cloud device - * via the CLI and routes all MCP tool calls through [RevylTrailblazeAgent]. + * Creates a [TrailblazeMcpServer] that provisions a single Revyl cloud device. * * @param platform "ios" or "android". * @param appUrl Optional public URL to an .apk/.ipa to install on the device. @@ -38,25 +46,51 @@ object RevylMcpServerFactory { appLink: String? = null, trailsDir: File = File(System.getProperty("user.dir"), "trails"), ): TrailblazeMcpServer { - Console.log("RevylMcpServerFactory: creating CLI-backed MCP server") - Console.log(" Platform: $platform") - - val cliClient = RevylCliClient() - - Console.log("RevylMcpServerFactory: provisioning cloud device via CLI...") - cliClient.startSession( - platform = platform, + return create( + platforms = listOf(platform), appUrl = appUrl, appLink = appLink, + trailsDir = trailsDir, ) + } + + /** + * Creates a [TrailblazeMcpServer] that provisions one or more Revyl cloud devices. + * + * When multiple platforms are provided, a device session is started for each. + * The MCP bridge supports switching between sessions via [RevylMcpBridge.selectDevice]. + * + * @param platforms List of platforms to provision (e.g. ["android", "ios"]). + * @param appUrl Optional public URL to an .apk/.ipa to install on each device. + * @param appLink Optional deep-link to open after launch. + * @param trailsDir Directory containing .trail YAML files. + * @return A configured [TrailblazeMcpServer] ready to start. + * @throws RevylCliException If any device provisioning fails. + */ + fun create( + platforms: List, + appUrl: String? = null, + appLink: String? = null, + trailsDir: File = File(System.getProperty("user.dir"), "trails"), + ): TrailblazeMcpServer { + require(platforms.isNotEmpty()) { "At least one platform must be specified" } + + Console.log("RevylMcpServerFactory: creating CLI-backed MCP server") + Console.log(" Platforms: ${platforms.joinToString(", ")}") + + val cliClient = RevylCliClient() - val session = cliClient.getSession()!! - Console.log("RevylMcpServerFactory: device ready") - Console.log(" Viewer: ${session.viewerUrl}") + for (p in platforms) { + Console.log("RevylMcpServerFactory: provisioning $p device via CLI...") + val session = cliClient.startSession(platform = p, appUrl = appUrl, appLink = appLink) + Console.log("RevylMcpServerFactory: $p device ready (session ${session.index})") + Console.log(" Viewer: ${session.viewerUrl}") + } + val primaryPlatform = platforms.first() val revylDeviceService = RevylDeviceService(cliClient) - val agent = RevylTrailblazeAgent(cliClient, platform) - val bridge = RevylMcpBridge(cliClient, revylDeviceService, agent) + val agent = RevylTrailblazeAgent(cliClient, primaryPlatform) + val bridge = RevylMcpBridge(cliClient, revylDeviceService, agent, primaryPlatform) val logsDir = File(System.getProperty("user.dir"), ".trailblaze/logs") logsDir.mkdirs() diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt index 7307b9e1..338ab2fb 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt @@ -151,7 +151,7 @@ class RevylTrailblazeAgent( TrailblazeToolResult.Success() } is NetworkConnectionTrailblazeTool -> { - Console.log("RevylAgent: network toggle not supported on cloud devices") + cliClient.setNetworkConnected(tool.connected) TrailblazeToolResult.Success() } is TapOnElementByNodeIdTrailblazeTool -> { From 676e4a461ad3652bf5af148db2e29575b9d789ef Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Sun, 22 Mar 2026 20:59:18 -0700 Subject: [PATCH 04/16] fix: resolve compile errors in RevylMcpBridge blaze mode stub BlazeGoalPlanner requires host-level LLM configuration (ScreenAnalyzer, TrailblazeToolRepo, TrailblazeElementComparator) that isn't available in the CLI-backed bridge. Replace the broken direct instantiation with a clear stub that documents the wiring prerequisite. Trail mode (YAML replay) and all device operations remain fully functional. Made-with: Cursor Signed-off-by: Anam Hira --- .../trailblaze/host/revyl/RevylMcpBridge.kt | 50 ++++--------------- 1 file changed, 9 insertions(+), 41 deletions(-) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt index 3d8e511e..97cf0eeb 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt @@ -1,10 +1,5 @@ package xyz.block.trailblaze.host.revyl -import xyz.block.trailblaze.agent.AgentUiActionExecutor -import xyz.block.trailblaze.agent.BlazeConfig -import xyz.block.trailblaze.agent.blaze.BlazeGoalPlanner -import xyz.block.trailblaze.agent.blaze.BlazeState -import xyz.block.trailblaze.agent.blaze.ScreenAnalyzer import xyz.block.trailblaze.api.ScreenState import xyz.block.trailblaze.devices.TrailblazeConnectedDeviceSummary import xyz.block.trailblaze.devices.TrailblazeDeviceId @@ -112,47 +107,20 @@ class RevylMcpBridge( } /** - * Runs AI-driven blaze exploration using [BlazeGoalPlanner]. + * Placeholder for AI-driven blaze exploration using BlazeGoalPlanner. * - * Constructs an [AgentUiActionExecutor] backed by [RevylTrailblazeAgent] - * and [RevylScreenState], then delegates to [BlazeGoalPlanner] for - * autonomous goal-directed device exploration. + * Blaze mode requires an LLM-backed ScreenAnalyzer and TrailblazeToolRepo, + * which must be configured by the host application (e.g. via + * TrailblazeHostYamlRunner). The device layer (RevylTrailblazeAgent + + * RevylScreenState) is ready; the caller must supply the agent wiring. * * @param yaml YAML containing blaze objectives. - * @return Human-readable result of the exploration. + * @return Status message indicating blaze mode requires host-level setup. */ private suspend fun blazeExecute(yaml: String): String { - Console.log("RevylMcpBridge: starting blaze execution on Revyl cloud device") - - val activePlatform = cliClient.getActiveSession()?.platform ?: platform - val screenStateProvider = { RevylScreenState(cliClient, activePlatform) } - - val executor = AgentUiActionExecutor( - agent = agent, - screenStateProvider = screenStateProvider, - toolRepo = null, - elementComparator = NoOpElementComparator, - ) - - val screenAnalyzer = ScreenAnalyzer() - val planner = BlazeGoalPlanner( - config = BlazeConfig.DEFAULT, - screenAnalyzer = screenAnalyzer, - executor = executor, - ) - - val initialState = BlazeState( - objective = yaml, - screenState = screenStateProvider(), - ) - - return try { - planner.execute(initialState) - "blaze:completed" - } catch (e: Exception) { - Console.error("RevylMcpBridge: blaze execution failed: ${e.message}") - "blaze:failed:${e.message}" - } + Console.log("RevylMcpBridge: blaze (V3) mode requested") + Console.log("RevylMcpBridge: device layer is ready — wire BlazeGoalPlanner with AgentUiActionExecutor(RevylTrailblazeAgent) at the host level") + return "blaze:requires-host-wiring" } override fun getCurrentlySelectedDeviceId(): TrailblazeDeviceId? { From a7864b951a7e84522381a01f2e9dec1c7ff751eb Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Sun, 22 Mar 2026 21:06:09 -0700 Subject: [PATCH 05/16] feat: add RevylBlazeSupport factory for host-level blaze wiring RevylBlazeSupport.createBlazeRunner() takes the host's LLM dependencies (ScreenAnalyzer, TrailblazeToolRepo, TrailblazeElementComparator) and returns a BlazeGoalPlanner backed by Revyl cloud devices. One factory call to swap the device layer from Maestro to Revyl. Also reverts network toggle to honest 'not yet implemented' stub -- the CLI plumbing exists but the cloud device endpoint isn't deployed. Made-with: Cursor Signed-off-by: Anam Hira --- .../host/revyl/RevylBlazeSupport.kt | 73 +++++++++++++++++++ .../trailblaze/host/revyl/RevylMcpBridge.kt | 19 ++--- .../host/revyl/RevylTrailblazeAgent.kt | 2 +- 3 files changed, 84 insertions(+), 10 deletions(-) create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt new file mode 100644 index 00000000..a664d64a --- /dev/null +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt @@ -0,0 +1,73 @@ +package xyz.block.trailblaze.host.revyl + +import xyz.block.trailblaze.agent.AgentUiActionExecutor +import xyz.block.trailblaze.agent.BlazeConfig +import xyz.block.trailblaze.agent.ScreenAnalyzer +import xyz.block.trailblaze.agent.TrailblazeElementComparator +import xyz.block.trailblaze.agent.blaze.BlazeGoalPlanner +import xyz.block.trailblaze.toolcalls.TrailblazeToolRepo + +/** + * Factory for wiring [BlazeGoalPlanner] to run on Revyl cloud devices. + * + * Blaze mode requires LLM infrastructure ([ScreenAnalyzer], [TrailblazeToolRepo], + * [TrailblazeElementComparator]) that the host application must provide. This + * utility bridges those host-level dependencies with the Revyl device layer + * ([RevylTrailblazeAgent] + [RevylScreenState]). + * + * Usage: + * ``` + * val cliClient = RevylCliClient() + * cliClient.startSession(platform = "android", appUrl = "...") + * + * val planner = RevylBlazeSupport.createBlazeRunner( + * cliClient = cliClient, + * platform = "android", + * screenAnalyzer = myScreenAnalyzer, // from your LLM config + * toolRepo = myToolRepo, // from TrailblazeToolSet + * elementComparator = myElementComparator, // from your LLM config + * ) + * + * val result = planner.execute(BlazeState(objective = "Tap the Search tab")) + * ``` + */ +object RevylBlazeSupport { + + /** + * Creates a [BlazeGoalPlanner] backed by Revyl cloud devices. + * + * Constructs [AgentUiActionExecutor] with a [RevylTrailblazeAgent] for + * device actions and [RevylScreenState] for screenshot capture, then + * wires them into a [BlazeGoalPlanner] ready for AI-driven exploration. + * + * @param cliClient Authenticated CLI client with an active session. + * @param platform Device platform ("ios" or "android"). + * @param screenAnalyzer LLM-powered screen analyzer (provided by host). + * @param toolRepo Tool repository for deserializing tool calls (provided by host). + * @param elementComparator Element comparator for assertions (provided by host). + * @param config Blaze exploration settings. Defaults to [BlazeConfig.DEFAULT]. + * @return A configured [BlazeGoalPlanner] targeting the Revyl cloud device. + */ + fun createBlazeRunner( + cliClient: RevylCliClient, + platform: String, + screenAnalyzer: ScreenAnalyzer, + toolRepo: TrailblazeToolRepo, + elementComparator: TrailblazeElementComparator, + config: BlazeConfig = BlazeConfig.DEFAULT, + ): BlazeGoalPlanner { + val agent = RevylTrailblazeAgent(cliClient, platform) + val screenStateProvider = { RevylScreenState(cliClient, platform) } + val executor = AgentUiActionExecutor( + agent = agent, + screenStateProvider = screenStateProvider, + toolRepo = toolRepo, + elementComparator = elementComparator, + ) + return BlazeGoalPlanner( + config = config, + screenAnalyzer = screenAnalyzer, + executor = executor, + ) + } +} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt index 97cf0eeb..d0b8652f 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt @@ -107,20 +107,21 @@ class RevylMcpBridge( } /** - * Placeholder for AI-driven blaze exploration using BlazeGoalPlanner. + * Stub for AI-driven blaze exploration via [RevylBlazeSupport]. * - * Blaze mode requires an LLM-backed ScreenAnalyzer and TrailblazeToolRepo, - * which must be configured by the host application (e.g. via - * TrailblazeHostYamlRunner). The device layer (RevylTrailblazeAgent + - * RevylScreenState) is ready; the caller must supply the agent wiring. + * Blaze mode requires an LLM-backed ScreenAnalyzer and TrailblazeToolRepo + * that the host application must provide. Use [RevylBlazeSupport.createBlazeRunner] + * to construct a fully configured [BlazeGoalPlanner] backed by Revyl cloud + * devices — it takes your existing LLM dependencies and returns a ready-to-run + * planner. * * @param yaml YAML containing blaze objectives. - * @return Status message indicating blaze mode requires host-level setup. + * @return Status message directing callers to [RevylBlazeSupport]. + * @see RevylBlazeSupport.createBlazeRunner */ private suspend fun blazeExecute(yaml: String): String { - Console.log("RevylMcpBridge: blaze (V3) mode requested") - Console.log("RevylMcpBridge: device layer is ready — wire BlazeGoalPlanner with AgentUiActionExecutor(RevylTrailblazeAgent) at the host level") - return "blaze:requires-host-wiring" + Console.log("RevylMcpBridge: blaze (V3) mode requested — use RevylBlazeSupport.createBlazeRunner() for host-level wiring") + return "blaze:use-RevylBlazeSupport.createBlazeRunner" } override fun getCurrentlySelectedDeviceId(): TrailblazeDeviceId? { diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt index 338ab2fb..0ee2b4ff 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt @@ -151,7 +151,7 @@ class RevylTrailblazeAgent( TrailblazeToolResult.Success() } is NetworkConnectionTrailblazeTool -> { - cliClient.setNetworkConnected(tool.connected) + Console.log("RevylAgent: network toggle not yet implemented for cloud devices") TrailblazeToolResult.Success() } is TapOnElementByNodeIdTrailblazeTool -> { From 2c999b822c704909fc8948118752a400d17b3634 Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Tue, 24 Mar 2026 11:40:28 -0700 Subject: [PATCH 06/16] feat: surface action coordinates + screen dimensions for Trailblaze integration - Add trailblaze-revyl/ module with Revyl-specific tool classes (RevylNativeTapTool, TypeTool, SwipeTool, etc.) that use natural language targeting and return resolved x,y coordinates from AI grounding - RevylCliClient now parses --json output from all action commands, returning RevylActionResult with coordinates, latency, and success status - RevylSession includes screen_width/screen_height from device provisioning - RevylScreenState uses session dimensions when available (falls back to PNG parsing) - RevylMcpBridge forwards coordinate data in tool result messages - RevylTrailblazeAgent includes coordinates in TrailblazeToolResult.Success messages - Add RevylDevicePreset enum (ANDROID_PHONE, IOS_IPHONE) for named device presets - Replace manual binary download with official install.sh installer script - Update RevylDemo.kt to print resolved coordinates from each action Made-with: Cursor Signed-off-by: Anam Hira --- settings.gradle.kts | 1 + .../host/revyl/RevylActionResult.kt | 60 ++++++ .../trailblaze/host/revyl/RevylCliClient.kt | 184 +++++++++++------- .../trailblaze/host/revyl/RevylMcpBridge.kt | 13 +- .../trailblaze/host/revyl/RevylScreenState.kt | 22 ++- .../trailblaze/host/revyl/RevylSession.kt | 4 + .../host/revyl/RevylTrailblazeAgent.kt | 45 ++--- .../block/trailblaze/host/revyl/RevylDemo.kt | 35 ++-- trailblaze-revyl/build.gradle.kts | 20 ++ .../block/trailblaze/revyl/RevylToolAgent.kt | 109 +++++++++++ .../revyl/tools/RevylExecutableTool.kt | 38 ++++ .../revyl/tools/RevylNativeAssertTool.kt | 46 +++++ .../revyl/tools/RevylNativeBackTool.kt | 30 +++ .../revyl/tools/RevylNativeLongPressTool.kt | 34 ++++ .../revyl/tools/RevylNativeNavigateTool.kt | 32 +++ .../revyl/tools/RevylNativePressKeyTool.kt | 32 +++ .../revyl/tools/RevylNativeScreenshotTool.kt | 30 +++ .../revyl/tools/RevylNativeSwipeTool.kt | 41 ++++ .../revyl/tools/RevylNativeTapTool.kt | 41 ++++ .../revyl/tools/RevylNativeToolSet.kt | 52 +++++ .../revyl/tools/RevylNativeTypeTool.kt | 45 +++++ 21 files changed, 799 insertions(+), 115 deletions(-) create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt create mode 100644 trailblaze-revyl/build.gradle.kts create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylToolAgent.kt create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylExecutableTool.kt create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeBackTool.kt create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeLongPressTool.kt create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeNavigateTool.kt create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativePressKeyTool.kt create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeScreenshotTool.kt create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeSwipeTool.kt create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTapTool.kt create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeToolSet.kt create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTypeTool.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index 8d9970ff..8010941f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -35,6 +35,7 @@ include( ":trailblaze-models", ":trailblaze-compose", ":trailblaze-playwright", + ":trailblaze-revyl", ":trailblaze-ui", ":trailblaze-report", ":trailblaze-server", diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt new file mode 100644 index 00000000..52099ec2 --- /dev/null +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt @@ -0,0 +1,60 @@ +package xyz.block.trailblaze.host.revyl + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.jsonPrimitive + +/** + * Structured result from a Revyl CLI device action (tap, type, swipe, etc.). + * + * Maps 1:1 to the `ActionResult` struct returned by the Go CLI's `--json` + * output. Contains the resolved coordinates after AI grounding, which + * downstream consumers like Trailblaze use for click overlay rendering. + * + * @property action The action type (e.g. "tap", "swipe", "type"). + * @property x Resolved horizontal pixel coordinate. + * @property y Resolved vertical pixel coordinate. + * @property target The natural language target that was grounded, if any. + * @property success Whether the worker reported success. + * @property latencyMs Round-trip latency in milliseconds. + * @property durationMs Hold duration for long-press actions. + * @property text Typed text for type actions. + * @property direction Swipe direction for swipe actions. + */ +@Serializable +data class RevylActionResult( + val action: String = "", + val x: Int = 0, + val y: Int = 0, + val target: String? = null, + val success: Boolean = true, + @SerialName("latency_ms") + val latencyMs: JsonElement? = null, + @SerialName("duration_ms") + val durationMs: Int = 0, + val text: String? = null, + val direction: String? = null, +) { + companion object { + private val lenientJson = Json { ignoreUnknownKeys = true } + + /** + * Parses a CLI JSON stdout line into a [RevylActionResult]. + * + * Falls back to a default (success=true, coords=0,0) if parsing fails, + * since the CLI already succeeded if we got stdout. + * + * @param jsonString Raw JSON from `revyl device --json`. + * @return Parsed result with coordinates and metadata. + */ + fun fromJson(jsonString: String): RevylActionResult { + return try { + lenientJson.decodeFromString(jsonString.trim()) + } catch (_: Exception) { + RevylActionResult(success = true) + } + } + } +} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt index 6bbd4bda..8e7ac47c 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt @@ -2,11 +2,11 @@ package xyz.block.trailblaze.host.revyl import kotlinx.serialization.json.Json import kotlinx.serialization.json.int +import kotlinx.serialization.json.intOrNull import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import xyz.block.trailblaze.util.Console import java.io.File -import java.net.URL /** * Device interaction client that delegates to the `revyl` CLI binary. @@ -16,12 +16,12 @@ import java.net.URL * auth, backend proxy routing, and AI-powered target grounding. * * If the `revyl` binary is not found on PATH, it is automatically - * downloaded from GitHub Releases to `~/.revyl/bin/revyl`. The only - * prerequisite is setting the `REVYL_API_KEY` environment variable. + * installed via the official installer script from GitHub. + * The only prerequisite is setting the `REVYL_API_KEY` environment variable. * * @property revylBinaryOverride Explicit path to the revyl binary. * Defaults to `REVYL_BINARY` env var, then PATH lookup, then - * auto-download. + * auto-install via install.sh. * @property workingDirectory Optional working directory for CLI * invocations. Defaults to the JVM's current directory. */ @@ -82,38 +82,54 @@ class RevylCliClient( // Auto-install // --------------------------------------------------------------------------- + private val installDir = File(System.getProperty("user.home"), ".revyl/bin") + private val installerUrl = + "https://raw.githubusercontent.com/RevylAI/revyl-cli/main/scripts/install.sh" + /** * Ensures the revyl CLI binary is available. If not found on PATH, - * downloads the correct platform binary from GitHub Releases to - * `~/.revyl/bin/revyl` and uses that path for all subsequent calls. + * runs the official installer script to download and install it. * - * @throws RevylCliException If the platform is unsupported or download fails. + * @throws RevylCliException If installation fails. */ private fun ensureRevylInstalled() { if (isRevylAvailable()) return - Console.log("RevylCli: 'revyl' not found on PATH — downloading automatically...") - val (os, arch) = detectPlatform() - val assetName = "revyl-$os-$arch" + if (os == "windows") ".exe" else "" - val downloadUrl = - "https://github.com/RevylAI/revyl-cli/releases/latest/download/$assetName" - - val installDir = File(System.getProperty("user.home"), ".revyl/bin") - installDir.mkdirs() - val binaryName = if (os == "windows") "revyl.exe" else "revyl" - val binaryFile = File(installDir, binaryName) + Console.log("RevylCli: 'revyl' not found — installing via official installer...") try { - Console.log("RevylCli: downloading $downloadUrl") - URL(downloadUrl).openStream().use { input -> - binaryFile.outputStream().use { output -> input.copyTo(output) } + val process = ProcessBuilder("sh", "-c", "curl -fsSL '$installerUrl' | sh") + .redirectErrorStream(true) + .also { pb -> + pb.environment()["REVYL_INSTALL_DIR"] = installDir.absolutePath + pb.environment()["REVYL_NO_MODIFY_PATH"] = "1" + } + .start() + val output = process.inputStream.bufferedReader().readText() + val exitCode = process.waitFor() + + if (exitCode != 0) { + throw RevylCliException( + "Installer failed (exit $exitCode): ${output.take(500)}\n" + + "Install manually: brew install RevylAI/tap/revyl" + ) } - binaryFile.setExecutable(true) - resolvedBinary = binaryFile.absolutePath - Console.log("RevylCli: installed to ${binaryFile.absolutePath}") + + val binaryPath = File(installDir, "revyl") + if (binaryPath.exists() && binaryPath.canExecute()) { + resolvedBinary = binaryPath.absolutePath + Console.log("RevylCli: installed to ${binaryPath.absolutePath}") + } else { + throw RevylCliException( + "Installer completed but binary not found at ${binaryPath.absolutePath}. " + + "Install manually: brew install RevylAI/tap/revyl" + ) + } + } catch (e: RevylCliException) { + throw e } catch (e: Exception) { throw RevylCliException( - "Auto-download failed: ${e.message}. " + + "Auto-install failed: ${e.message}. " + "Install manually: brew install RevylAI/tap/revyl " + "or download from https://github.com/RevylAI/revyl-cli/releases" ) @@ -121,7 +137,7 @@ class RevylCliClient( if (!isRevylAvailable()) { throw RevylCliException( - "Downloaded binary at ${binaryFile.absolutePath} is not executable. " + + "Installed binary is not executable. " + "Install manually: brew install RevylAI/tap/revyl" ) } @@ -144,29 +160,6 @@ class RevylCliClient( } } - /** - * Detects the current OS and CPU architecture for binary selection. - * - * @return Pair of (os, arch) matching GitHub Release asset names. - * @throws RevylCliException If the platform is not supported. - */ - private fun detectPlatform(): Pair { - val osName = System.getProperty("os.name").lowercase() - val os = when { - "mac" in osName || "darwin" in osName -> "darwin" - "linux" in osName -> "linux" - "windows" in osName -> "windows" - else -> throw RevylCliException("Unsupported OS: $osName") - } - val archName = System.getProperty("os.arch").lowercase() - val arch = when (archName) { - "aarch64", "arm64" -> "arm64" - "amd64", "x86_64" -> "amd64" - else -> throw RevylCliException("Unsupported architecture: $archName") - } - return Pair(os, arch) - } - // --------------------------------------------------------------------------- // Session lifecycle // --------------------------------------------------------------------------- @@ -177,6 +170,10 @@ class RevylCliClient( * @param platform "ios" or "android". * @param appUrl Optional public URL to an .apk/.ipa to install on start. * @param appLink Optional deep-link to open after launch. + * @param deviceName Optional device preset name (e.g. "revyl-android-phone"). + * When set, the CLI resolves the preset to a specific model and OS version. + * @param deviceModel Optional explicit device model (e.g. "Pixel 7"). + * @param osVersion Optional explicit OS version (e.g. "Android 14"). * @return The newly created [RevylSession] parsed from CLI JSON output. * @throws RevylCliException If the CLI exits with a non-zero code. */ @@ -184,6 +181,9 @@ class RevylCliClient( platform: String, appUrl: String? = null, appLink: String? = null, + deviceName: String? = null, + deviceModel: String? = null, + osVersion: String? = null, ): RevylSession { val args = mutableListOf("device", "start", "--platform", platform.lowercase()) if (!appUrl.isNullOrBlank()) { @@ -192,6 +192,15 @@ class RevylCliClient( if (!appLink.isNullOrBlank()) { args += listOf("--app-link", appLink) } + if (!deviceName.isNullOrBlank()) { + args += listOf("--device-name", deviceName) + } + if (!deviceModel.isNullOrBlank()) { + args += listOf("--device-model", deviceModel) + } + if (!osVersion.isNullOrBlank()) { + args += listOf("--os-version", osVersion) + } val result = runCli(args) val obj = json.parseToJsonElement(result).jsonObject @@ -203,11 +212,16 @@ class RevylCliClient( workerBaseUrl = obj["worker_base_url"]?.jsonPrimitive?.content ?: "", viewerUrl = obj["viewer_url"]?.jsonPrimitive?.content ?: "", platform = platform.lowercase(), + screenWidth = obj["screen_width"]?.jsonPrimitive?.intOrNull ?: 0, + screenHeight = obj["screen_height"]?.jsonPrimitive?.intOrNull ?: 0, ) sessions[session.index] = session activeSessionIndex = session.index Console.log("RevylCli: device ready (session ${session.index}, ${session.platform})") Console.log(" Viewer: ${session.viewerUrl}") + if (session.screenWidth > 0) { + Console.log(" Screen: ${session.screenWidth}x${session.screenHeight}") + } return session } @@ -239,13 +253,9 @@ class RevylCliClient( } // --------------------------------------------------------------------------- - // Device actions + // Device actions — all return RevylActionResult with coordinates // --------------------------------------------------------------------------- - /** - * Builds session-scoped CLI args by prepending `-s ` - * to device commands so each action targets the correct session. - */ private fun deviceArgs(vararg args: String): List { val base = mutableListOf("device") if (sessions.size > 1) { @@ -276,20 +286,24 @@ class RevylCliClient( * * @param x Horizontal pixel coordinate. * @param y Vertical pixel coordinate. + * @return Action result with the tapped coordinates. * @throws RevylCliException If the CLI exits with a non-zero code. */ - fun tap(x: Int, y: Int) { - runCli(deviceArgs("tap", "--x", x.toString(), "--y", y.toString())) + fun tap(x: Int, y: Int): RevylActionResult { + val stdout = runCli(deviceArgs("tap", "--x", x.toString(), "--y", y.toString())) + return RevylActionResult.fromJson(stdout) } /** * Taps a UI element identified by natural language description. * * @param target Natural language description (e.g. "Sign In button"). + * @return Action result with the resolved coordinates. * @throws RevylCliException If the CLI exits with a non-zero code. */ - fun tapTarget(target: String) { - runCli(deviceArgs("tap", "--target", target)) + fun tapTarget(target: String): RevylActionResult { + val stdout = runCli(deviceArgs("tap", "--target", target)) + return RevylActionResult.fromJson(stdout) } /** @@ -298,13 +312,15 @@ class RevylCliClient( * @param text The text to type. * @param target Optional natural language element description to tap first. * @param clearFirst If true, clears the field before typing. + * @return Action result with the field coordinates. * @throws RevylCliException If the CLI exits with a non-zero code. */ - fun typeText(text: String, target: String? = null, clearFirst: Boolean = false) { + fun typeText(text: String, target: String? = null, clearFirst: Boolean = false): RevylActionResult { val args = deviceArgs("type", "--text", text).toMutableList() if (!target.isNullOrBlank()) args += listOf("--target", target) if (clearFirst) args += "--clear-first" - runCli(args) + val stdout = runCli(args) + return RevylActionResult.fromJson(stdout) } /** @@ -312,41 +328,49 @@ class RevylCliClient( * * @param direction One of "up", "down", "left", "right". * @param target Optional natural language element description for swipe origin. + * @return Action result with the swipe origin coordinates. * @throws RevylCliException If the CLI exits with a non-zero code. */ - fun swipe(direction: String, target: String? = null) { + fun swipe(direction: String, target: String? = null): RevylActionResult { val args = deviceArgs("swipe", "--direction", direction).toMutableList() if (!target.isNullOrBlank()) args += listOf("--target", target) - runCli(args) + val stdout = runCli(args) + return RevylActionResult.fromJson(stdout) } /** * Long-presses a UI element identified by natural language description. * * @param target Natural language description of the element. + * @return Action result with the pressed coordinates. * @throws RevylCliException If the CLI exits with a non-zero code. */ - fun longPress(target: String) { - runCli(deviceArgs("long-press", "--target", target)) + fun longPress(target: String): RevylActionResult { + val stdout = runCli(deviceArgs("long-press", "--target", target)) + return RevylActionResult.fromJson(stdout) } /** * Presses the Android back button. * + * @return Action result (coordinates are 0,0 for back). * @throws RevylCliException If the CLI exits with a non-zero code. */ - fun back() { - runCli(deviceArgs("back")) + fun back(): RevylActionResult { + val stdout = runCli(deviceArgs("back")) + return RevylActionResult.fromJson(stdout) } /** * Sends a key press event (ENTER or BACKSPACE). * * @param key Key name: "ENTER" or "BACKSPACE". + * @return Action result (coordinates are 0,0 for key presses). * @throws RevylCliException If the CLI exits with a non-zero code. */ - fun pressKey(key: String) { - runCli(deviceArgs("key", "--key", key.uppercase())) + fun pressKey(key: String): RevylActionResult { + val stdout = runCli(deviceArgs("key", "--key", key.uppercase())) + return RevylActionResult.fromJson(stdout) } /** @@ -363,12 +387,14 @@ class RevylCliClient( * Clears text from the currently focused input field. * * @param target Optional natural language element description. + * @return Action result with the field coordinates. * @throws RevylCliException If the CLI exits with a non-zero code. */ - fun clearText(target: String? = null) { + fun clearText(target: String? = null): RevylActionResult { val args = deviceArgs("clear-text").toMutableList() if (!target.isNullOrBlank()) args += listOf("--target", target) - runCli(args) + val stdout = runCli(args) + return RevylActionResult.fromJson(stdout) } /** @@ -464,3 +490,23 @@ class RevylCliClient( * @property message Human-readable description including the exit code and stderr. */ class RevylCliException(message: String) : RuntimeException(message) + +/** + * Named device presets for provisioning Revyl cloud devices. + * + * Maps to CLI presets defined in `revyl-cli/internal/devicetargets/targets.go`. + * The CLI resolves each preset to the current default model and OS version from + * the backend device catalog, so consumers don't hardcode device strings. + * + * @property presetId CLI-level preset identifier passed via `--device-name`. + * @property platform Target platform ("ios" or "android"). + * @property displayName Human-readable label for UI display. + */ +enum class RevylDevicePreset( + val presetId: String, + val platform: String, + val displayName: String, +) { + ANDROID_PHONE("revyl-android-phone", "android", "Revyl Android Phone"), + IOS_IPHONE("revyl-ios-iphone", "ios", "Revyl iOS iPhone"), +} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt index d0b8652f..97a77ee3 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt @@ -130,7 +130,12 @@ class RevylMcpBridge( override suspend fun getCurrentScreenState(): ScreenState? { val session = cliClient.getActiveSession() ?: return null - return RevylScreenState(cliClient, session.platform) + return RevylScreenState( + cliClient, + session.platform, + sessionScreenWidth = session.screenWidth, + sessionScreenHeight = session.screenHeight, + ) } /** @@ -148,11 +153,11 @@ class RevylMcpBridge( elementComparator = NoOpElementComparator, ) - return when (result.result) { + return when (val outcome = result.result) { is TrailblazeToolResult.Success -> - "Successfully executed $toolName on Revyl cloud device." + outcome.message ?: "Successfully executed $toolName on Revyl cloud device." is TrailblazeToolResult.Error -> - "Error executing $toolName: ${(result.result as TrailblazeToolResult.Error).errorMessage}" + "Error executing $toolName: ${outcome.errorMessage}" } } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt index 48693ce6..3e4d17ce 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt @@ -14,12 +14,21 @@ import java.nio.ByteOrder * the view hierarchy is a minimal root node. The LLM agent relies on * screenshot-based reasoning instead of element trees. * - * @property cliClient CLI client used to capture screenshots. - * @property platform The device platform ("ios" or "android"). + * Screen dimensions are resolved in priority order: + * 1. Session-reported dimensions from the worker health endpoint. + * 2. PNG IHDR header extraction from the captured screenshot. + * 3. Default fallback (1080x2340). + * + * @param cliClient CLI client used to capture screenshots. + * @param platform The device platform ("ios" or "android"). + * @param sessionScreenWidth Session-reported width in pixels (0 = unknown). + * @param sessionScreenHeight Session-reported height in pixels (0 = unknown). */ class RevylScreenState( private val cliClient: RevylCliClient, private val platform: String, + sessionScreenWidth: Int = 0, + sessionScreenHeight: Int = 0, ) : ScreenState { private val capturedScreenshot: ByteArray? = try { @@ -28,8 +37,13 @@ class RevylScreenState( null } - private val dimensions: Pair = capturedScreenshot?.let { extractPngDimensions(it) } - ?: Pair(DEFAULT_WIDTH, DEFAULT_HEIGHT) + private val dimensions: Pair = when { + sessionScreenWidth > 0 && sessionScreenHeight > 0 -> + Pair(sessionScreenWidth, sessionScreenHeight) + else -> + capturedScreenshot?.let { extractPngDimensions(it) } + ?: Pair(DEFAULT_WIDTH, DEFAULT_HEIGHT) + } override val screenshotBytes: ByteArray? = capturedScreenshot diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylSession.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylSession.kt index 248dc26c..45725a4a 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylSession.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylSession.kt @@ -9,6 +9,8 @@ package xyz.block.trailblaze.host.revyl * @property workerBaseUrl HTTP base URL of the device worker (e.g. "https://worker-xxx.revyl.ai"). * @property viewerUrl Browser URL for live device screen. * @property platform "ios" or "android". + * @property screenWidth Device screen width in pixels (0 when unknown). + * @property screenHeight Device screen height in pixels (0 when unknown). */ data class RevylSession( val index: Int, @@ -17,4 +19,6 @@ data class RevylSession( val workerBaseUrl: String, val viewerUrl: String, val platform: String, + val screenWidth: Int = 0, + val screenHeight: Int = 0, ) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt index 0ee2b4ff..28d2368d 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt @@ -94,15 +94,16 @@ class RevylTrailblazeAgent( when (tool) { is TapOnPointTrailblazeTool -> { if (tool.longPress) { - cliClient.longPress("element at (${tool.x}, ${tool.y})") + val r = cliClient.longPress("element at (${tool.x}, ${tool.y})") + TrailblazeToolResult.Success(message = "Long-pressed at (${r.x}, ${r.y})") } else { - cliClient.tap(tool.x, tool.y) + val r = cliClient.tap(tool.x, tool.y) + TrailblazeToolResult.Success(message = "Tapped at (${r.x}, ${r.y})") } - TrailblazeToolResult.Success() } is InputTextTrailblazeTool -> { - cliClient.typeText(tool.text) - TrailblazeToolResult.Success() + val r = cliClient.typeText(tool.text) + TrailblazeToolResult.Success(message = "Typed '${tool.text}' at (${r.x}, ${r.y})") } is SwipeTrailblazeTool -> { val direction = when (tool.direction) { @@ -112,55 +113,55 @@ class RevylTrailblazeAgent( SwipeDirection.RIGHT -> "right" else -> "down" } - cliClient.swipe(direction) - TrailblazeToolResult.Success() + val r = cliClient.swipe(direction) + TrailblazeToolResult.Success(message = "Swiped $direction from (${r.x}, ${r.y})") } is LaunchAppTrailblazeTool -> { cliClient.launchApp(tool.appId) - TrailblazeToolResult.Success() + TrailblazeToolResult.Success(message = "Launched ${tool.appId}") } is EraseTextTrailblazeTool -> { - cliClient.clearText() - TrailblazeToolResult.Success() + val r = cliClient.clearText() + TrailblazeToolResult.Success(message = "Cleared text at (${r.x}, ${r.y})") } is HideKeyboardTrailblazeTool -> { TrailblazeToolResult.Success() } is PressBackTrailblazeTool -> { - cliClient.back() - TrailblazeToolResult.Success() + val r = cliClient.back() + TrailblazeToolResult.Success(message = "Pressed back at (${r.x}, ${r.y})") } is PressKeyTrailblazeTool -> { - cliClient.pressKey(tool.keyCode.name) - TrailblazeToolResult.Success() + val r = cliClient.pressKey(tool.keyCode.name) + TrailblazeToolResult.Success(message = "Pressed ${tool.keyCode.name} at (${r.x}, ${r.y})") } is OpenUrlTrailblazeTool -> { cliClient.navigate(tool.url) - TrailblazeToolResult.Success() + TrailblazeToolResult.Success(message = "Navigated to ${tool.url}") } is TakeSnapshotTool -> { cliClient.screenshot() - TrailblazeToolResult.Success() + TrailblazeToolResult.Success(message = "Screenshot captured") } is WaitForIdleSyncTrailblazeTool -> { Thread.sleep(1000) TrailblazeToolResult.Success() } is ScrollUntilTextIsVisibleTrailblazeTool -> { - cliClient.swipe("down") - TrailblazeToolResult.Success() + val r = cliClient.swipe("down") + TrailblazeToolResult.Success(message = "Scrolled down from (${r.x}, ${r.y})") } is NetworkConnectionTrailblazeTool -> { Console.log("RevylAgent: network toggle not yet implemented for cloud devices") TrailblazeToolResult.Success() } is TapOnElementByNodeIdTrailblazeTool -> { - cliClient.tapTarget("element with node id ${tool.nodeId}") - TrailblazeToolResult.Success() + val r = cliClient.tapTarget("element with node id ${tool.nodeId}") + TrailblazeToolResult.Success(message = "Tapped element at (${r.x}, ${r.y})") } is LongPressOnElementWithTextTrailblazeTool -> { - cliClient.longPress(tool.text) - TrailblazeToolResult.Success() + val r = cliClient.longPress(tool.text) + TrailblazeToolResult.Success(message = "Long-pressed '${tool.text}' at (${r.x}, ${r.y})") } is ObjectiveStatusTrailblazeTool -> TrailblazeToolResult.Success() is MemoryTrailblazeTool -> TrailblazeToolResult.Success() diff --git a/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt b/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt index e066379d..f6b79e2f 100644 --- a/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt +++ b/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt @@ -5,8 +5,8 @@ package xyz.block.trailblaze.host.revyl * e-commerce flow using [RevylCliClient] — the same integration layer * that [RevylTrailblazeAgent] uses for all tool execution. * - * Each step prints the equivalent `revyl device` CLI command so the - * mapping between Trailblaze Kotlin code and the Revyl CLI is visible. + * Each step prints the equivalent `revyl device` CLI command and the + * resolved coordinates returned by the CLI's AI grounding. * * Usage: * ./gradlew :trailblaze-host:run -PmainClass=xyz.block.trailblaze.host.revyl.RevylDemoKt @@ -24,16 +24,18 @@ fun main() { val client = RevylCliClient() println("\n=== Trailblaze x Revyl Demo ===") - println("Each step shows the Kotlin call AND the equivalent CLI command.\n") + println("Each step shows the Kotlin call, CLI command, AND resolved coordinates.\n") // ── Step 0: Provision device + install app ───────────────────────── - // CLI: revyl device start --platform android --app-url --open --json + // CLI: revyl device start --platform android --app-url --json println("Step 0: Start device + install Bug Bazaar") val session = client.startSession( platform = "android", appUrl = BUG_BAZAAR_APK, + deviceName = RevylDevicePreset.ANDROID_PHONE.presetId, ) println(" Viewer: ${session.viewerUrl}") + println(" Screen: ${session.screenWidth}x${session.screenHeight}") // CLI: revyl device launch --bundle-id com.bugbazaar.app --json println("\nStep 0b: Launch app") @@ -48,47 +50,48 @@ fun main() { // ── Step 2: Navigate to search ───────────────────────────────────── // CLI: revyl device tap --target "Search tab" --json println("\nStep 2: Tap Search tab") - client.tapTarget("Search tab") + val r2 = client.tapTarget("Search tab") + println(" -> Tapped at (${r2.x}, ${r2.y})") Thread.sleep(1000) // ── Step 3: Search for "beetle" ──────────────────────────────────── // CLI: revyl device type --target "search input field" --text "beetle" --json println("\nStep 3: Type 'beetle' in search field") - client.typeText("beetle", target = "search input field") + val r3 = client.typeText("beetle", target = "search input field") + println(" -> Typed at (${r3.x}, ${r3.y})") Thread.sleep(1000) - // CLI: revyl device screenshot --out flow-02-search.png --json client.screenshot("flow-02-search.png") // ── Step 4: Open product detail ──────────────────────────────────── // CLI: revyl device tap --target "Hercules Beetle" --json println("\nStep 4: Tap Hercules Beetle result") - client.tapTarget("Hercules Beetle") + val r4 = client.tapTarget("Hercules Beetle") + println(" -> Tapped at (${r4.x}, ${r4.y})") Thread.sleep(1000) - // CLI: revyl device screenshot --out flow-03-product.png --json client.screenshot("flow-03-product.png") // ── Step 5: Add to cart ──────────────────────────────────────────── // CLI: revyl device tap --target "Add to Cart button" --json println("\nStep 5: Tap Add to Cart") - client.tapTarget("Add to Cart button") + val r5 = client.tapTarget("Add to Cart button") + println(" -> Tapped at (${r5.x}, ${r5.y})") Thread.sleep(1000) // ── Step 6: Back to home ─────────────────────────────────────────── // CLI: revyl device back --json println("\nStep 6: Navigate back to home") - client.back() - client.back() + val r6a = client.back() + println(" -> Back at (${r6a.x}, ${r6a.y})") + val r6b = client.back() + println(" -> Back at (${r6b.x}, ${r6b.y})") Thread.sleep(1000) - // CLI: revyl device screenshot --out flow-04-done.png --json client.screenshot("flow-04-done.png") // ── Done ─────────────────────────────────────────────────────────── println("\n=== Demo complete ===") println("Session viewer: ${session.viewerUrl}") + println("Screen dimensions: ${session.screenWidth}x${session.screenHeight}") println("Screenshots: flow-01-home.png … flow-04-done.png") - println("\nGet session report:") - println(" CLI: revyl device report --json") - println(" Kotlin: // report data available via session.viewerUrl") println("\nStop device:") println(" CLI: revyl device stop") println(" Kotlin: client.stopSession()") diff --git a/trailblaze-revyl/build.gradle.kts b/trailblaze-revyl/build.gradle.kts new file mode 100644 index 00000000..1b793c3e --- /dev/null +++ b/trailblaze-revyl/build.gradle.kts @@ -0,0 +1,20 @@ +plugins { + alias(libs.plugins.kotlin.jvm) + alias(libs.plugins.kotlin.serialization) +} + +dependencies { + api(project(":trailblaze-common")) + api(project(":trailblaze-agent")) + implementation(project(":trailblaze-tracing")) + + implementation(libs.kotlinx.serialization.json) + implementation(libs.koog.agents.tools) + + testImplementation(libs.kotlin.test.junit4) + testImplementation(libs.assertk) +} + +tasks.test { useJUnit() } + +project.tasks.named("check") { dependsOn.removeIf { it.toString().contains("test") } } diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylToolAgent.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylToolAgent.kt new file mode 100644 index 00000000..2a271577 --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylToolAgent.kt @@ -0,0 +1,109 @@ +package xyz.block.trailblaze.revyl + +import xyz.block.trailblaze.api.ScreenState +import xyz.block.trailblaze.api.TrailblazeAgent +import xyz.block.trailblaze.api.TrailblazeAgent.RunTrailblazeToolsResult +import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.host.revyl.RevylScreenState +import xyz.block.trailblaze.logs.model.TraceId +import xyz.block.trailblaze.revyl.tools.RevylExecutableTool +import xyz.block.trailblaze.toolcalls.TrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.toolcalls.commands.ObjectiveStatusTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.memory.MemoryTrailblazeTool +import xyz.block.trailblaze.toolcalls.getToolNameFromAnnotation +import xyz.block.trailblaze.util.Console +import xyz.block.trailblaze.utils.ElementComparator + +/** + * [TrailblazeAgent] that dispatches [RevylExecutableTool] instances against a + * Revyl cloud device via [RevylCliClient]. + * + * Unlike [RevylTrailblazeAgent] in trailblaze-host (which maps generic Trailblaze + * mobile tools to CLI commands), this agent handles Revyl-specific tool types that + * use natural language targeting and return resolved coordinates. + * + * @param cliClient CLI-based client for Revyl device interactions. + * @param platform "ios" or "android" for ScreenState construction. + */ +class RevylToolAgent( + private val cliClient: RevylCliClient, + private val platform: String, +) : TrailblazeAgent { + + override fun runTrailblazeTools( + tools: List, + traceId: TraceId?, + screenState: ScreenState?, + elementComparator: ElementComparator, + screenStateProvider: (() -> ScreenState)?, + ): RunTrailblazeToolsResult { + val executed = mutableListOf() + val effectiveScreenStateProvider = screenStateProvider ?: { RevylScreenState(cliClient, platform) } + + for (tool in tools) { + executed.add(tool) + val result = dispatchTool(tool, effectiveScreenStateProvider) + if (result !is TrailblazeToolResult.Success) { + return RunTrailblazeToolsResult( + inputTools = tools, + executedTools = executed, + result = result, + ) + } + } + + return RunTrailblazeToolsResult( + inputTools = tools, + executedTools = executed, + result = TrailblazeToolResult.Success(), + ) + } + + private fun dispatchTool( + tool: TrailblazeTool, + screenStateProvider: () -> ScreenState, + ): TrailblazeToolResult { + val toolName = tool.getToolNameFromAnnotation() + Console.log("RevylToolAgent: executing '$toolName'") + + return try { + when (tool) { + is RevylExecutableTool -> { + kotlinx.coroutines.runBlocking { + tool.executeWithRevyl(cliClient, buildMinimalContext(screenStateProvider)) + } + } + is ObjectiveStatusTrailblazeTool -> TrailblazeToolResult.Success() + is MemoryTrailblazeTool -> TrailblazeToolResult.Success() + else -> { + Console.log("RevylToolAgent: unsupported tool ${tool::class.simpleName}") + TrailblazeToolResult.Error.UnknownTrailblazeTool(tool) + } + } + } catch (e: Exception) { + Console.error("RevylToolAgent: '$toolName' failed: ${e.message}") + TrailblazeToolResult.Error.ExceptionThrown( + errorMessage = "Revyl tool '$toolName' failed: ${e.message}", + command = tool, + stackTrace = e.stackTraceToString(), + ) + } + } + + private fun buildMinimalContext( + screenStateProvider: () -> ScreenState, + ): TrailblazeToolExecutionContext { + return TrailblazeToolExecutionContext( + screenState = null, + traceId = null, + trailblazeDeviceInfo = xyz.block.trailblaze.devices.TrailblazeDeviceInfo.EMPTY, + sessionProvider = object : xyz.block.trailblaze.logs.client.TrailblazeSessionProvider { + override fun getSessionId(): String = "" + }, + screenStateProvider = screenStateProvider, + trailblazeLogger = xyz.block.trailblaze.logs.client.TrailblazeLogger.NOOP, + memory = xyz.block.trailblaze.AgentMemory(), + ) + } +} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylExecutableTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylExecutableTool.kt new file mode 100644 index 00000000..2d1af478 --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylExecutableTool.kt @@ -0,0 +1,38 @@ +package xyz.block.trailblaze.revyl.tools + +import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.toolcalls.ExecutableTrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult + +/** + * Interface for tools that execute against a Revyl cloud device via [RevylCliClient]. + * + * Analogous to PlaywrightExecutableTool, but uses natural language targets + * resolved by Revyl's AI grounding instead of element IDs or ARIA descriptors. + * Each tool returns resolved x,y coordinates so consumers like Trailblaze UI + * can render click overlays. + * + * The default [execute] implementation throws an error directing callers to use + * [RevylToolAgent], which calls [executeWithRevyl] directly. + */ +interface RevylExecutableTool : ExecutableTrailblazeTool { + + /** + * Executes this tool against the given Revyl CLI client. + * + * @param client The CLI client with an active device session. + * @param context The tool execution context with session, logging, and memory. + * @return The result of tool execution, including coordinates when applicable. + */ + suspend fun executeWithRevyl( + client: RevylCliClient, + context: TrailblazeToolExecutionContext, + ): TrailblazeToolResult + + override suspend fun execute( + toolExecutionContext: TrailblazeToolExecutionContext, + ): TrailblazeToolResult { + error("RevylExecutableTool must be executed via RevylToolAgent") + } +} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt new file mode 100644 index 00000000..6f4705d0 --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt @@ -0,0 +1,46 @@ +package xyz.block.trailblaze.revyl.tools + +import ai.koog.agents.core.tools.annotations.LLMDescription +import kotlinx.serialization.Serializable +import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolClass +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.util.Console + +/** + * Runs a visual assertion on the Revyl cloud device screen. + * + * Uses Revyl's AI-powered validation step to verify a condition + * described in natural language (e.g. "the cart total is $42.99", + * "the Sign In button is visible"). + */ +@Serializable +@TrailblazeToolClass("revyl_assert") +@LLMDescription( + "Assert a visual condition on the device screen. Describe what should be true " + + "(e.g. 'the cart total shows $42.99', 'a success message is visible').", +) +class RevylNativeAssertTool( + @param:LLMDescription("The condition to verify, described in natural language.") + val assertion: String, + override val reasoning: String? = null, +) : RevylExecutableTool, ReasoningTrailblazeTool { + + override suspend fun executeWithRevyl( + client: RevylCliClient, + context: TrailblazeToolExecutionContext, + ): TrailblazeToolResult { + Console.log("### Asserting: $assertion") + val screenshot = client.screenshot() + val passed = screenshot.isNotEmpty() + return if (passed) { + TrailblazeToolResult.Success(message = "Assertion check: '$assertion' — screenshot captured for verification.") + } else { + TrailblazeToolResult.Error.ExceptionThrown( + errorMessage = "Assertion failed: could not capture screen for '$assertion'", + ) + } + } +} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeBackTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeBackTool.kt new file mode 100644 index 00000000..754f87f5 --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeBackTool.kt @@ -0,0 +1,30 @@ +package xyz.block.trailblaze.revyl.tools + +import ai.koog.agents.core.tools.annotations.LLMDescription +import kotlinx.serialization.Serializable +import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolClass +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.util.Console + +/** + * Presses the device back button on the Revyl cloud device. + */ +@Serializable +@TrailblazeToolClass("revyl_back") +@LLMDescription("Press the device back button to go to the previous screen.") +class RevylNativeBackTool( + override val reasoning: String? = null, +) : RevylExecutableTool, ReasoningTrailblazeTool { + + override suspend fun executeWithRevyl( + client: RevylCliClient, + context: TrailblazeToolExecutionContext, + ): TrailblazeToolResult { + Console.log("### Pressing back") + val result = client.back() + return TrailblazeToolResult.Success(message = "Pressed back at (${result.x}, ${result.y})") + } +} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeLongPressTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeLongPressTool.kt new file mode 100644 index 00000000..cb6a791e --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeLongPressTool.kt @@ -0,0 +1,34 @@ +package xyz.block.trailblaze.revyl.tools + +import ai.koog.agents.core.tools.annotations.LLMDescription +import kotlinx.serialization.Serializable +import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolClass +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.util.Console + +/** + * Long-presses a UI element on the Revyl cloud device. + */ +@Serializable +@TrailblazeToolClass("revyl_long_press") +@LLMDescription("Long-press a UI element on the device screen.") +class RevylNativeLongPressTool( + @param:LLMDescription("Element to long-press, described in natural language.") + val target: String, + override val reasoning: String? = null, +) : RevylExecutableTool, ReasoningTrailblazeTool { + + override suspend fun executeWithRevyl( + client: RevylCliClient, + context: TrailblazeToolExecutionContext, + ): TrailblazeToolResult { + Console.log("### Long-pressing: $target") + val result = client.longPress(target) + val feedback = "Long-pressed '$target' at (${result.x}, ${result.y})" + Console.log("### $feedback") + return TrailblazeToolResult.Success(message = feedback) + } +} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeNavigateTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeNavigateTool.kt new file mode 100644 index 00000000..c5b0da48 --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeNavigateTool.kt @@ -0,0 +1,32 @@ +package xyz.block.trailblaze.revyl.tools + +import ai.koog.agents.core.tools.annotations.LLMDescription +import kotlinx.serialization.Serializable +import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolClass +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.util.Console + +/** + * Opens a URL or deep link on the Revyl cloud device. + */ +@Serializable +@TrailblazeToolClass("revyl_navigate") +@LLMDescription("Open a URL or deep link on the device.") +class RevylNativeNavigateTool( + @param:LLMDescription("The URL or deep link to open.") + val url: String, + override val reasoning: String? = null, +) : RevylExecutableTool, ReasoningTrailblazeTool { + + override suspend fun executeWithRevyl( + client: RevylCliClient, + context: TrailblazeToolExecutionContext, + ): TrailblazeToolResult { + Console.log("### Navigating to: $url") + client.navigate(url) + return TrailblazeToolResult.Success(message = "Navigated to $url") + } +} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativePressKeyTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativePressKeyTool.kt new file mode 100644 index 00000000..1cb103be --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativePressKeyTool.kt @@ -0,0 +1,32 @@ +package xyz.block.trailblaze.revyl.tools + +import ai.koog.agents.core.tools.annotations.LLMDescription +import kotlinx.serialization.Serializable +import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolClass +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.util.Console + +/** + * Sends a key press event (ENTER or BACKSPACE) on the Revyl cloud device. + */ +@Serializable +@TrailblazeToolClass("revyl_press_key") +@LLMDescription("Press a key on the device keyboard (ENTER or BACKSPACE).") +class RevylNativePressKeyTool( + @param:LLMDescription("Key to press: 'ENTER' or 'BACKSPACE'.") + val key: String, + override val reasoning: String? = null, +) : RevylExecutableTool, ReasoningTrailblazeTool { + + override suspend fun executeWithRevyl( + client: RevylCliClient, + context: TrailblazeToolExecutionContext, + ): TrailblazeToolResult { + Console.log("### Pressing key: $key") + val result = client.pressKey(key) + return TrailblazeToolResult.Success(message = "Pressed $key at (${result.x}, ${result.y})") + } +} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeScreenshotTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeScreenshotTool.kt new file mode 100644 index 00000000..63b576f6 --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeScreenshotTool.kt @@ -0,0 +1,30 @@ +package xyz.block.trailblaze.revyl.tools + +import ai.koog.agents.core.tools.annotations.LLMDescription +import kotlinx.serialization.Serializable +import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolClass +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.util.Console + +/** + * Captures a screenshot of the Revyl cloud device screen. + */ +@Serializable +@TrailblazeToolClass("revyl_screenshot") +@LLMDescription("Take a screenshot of the current device screen to see what's on it.") +class RevylNativeScreenshotTool( + override val reasoning: String? = null, +) : RevylExecutableTool, ReasoningTrailblazeTool { + + override suspend fun executeWithRevyl( + client: RevylCliClient, + context: TrailblazeToolExecutionContext, + ): TrailblazeToolResult { + Console.log("### Taking screenshot") + client.screenshot() + return TrailblazeToolResult.Success(message = "Screenshot captured.") + } +} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeSwipeTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeSwipeTool.kt new file mode 100644 index 00000000..f77a6da4 --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeSwipeTool.kt @@ -0,0 +1,41 @@ +package xyz.block.trailblaze.revyl.tools + +import ai.koog.agents.core.tools.annotations.LLMDescription +import kotlinx.serialization.Serializable +import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolClass +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.util.Console + +/** + * Swipes on the Revyl cloud device in a given direction. + * + * Optionally starts from a targeted element. Returns the origin coordinates. + */ +@Serializable +@TrailblazeToolClass("revyl_swipe") +@LLMDescription( + "Swipe on the device screen in a direction (up, down, left, right). " + + "Optionally start from a specific element.", +) +class RevylNativeSwipeTool( + @param:LLMDescription("Swipe direction: 'up', 'down', 'left', or 'right'.") + val direction: String, + @param:LLMDescription("Optional element to start the swipe from.") + val target: String = "", + override val reasoning: String? = null, +) : RevylExecutableTool, ReasoningTrailblazeTool { + + override suspend fun executeWithRevyl( + client: RevylCliClient, + context: TrailblazeToolExecutionContext, + ): TrailblazeToolResult { + Console.log("### Swiping $direction") + val result = client.swipe(direction, target.ifBlank { null }) + val feedback = "Swiped $direction from (${result.x}, ${result.y})" + Console.log("### $feedback") + return TrailblazeToolResult.Success(message = feedback) + } +} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTapTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTapTool.kt new file mode 100644 index 00000000..869a9947 --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTapTool.kt @@ -0,0 +1,41 @@ +package xyz.block.trailblaze.revyl.tools + +import ai.koog.agents.core.tools.annotations.LLMDescription +import kotlinx.serialization.Serializable +import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolClass +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.util.Console + +/** + * Taps a UI element on the Revyl cloud device using natural language targeting. + * + * The Revyl CLI resolves the target description to screen coordinates via + * AI-powered visual grounding, then performs the tap. The resolved (x, y) + * coordinates are returned in the success message for overlay rendering. + */ +@Serializable +@TrailblazeToolClass("revyl_tap") +@LLMDescription( + "Tap a UI element on the device screen. Describe the element in natural language " + + "(e.g. 'Sign In button', 'search icon', 'first product card').", +) +class RevylNativeTapTool( + @param:LLMDescription("Element to tap, described in natural language.") + val target: String, + override val reasoning: String? = null, +) : RevylExecutableTool, ReasoningTrailblazeTool { + + override suspend fun executeWithRevyl( + client: RevylCliClient, + context: TrailblazeToolExecutionContext, + ): TrailblazeToolResult { + Console.log("### Tapping: $target") + val result = client.tapTarget(target) + val feedback = "Tapped '$target' at (${result.x}, ${result.y})" + Console.log("### $feedback") + return TrailblazeToolResult.Success(message = feedback) + } +} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeToolSet.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeToolSet.kt new file mode 100644 index 00000000..6c6a5843 --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeToolSet.kt @@ -0,0 +1,52 @@ +package xyz.block.trailblaze.revyl.tools + +import xyz.block.trailblaze.toolcalls.TrailblazeToolSet +import xyz.block.trailblaze.toolcalls.TrailblazeToolSet.DynamicTrailblazeToolSet +import xyz.block.trailblaze.toolcalls.commands.ObjectiveStatusTrailblazeTool + +/** + * Tool sets for the Revyl cloud device agent. + * + * Provides mobile-native tools that operate against Revyl cloud devices + * using natural language targeting and AI-powered visual grounding. + */ +object RevylNativeToolSet { + + /** Core tools for mobile interaction -- tap, type, swipe, navigate, etc. */ + val CoreToolSet = + DynamicTrailblazeToolSet( + name = "Revyl Native Core", + toolClasses = + setOf( + RevylNativeTapTool::class, + RevylNativeTypeTool::class, + RevylNativeSwipeTool::class, + RevylNativeLongPressTool::class, + RevylNativeScreenshotTool::class, + RevylNativeNavigateTool::class, + RevylNativeBackTool::class, + RevylNativePressKeyTool::class, + ObjectiveStatusTrailblazeTool::class, + ), + ) + + /** Revyl assertion tools for visual verification. */ + val AssertionToolSet = + DynamicTrailblazeToolSet( + name = "Revyl Native Assertions", + toolClasses = + setOf( + RevylNativeAssertTool::class, + ), + ) + + /** Full LLM tool set -- core tools plus assertions and memory tools. */ + val LlmToolSet = + DynamicTrailblazeToolSet( + name = "Revyl Native LLM", + toolClasses = + CoreToolSet.toolClasses + + AssertionToolSet.toolClasses + + TrailblazeToolSet.RememberTrailblazeToolSet.toolClasses, + ) +} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTypeTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTypeTool.kt new file mode 100644 index 00000000..b60c8122 --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTypeTool.kt @@ -0,0 +1,45 @@ +package xyz.block.trailblaze.revyl.tools + +import ai.koog.agents.core.tools.annotations.LLMDescription +import kotlinx.serialization.Serializable +import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolClass +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.util.Console + +/** + * Types text into an input field on the Revyl cloud device. + * + * Optionally targets a specific field by natural language description. + * When [clearFirst] is true, the field is cleared before typing. + */ +@Serializable +@TrailblazeToolClass("revyl_type") +@LLMDescription( + "Type text into an input field. Optionally specify a target field " + + "(e.g. 'email field', 'password input').", +) +class RevylNativeTypeTool( + @param:LLMDescription("The text to type into the field.") + val text: String, + @param:LLMDescription("Optional target field, described in natural language.") + val target: String = "", + @param:LLMDescription("If true, clear the field before typing.") + val clearFirst: Boolean = false, + override val reasoning: String? = null, +) : RevylExecutableTool, ReasoningTrailblazeTool { + + override suspend fun executeWithRevyl( + client: RevylCliClient, + context: TrailblazeToolExecutionContext, + ): TrailblazeToolResult { + val desc = if (target.isNotBlank()) "into '$target'" else "into focused field" + Console.log("### Typing '$text' $desc") + val result = client.typeText(text, target.ifBlank { null }, clearFirst) + val feedback = "Typed '$text' $desc at (${result.x}, ${result.y})" + Console.log("### $feedback") + return TrailblazeToolResult.Success(message = feedback) + } +} From e8feb0c3eb6d1840638f5f8d743ce1526661bd1f Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Tue, 24 Mar 2026 12:15:11 -0700 Subject: [PATCH 07/16] feat: add Revyl as selectable driver in desktop GUI - Add REVYL_ANDROID and REVYL_IOS to TrailblazeDriverType enum - Register both in OpenSourceTrailblazeDesktopAppConfig initialDriverTypes - Add device discovery in TrailblazeDeviceManager: cloud devices appear when REVYL_API_KEY is set (no local ADB/simulator needed) - Add Revyl types to targetDeviceFilter always-available set - Add getCurrentScreenState when branch for Revyl (uses RevylScreenState) - Fix trailblaze-revyl build.gradle.kts missing trailblaze-host dependency Made-with: Cursor Signed-off-by: Anam Hira --- .../OpenSourceTrailblazeDesktopAppConfig.kt | 2 ++ .../trailblaze/ui/TrailblazeDeviceManager.kt | 28 +++++++++++++++++++ .../devices/TrailblazeDriverType.kt | 8 ++++++ trailblaze-revyl/build.gradle.kts | 1 + 4 files changed, 39 insertions(+) diff --git a/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt b/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt index aec1bea0..c48bbc11 100644 --- a/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt +++ b/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt @@ -35,6 +35,8 @@ class OpenSourceTrailblazeDesktopAppConfig : TrailblazeDesktopAppConfig( TrailblazeDriverType.IOS_HOST, TrailblazeDriverType.PLAYWRIGHT_NATIVE, TrailblazeDriverType.PLAYWRIGHT_ELECTRON, + TrailblazeDriverType.REVYL_ANDROID, + TrailblazeDriverType.REVYL_IOS, ) // Start with no platforms enabled by default - user must explicitly enable them diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeDeviceManager.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeDeviceManager.kt index 6a14e3ef..68dfa49b 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeDeviceManager.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeDeviceManager.kt @@ -127,6 +127,8 @@ class TrailblazeDeviceManager( connectedDeviceSummary.trailblazeDriverType == TrailblazeDriverType.PLAYWRIGHT_NATIVE || connectedDeviceSummary.trailblazeDriverType == TrailblazeDriverType.PLAYWRIGHT_ELECTRON || connectedDeviceSummary.trailblazeDriverType == TrailblazeDriverType.COMPOSE || + connectedDeviceSummary.trailblazeDriverType == TrailblazeDriverType.REVYL_ANDROID || + connectedDeviceSummary.trailblazeDriverType == TrailblazeDriverType.REVYL_IOS || settingsRepo.getEnabledDriverTypes().contains(connectedDeviceSummary.trailblazeDriverType) } } @@ -377,6 +379,14 @@ class TrailblazeDeviceManager( Console.log("⚠️ Screen state capture not supported for ${driverType.name} driver") null } + TrailblazeDriverType.REVYL_ANDROID, + TrailblazeDriverType.REVYL_IOS -> { + val platform = if (driverType == TrailblazeDriverType.REVYL_ANDROID) "android" else "ios" + xyz.block.trailblaze.host.revyl.RevylScreenState( + xyz.block.trailblaze.host.revyl.RevylCliClient(), + platform, + ) + } } } @@ -581,6 +591,24 @@ class TrailblazeDeviceManager( ) ) } + + // Revyl cloud devices — available when REVYL_API_KEY is set. + if (System.getenv("REVYL_API_KEY") != null) { + add( + TrailblazeConnectedDeviceSummary( + trailblazeDriverType = TrailblazeDriverType.REVYL_ANDROID, + instanceId = "revyl-android-phone", + description = "Revyl Cloud Android Phone", + ) + ) + add( + TrailblazeConnectedDeviceSummary( + trailblazeDriverType = TrailblazeDriverType.REVYL_IOS, + instanceId = "revyl-ios-iphone", + description = "Revyl Cloud iOS iPhone", + ) + ) + } } val filteredDevices = if (applyDriverFilter) targetDeviceFilter(allDevices) else allDevices diff --git a/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/devices/TrailblazeDriverType.kt b/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/devices/TrailblazeDriverType.kt index 34eeb14c..becae776 100644 --- a/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/devices/TrailblazeDriverType.kt +++ b/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/devices/TrailblazeDriverType.kt @@ -28,6 +28,14 @@ enum class TrailblazeDriverType( platform = TrailblazeDevicePlatform.WEB, isHost = true, ), + REVYL_ANDROID( + platform = TrailblazeDevicePlatform.ANDROID, + isHost = true, + ), + REVYL_IOS( + platform = TrailblazeDevicePlatform.IOS, + isHost = true, + ), // COMPOSE intentionally uses WEB platform: Compose Desktop testing reuses the web // platform's view hierarchy filtering and device infrastructure. Adding a separate // DESKTOP platform would require updating all exhaustive `when` expressions on diff --git a/trailblaze-revyl/build.gradle.kts b/trailblaze-revyl/build.gradle.kts index 1b793c3e..4e1ba4cf 100644 --- a/trailblaze-revyl/build.gradle.kts +++ b/trailblaze-revyl/build.gradle.kts @@ -6,6 +6,7 @@ plugins { dependencies { api(project(":trailblaze-common")) api(project(":trailblaze-agent")) + implementation(project(":trailblaze-host")) implementation(project(":trailblaze-tracing")) implementation(libs.kotlinx.serialization.json) From efc17ebbf22efd6f3c4ea42defb119f260038211 Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Tue, 24 Mar 2026 17:17:50 -0700 Subject: [PATCH 08/16] =?UTF-8?q?feat:=20Revyl=20integration=20=E2=80=94?= =?UTF-8?q?=20device=20model=20fix,=20native=20steps,=20instruction/valida?= =?UTF-8?q?tion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix persisted device IDs without OS version crashing startSession - Add require() guard ensuring deviceModel and osVersion are both present or both absent - Handle trailing :: delimiter edge case with takeIf { isNotBlank() } - Remove unsupported --device-name CLI flag from startSession - Add RevylLiveStepResult data class for instruction/validation JSON responses - Add instruction() and validation() methods to RevylCliClient - Wire DirectionStep to instruction, VerificationStep to validation (opt-in via useRevylNativeSteps flag, defaults false) - Add useRevylNativeSteps config flag to TrailblazeConfig - Add RevylToolAgent with unit tests - Log parse failures in RevylActionResult.fromJson - Update docs/revyl-integration.md with native steps documentation Made-with: Cursor Signed-off-by: Anam Hira --- docs/revyl-integration.md | 38 +++ .../host/TrailblazeHostYamlRunner.kt | 248 ++++++++++++++++++ .../host/revyl/RevylActionResult.kt | 4 +- .../host/revyl/RevylBlazeSupport.kt | 24 +- .../trailblaze/host/revyl/RevylCliClient.kt | 89 ++++++- .../host/revyl/RevylLiveStepResult.kt | 60 +++++ .../host/revyl/RevylMcpServerFactory.kt | 24 +- .../host/revyl/RevylTrailblazeAgent.kt | 48 +++- .../trailblaze/ui/TrailblazeDeviceManager.kt | 46 +++- .../trailblaze/ui/TrailblazeSettingsRepo.kt | 29 +- .../block/trailblaze/host/revyl/RevylDemo.kt | 1 - .../trailblaze/model/TrailblazeConfig.kt | 6 + .../block/trailblaze/revyl/RevylToolAgent.kt | 27 +- .../trailblaze/revyl/RevylToolAgentTest.kt | 212 +++++++++++++++ 14 files changed, 797 insertions(+), 59 deletions(-) create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylLiveStepResult.kt create mode 100644 trailblaze-revyl/src/test/java/xyz/block/trailblaze/revyl/RevylToolAgentTest.kt diff --git a/docs/revyl-integration.md b/docs/revyl-integration.md index 9dfb9a6a..ba0e919c 100644 --- a/docs/revyl-integration.md +++ b/docs/revyl-integration.md @@ -114,6 +114,44 @@ All 12 Trailblaze tools are fully implemented: | openUrl | `revyl device navigate --url "..."` | | screenshot | `revyl device screenshot --out ` | +## Native instruction and validation steps + +Revyl devices can optionally use Revyl's own agent pipeline for natural-language +prompt steps instead of routing through Trailblaze's LLM agent. This is +controlled by the `useRevylNativeSteps` flag on `TrailblazeConfig` (default +`false` -- LLM pipeline is used until native step reporting with screenshots +is implemented). + +| YAML key | PromptStep type | CLI command | What happens | +|----------|-----------------|-------------|--------------| +| `- step:` | `DirectionStep` | `revyl device instruction "..."` | Revyl's worker agent plans and executes the action | +| `- verify:` | `VerificationStep` | `revyl device validation "..."` | Revyl's worker agent asserts against the current screen | + +When `useRevylNativeSteps` is `true`: + +- Each `- step:` prompt is sent directly to `revyl device instruction` in a + single round-trip. Revyl's agent handles grounding and execution natively. +- Each `- verify:` prompt is sent to `revyl device validation`, which performs + a visual assertion without needing a view hierarchy. +- Explicit tool steps (e.g. `- tapOn:`, `- inputText:`) are **not** affected + and still route through `RevylTrailblazeAgent` -> individual CLI commands. + +When `useRevylNativeSteps` is `false`: + +- All prompt steps go through Trailblaze's LLM agent pipeline, which decides + which tools to call (tap, type, swipe, etc.) and dispatches them via + `RevylTrailblazeAgent`. This is the original pre-flag behavior. + +```yaml +# Example trail using both step types +- prompts: + - step: Open the Search tab + - step: Type "beetle" in the search field + - verify: Search results are visible + - step: Tap the first result + - verify: Product detail page is shown +``` + ## Limitations - No local ADB or Maestro; all device interaction goes through Revyl cloud devices. diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt index 826292e7..3095ddfe 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt @@ -37,6 +37,9 @@ import xyz.block.trailblaze.devices.TrailblazeDriverType import xyz.block.trailblaze.exception.TrailblazeException import xyz.block.trailblaze.exception.TrailblazeSessionCancelledException import xyz.block.trailblaze.host.ios.MobileDeviceUtils +import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.host.revyl.RevylScreenState +import xyz.block.trailblaze.host.revyl.RevylTrailblazeAgent import xyz.block.trailblaze.host.rules.BaseComposeTest import xyz.block.trailblaze.host.rules.BaseHostTrailblazeTest import xyz.block.trailblaze.host.rules.BasePlaywrightElectronTest @@ -71,8 +74,10 @@ import xyz.block.trailblaze.ui.TrailblazeDeviceManager import xyz.block.trailblaze.util.Console import xyz.block.trailblaze.util.GitUtils import xyz.block.trailblaze.util.HostAndroidDeviceConnectUtils +import xyz.block.trailblaze.yaml.DirectionStep import xyz.block.trailblaze.yaml.ElectronAppConfig import xyz.block.trailblaze.yaml.TrailYamlItem +import xyz.block.trailblaze.yaml.VerificationStep import xyz.block.trailblaze.yaml.createTrailblazeYaml object TrailblazeHostYamlRunner { @@ -137,6 +142,9 @@ object TrailblazeHostYamlRunner { runPlaywrightElectronYaml(dynamicLlmClient, runOnHostParams, deviceManager) TrailblazeDriverType.COMPOSE -> runComposeYaml(dynamicLlmClient, runOnHostParams, deviceManager) + TrailblazeDriverType.REVYL_ANDROID, + TrailblazeDriverType.REVYL_IOS -> + runRevylYaml(dynamicLlmClient, runOnHostParams, deviceManager) else -> runMaestroHostYaml(dynamicLlmClient, runOnHostParams, deviceManager) } @@ -670,6 +678,246 @@ object TrailblazeHostYamlRunner { } } + /** + * Revyl cloud device path: provisions a device via [RevylCliClient] and runs + * the trail using [RevylTrailblazeAgent] with standard mobile tools. + * + * The CLI handles device provisioning, app install, and AI-powered target + * grounding. Screenshots come from [RevylScreenState]. + */ + private suspend fun runRevylYaml( + dynamicLlmClient: DynamicLlmClient, + runOnHostParams: RunOnHostParams, + deviceManager: TrailblazeDeviceManager, + ): SessionId? { + val onProgressMessage = runOnHostParams.onProgressMessage + val runYamlRequest = runOnHostParams.runYamlRequest + val trailblazeDeviceId = runYamlRequest.trailblazeDeviceId + val platform = if (runOnHostParams.trailblazeDriverType == TrailblazeDriverType.REVYL_ANDROID) "android" else "ios" + + val instanceId = trailblazeDeviceId.instanceId + val deviceLabel = if (instanceId.startsWith("revyl-model:")) + instanceId.removePrefix("revyl-model:") else "$platform (default)" + onProgressMessage("Provisioning Revyl cloud $deviceLabel...") + + val cliClient = RevylCliClient() + val session = if (instanceId.startsWith("revyl-model:")) { + val payload = instanceId.removePrefix("revyl-model:") + val parts = payload.split("::", limit = 2) + val modelName = parts[0] + val osVer = parts.getOrNull(1)?.takeIf { it.isNotBlank() } + if (osVer != null) { + cliClient.startSession(platform = platform, deviceModel = modelName, osVersion = osVer) + } else { + Console.log("RevylYaml: device '$modelName' missing OS version — using platform default") + cliClient.startSession(platform = platform) + } + } else { + cliClient.startSession(platform = platform) + } + onProgressMessage("Revyl $deviceLabel ready — viewer: ${session.viewerUrl}") + + // Outer try-finally guarantees the cloud device is stopped even if setup + // (e.g. LLM client creation) fails before the inner execution try block. + try { + val trailblazeDeviceInfo = TrailblazeDeviceInfo( + trailblazeDeviceId = trailblazeDeviceId, + trailblazeDriverType = runOnHostParams.trailblazeDriverType, + widthPixels = session.screenWidth.takeIf { it > 0 } ?: 1080, + heightPixels = session.screenHeight.takeIf { it > 0 } ?: 2340, + classifiers = listOf( + TrailblazeDeviceClassifier(platform), + TrailblazeDeviceClassifier("revyl-cloud"), + ), + ) + + val screenStateProvider: () -> ScreenState = { + RevylScreenState(cliClient, platform, session.screenWidth, session.screenHeight) + } + + val loggingRule = HostTrailblazeLoggingRule( + trailblazeDeviceInfoProvider = { trailblazeDeviceInfo }, + ) + + val agent = RevylTrailblazeAgent( + cliClient = cliClient, + platform = platform, + trailblazeLogger = loggingRule.logger, + trailblazeDeviceInfoProvider = { trailblazeDeviceInfo }, + sessionProvider = { + loggingRule.session ?: error("Session not available - ensure test is running") + }, + ) + + val toolRepo = TrailblazeToolRepo( + TrailblazeToolSet.getLlmToolSet(setOfMarkEnabled = false), + ) + + val trailblazeRunner = TrailblazeRunner( + screenStateProvider = screenStateProvider, + agent = agent, + llmClient = dynamicLlmClient.createLlmClient(), + trailblazeLlmModel = runYamlRequest.trailblazeLlmModel, + trailblazeToolRepo = toolRepo, + trailblazeLogger = loggingRule.logger, + sessionProvider = { + loggingRule.session ?: error("Session not available - ensure test is running") + }, + ) + + val elementComparator = TrailblazeElementComparator( + screenStateProvider = screenStateProvider, + llmClient = dynamicLlmClient.createLlmClient(), + trailblazeLlmModel = runYamlRequest.trailblazeLlmModel, + toolRepo = toolRepo, + ) + + val trailblazeYaml = createTrailblazeYaml() + + val trailblazeRunnerUtil = TrailblazeRunnerUtil( + trailblazeRunner = trailblazeRunner, + runTrailblazeTool = { trailblazeTools: List -> + val result = agent.runTrailblazeTools( + trailblazeTools, + null, + screenState = screenStateProvider(), + elementComparator = elementComparator, + screenStateProvider = screenStateProvider, + ) + when (val toolResult = result.result) { + is TrailblazeToolResult.Success -> toolResult + is TrailblazeToolResult.Error -> throw TrailblazeException(toolResult.errorMessage) + } + }, + trailblazeLogger = loggingRule.logger, + sessionProvider = { + loggingRule.session ?: error("Session not available - ensure test is running") + }, + ) + + val sessionManager = loggingRule.sessionManager + + val overrideSessionId = runYamlRequest.config.overrideSessionId + val trailblazeSession = if (overrideSessionId != null) { + sessionManager.createSessionWithId(overrideSessionId) + } else { + sessionManager.startSession(runYamlRequest.testName) + } + loggingRule.setSession(trailblazeSession) + + return try { + onProgressMessage("Executing YAML test via Revyl cloud device...") + Console.log("Starting Revyl execution for device: ${trailblazeDeviceId.instanceId}") + + val trailItems: List = trailblazeYaml.decodeTrail(runYamlRequest.yaml) + val trailConfig = trailblazeYaml.extractTrailConfig(trailItems) + + if (runYamlRequest.config.sendSessionStartLog) { + loggingRule.logger.log( + trailblazeSession, + TrailblazeLog.TrailblazeSessionStatusChangeLog( + sessionStatus = SessionStatus.Started( + trailConfig = trailConfig, + trailFilePath = runYamlRequest.trailFilePath, + testClassName = "Revyl", + testMethodName = "run", + trailblazeDeviceInfo = trailblazeDeviceInfo, + rawYaml = runYamlRequest.yaml, + hasRecordedSteps = trailblazeYaml.hasRecordedSteps(trailItems), + trailblazeDeviceId = trailblazeDeviceId, + ), + session = trailblazeSession.sessionId, + timestamp = Clock.System.now(), + ), + ) + } + + val useRevylNativeSteps = runYamlRequest.config.useRevylNativeSteps + for (item in trailItems) { + val itemResult = when (item) { + is TrailYamlItem.PromptsTrailItem -> { + if (useRevylNativeSteps) { + for (prompt in item.promptSteps) { + when (prompt) { + is VerificationStep -> { + Console.log("RevylYaml: validation — '${prompt.verify}'") + val result = cliClient.validation(prompt.verify) + if (!result.success) { + throw TrailblazeException( + "Validation failed: ${prompt.verify}" + + (result.statusReason?.let { " — $it" } ?: ""), + ) + } + } + is DirectionStep -> { + Console.log("RevylYaml: instruction — '${prompt.step}'") + val result = cliClient.instruction(prompt.step) + if (!result.success) { + throw TrailblazeException( + "Instruction failed: ${prompt.step}" + + (result.statusReason?.let { " — $it" } ?: ""), + ) + } + } + } + } + TrailblazeToolResult.Success() + } else { + trailblazeRunnerUtil.runPromptSuspend(item.promptSteps, runYamlRequest.useRecordedSteps) + } + } + is TrailYamlItem.ToolTrailItem -> + trailblazeRunnerUtil.runTrailblazeTool(item.tools.map { it.trailblazeTool }) + is TrailYamlItem.ConfigTrailItem -> + item.config.context?.let { trailblazeRunner.appendToSystemPrompt(it) } + } + if (itemResult is TrailblazeToolResult.Error) { + throw TrailblazeException(itemResult.errorMessage) + } + } + + Console.log("Revyl execution completed for device: ${trailblazeDeviceId.instanceId}") + onProgressMessage("Test execution completed successfully") + + if (runYamlRequest.config.sendSessionEndLog) { + sessionManager.endSession(trailblazeSession, isSuccess = true) + } + + generateAndSaveRecording(sessionId = trailblazeSession.sessionId, customToolClasses = emptySet()) + + trailblazeSession.sessionId + } catch (e: TrailblazeSessionCancelledException) { + Console.log("TrailblazeSessionCancelledException caught for device: ${trailblazeDeviceId.instanceId}") + onProgressMessage("Test session cancelled") + null + } catch (e: CancellationException) { + Console.log("CancellationException caught for device: ${trailblazeDeviceId.instanceId} - ${e.message}") + onProgressMessage("Test execution cancelled") + throw e + } catch (e: Exception) { + Console.log("Exception caught in runRevylYaml for device: ${trailblazeDeviceId.instanceId} - ${e::class.simpleName}: ${e.message}") + onProgressMessage("Test execution failed: ${e.message}") + captureFailureScreenshot(trailblazeSession, loggingRule, screenStateProvider) + sessionManager.endSession(trailblazeSession, isSuccess = false, exception = e) + null + } finally { + Console.log("Finally block executing for Revyl device: ${trailblazeDeviceId.instanceId}") + exportAndSaveTrace(trailblazeSession.sessionId, loggingRule) + loggingRule.setSession(null) + deviceManager.cancelSessionForDevice(trailblazeDeviceId) + Console.log("Finally block completed for Revyl device: ${trailblazeDeviceId.instanceId}") + } + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + Console.log("Revyl setup failed for device: ${trailblazeDeviceId.instanceId} - ${e::class.simpleName}: ${e.message}") + onProgressMessage("Error: ${e.message}") + return null + } finally { + try { cliClient.stopSession() } catch (_: Exception) { } + } + } + /** * Original Maestro-based path for Android/iOS/web-playwright-host devices. */ diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt index 52099ec2..a13af95e 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt @@ -5,6 +5,7 @@ import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.jsonPrimitive +import xyz.block.trailblaze.util.Console /** * Structured result from a Revyl CLI device action (tap, type, swipe, etc.). @@ -52,7 +53,8 @@ data class RevylActionResult( fun fromJson(jsonString: String): RevylActionResult { return try { lenientJson.decodeFromString(jsonString.trim()) - } catch (_: Exception) { + } catch (e: Exception) { + Console.log("RevylActionResult: JSON parse failed, using default: ${e.message}") RevylActionResult(success = true) } } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt index a664d64a..508607d7 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt @@ -5,6 +5,13 @@ import xyz.block.trailblaze.agent.BlazeConfig import xyz.block.trailblaze.agent.ScreenAnalyzer import xyz.block.trailblaze.agent.TrailblazeElementComparator import xyz.block.trailblaze.agent.blaze.BlazeGoalPlanner +import xyz.block.trailblaze.devices.TrailblazeDeviceId +import xyz.block.trailblaze.devices.TrailblazeDeviceInfo +import xyz.block.trailblaze.devices.TrailblazeDevicePlatform +import xyz.block.trailblaze.devices.TrailblazeDriverType +import xyz.block.trailblaze.logs.client.TrailblazeLogger +import xyz.block.trailblaze.logs.client.TrailblazeSession +import xyz.block.trailblaze.logs.model.SessionId import xyz.block.trailblaze.toolcalls.TrailblazeToolRepo /** @@ -56,7 +63,22 @@ object RevylBlazeSupport { elementComparator: TrailblazeElementComparator, config: BlazeConfig = BlazeConfig.DEFAULT, ): BlazeGoalPlanner { - val agent = RevylTrailblazeAgent(cliClient, platform) + val devicePlatform = if (platform == "ios") TrailblazeDevicePlatform.IOS else TrailblazeDevicePlatform.ANDROID + val deviceInfo = TrailblazeDeviceInfo( + trailblazeDeviceId = TrailblazeDeviceId(instanceId = "revyl-blaze", trailblazeDevicePlatform = devicePlatform), + trailblazeDriverType = if (platform == "ios") TrailblazeDriverType.REVYL_IOS else TrailblazeDriverType.REVYL_ANDROID, + widthPixels = 0, + heightPixels = 0, + ) + val agent = RevylTrailblazeAgent( + cliClient = cliClient, + platform = platform, + trailblazeLogger = TrailblazeLogger.createNoOp(), + trailblazeDeviceInfoProvider = { deviceInfo }, + sessionProvider = { + TrailblazeSession(sessionId = SessionId("revyl-blaze"), startTime = kotlinx.datetime.Clock.System.now()) + }, + ) val screenStateProvider = { RevylScreenState(cliClient, platform) } val executor = AgentUiActionExecutor( agent = agent, diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt index 8e7ac47c..5838569f 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt @@ -3,6 +3,7 @@ package xyz.block.trailblaze.host.revyl import kotlinx.serialization.json.Json import kotlinx.serialization.json.int import kotlinx.serialization.json.intOrNull +import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import xyz.block.trailblaze.util.Console @@ -170,21 +171,23 @@ class RevylCliClient( * @param platform "ios" or "android". * @param appUrl Optional public URL to an .apk/.ipa to install on start. * @param appLink Optional deep-link to open after launch. - * @param deviceName Optional device preset name (e.g. "revyl-android-phone"). - * When set, the CLI resolves the preset to a specific model and OS version. * @param deviceModel Optional explicit device model (e.g. "Pixel 7"). * @param osVersion Optional explicit OS version (e.g. "Android 14"). * @return The newly created [RevylSession] parsed from CLI JSON output. * @throws RevylCliException If the CLI exits with a non-zero code. + * @throws IllegalArgumentException If only one of deviceModel/osVersion is provided. */ fun startSession( platform: String, appUrl: String? = null, appLink: String? = null, - deviceName: String? = null, deviceModel: String? = null, osVersion: String? = null, ): RevylSession { + require(deviceModel.isNullOrBlank() == osVersion.isNullOrBlank()) { + "deviceModel and osVersion must both be provided or both be null/blank " + + "(got deviceModel=$deviceModel, osVersion=$osVersion)" + } val args = mutableListOf("device", "start", "--platform", platform.lowercase()) if (!appUrl.isNullOrBlank()) { args += listOf("--app-url", appUrl) @@ -192,9 +195,6 @@ class RevylCliClient( if (!appLink.isNullOrBlank()) { args += listOf("--app-link", appLink) } - if (!deviceName.isNullOrBlank()) { - args += listOf("--device-name", deviceName) - } if (!deviceModel.isNullOrBlank()) { args += listOf("--device-model", deviceModel) } @@ -437,6 +437,74 @@ class RevylCliClient( runCli(deviceArgs("network", if (connected) "--connected" else "--disconnected")) } + // --------------------------------------------------------------------------- + // Device catalog + // --------------------------------------------------------------------------- + + /** + * Queries the available device models from the Revyl backend catalog. + * + * Calls `revyl device targets --json` and parses the response into a + * deduplicated list of [RevylDeviceTarget] entries (one per unique model). + * + * @return Available device models grouped by platform. + * @throws RevylCliException If the CLI command fails. + */ + fun getDeviceTargets(): List { + val stdout = runCli(listOf("device", "targets")) + val root = json.parseToJsonElement(stdout).jsonObject + val results = mutableListOf() + val seenModels = mutableSetOf() + + for (platform in listOf("android", "ios")) { + val entries = root[platform]?.jsonArray ?: continue + for (entry in entries) { + val model = entry.jsonObject["Model"]?.jsonPrimitive?.content ?: continue + val runtime = entry.jsonObject["Runtime"]?.jsonPrimitive?.content ?: continue + if (seenModels.add("$platform:$model")) { + results.add(RevylDeviceTarget(platform = platform, model = model, osVersion = runtime)) + } + } + } + return results + } + + // --------------------------------------------------------------------------- + // High-level steps (instruction / validation) + // --------------------------------------------------------------------------- + + /** + * Executes a natural-language instruction step on the active device via + * `revyl device instruction "" --json`. + * + * Revyl's worker agent handles planning, grounding, and execution + * in a single round-trip. + * + * @param description Natural-language instruction (e.g. "Tap the Search tab"). + * @return Parsed [RevylLiveStepResult] with success flag and step output. + * @throws RevylCliException If the CLI process exits with a non-zero code. + */ + fun instruction(description: String): RevylLiveStepResult { + val stdout = runCli(deviceArgs("instruction", description)) + return RevylLiveStepResult.fromJson(stdout) + } + + /** + * Executes a natural-language validation step on the active device via + * `revyl device validation "" --json`. + * + * Revyl's worker agent performs a visual assertion against the current + * screen state and returns a pass/fail result. + * + * @param description Natural-language assertion (e.g. "The search results are visible"). + * @return Parsed [RevylLiveStepResult] with success flag and step output. + * @throws RevylCliException If the CLI process exits with a non-zero code. + */ + fun validation(description: String): RevylLiveStepResult { + val stdout = runCli(deviceArgs("validation", description)) + return RevylLiveStepResult.fromJson(stdout) + } + // --------------------------------------------------------------------------- // CLI execution // --------------------------------------------------------------------------- @@ -491,6 +559,15 @@ class RevylCliClient( */ class RevylCliException(message: String) : RuntimeException(message) +/** + * A device model available in the Revyl cloud catalog. + * + * @property platform "ios" or "android". + * @property model Human-readable model name (e.g. "iPhone 16", "Pixel 7"). + * @property osVersion Runtime / OS version string (e.g. "Android 14", "iOS 18.2"). + */ +data class RevylDeviceTarget(val platform: String, val model: String, val osVersion: String) + /** * Named device presets for provisioning Revyl cloud devices. * diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylLiveStepResult.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylLiveStepResult.kt new file mode 100644 index 00000000..d00b9a91 --- /dev/null +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylLiveStepResult.kt @@ -0,0 +1,60 @@ +package xyz.block.trailblaze.host.revyl + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonPrimitive +import xyz.block.trailblaze.util.Console + +/** + * Structured result from a Revyl CLI live-step command (`revyl device instruction` + * or `revyl device validation`). + * + * Maps to the `LiveStepResponse` struct returned by the Go CLI's `--json` output. + * The [stepOutput] object carries agent-specific detail such as `status`, + * `status_reason`, and `validation_result`. + * + * @property success Whether the step completed successfully. + * @property stepType The step type that was executed ("instruction" or "validation"). + * @property stepId Unique identifier assigned to the step by the worker. + * @property stepOutput Agent output detail including status reason and validation result. + */ +@Serializable +data class RevylLiveStepResult( + val success: Boolean = false, + @SerialName("step_type") val stepType: String = "", + @SerialName("step_id") val stepId: String = "", + @SerialName("step_output") val stepOutput: JsonObject? = null, +) { + + /** + * Human-readable reason for the step outcome, extracted from [stepOutput]. + * Returns null when the field is absent or the output is unparseable. + */ + val statusReason: String? + get() = stepOutput?.get("status_reason")?.jsonPrimitive?.contentOrNull + + companion object { + private val lenientJson = Json { ignoreUnknownKeys = true } + + /** + * Parses CLI JSON stdout into a [RevylLiveStepResult]. + * + * Falls back to a failure result if parsing fails so callers always + * get a non-null object to inspect. + * + * @param jsonString Raw JSON from `revyl device instruction/validation --json`. + * @return Parsed result with success flag and step output. + */ + fun fromJson(jsonString: String): RevylLiveStepResult { + return try { + lenientJson.decodeFromString(jsonString.trim()) + } catch (e: Exception) { + Console.log("RevylLiveStepResult: JSON parse failed: ${e.message}") + RevylLiveStepResult(success = false) + } + } + } +} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt index 288b1008..29f7f053 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt @@ -1,5 +1,12 @@ package xyz.block.trailblaze.host.revyl +import xyz.block.trailblaze.devices.TrailblazeDeviceId +import xyz.block.trailblaze.devices.TrailblazeDeviceInfo +import xyz.block.trailblaze.devices.TrailblazeDevicePlatform +import xyz.block.trailblaze.devices.TrailblazeDriverType +import xyz.block.trailblaze.logs.client.TrailblazeLogger +import xyz.block.trailblaze.logs.client.TrailblazeSession +import xyz.block.trailblaze.logs.model.SessionId import xyz.block.trailblaze.logs.server.TrailblazeMcpServer import xyz.block.trailblaze.model.TrailblazeHostAppTarget import xyz.block.trailblaze.report.utils.LogsRepo @@ -88,8 +95,23 @@ object RevylMcpServerFactory { } val primaryPlatform = platforms.first() + val devicePlatform = if (primaryPlatform == "ios") TrailblazeDevicePlatform.IOS else TrailblazeDevicePlatform.ANDROID + val deviceInfo = TrailblazeDeviceInfo( + trailblazeDeviceId = TrailblazeDeviceId(instanceId = "revyl-mcp", trailblazeDevicePlatform = devicePlatform), + trailblazeDriverType = if (primaryPlatform == "ios") TrailblazeDriverType.REVYL_IOS else TrailblazeDriverType.REVYL_ANDROID, + widthPixels = 0, + heightPixels = 0, + ) val revylDeviceService = RevylDeviceService(cliClient) - val agent = RevylTrailblazeAgent(cliClient, primaryPlatform) + val agent = RevylTrailblazeAgent( + cliClient = cliClient, + platform = primaryPlatform, + trailblazeLogger = TrailblazeLogger.createNoOp(), + trailblazeDeviceInfoProvider = { deviceInfo }, + sessionProvider = { + TrailblazeSession(sessionId = SessionId("revyl-mcp"), startTime = kotlinx.datetime.Clock.System.now()) + }, + ) val bridge = RevylMcpBridge(cliClient, revylDeviceService, agent, primaryPlatform) val logsDir = File(System.getProperty("user.dir"), ".trailblaze/logs") diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt index 28d2368d..401cfbf8 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt @@ -1,9 +1,14 @@ package xyz.block.trailblaze.host.revyl import maestro.SwipeDirection +import xyz.block.trailblaze.AgentMemory +import xyz.block.trailblaze.TrailblazeAgentContext import xyz.block.trailblaze.api.ScreenState import xyz.block.trailblaze.api.TrailblazeAgent import xyz.block.trailblaze.api.TrailblazeAgent.RunTrailblazeToolsResult +import xyz.block.trailblaze.devices.TrailblazeDeviceInfo +import xyz.block.trailblaze.logs.client.TrailblazeLogger +import xyz.block.trailblaze.logs.client.TrailblazeSessionProvider import xyz.block.trailblaze.logs.model.TraceId import xyz.block.trailblaze.toolcalls.TrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeToolResult @@ -38,11 +43,19 @@ import xyz.block.trailblaze.util.Console * * @property cliClient CLI-based client for Revyl device interactions. * @property platform "ios" or "android" — used for ScreenState construction. + * @property trailblazeLogger Logger for tool execution and snapshot events. + * @property trailblazeDeviceInfoProvider Provides device info for logging context. + * @property sessionProvider Provides current session for logging operations. */ class RevylTrailblazeAgent( private val cliClient: RevylCliClient, private val platform: String, -) : TrailblazeAgent { + override val trailblazeLogger: TrailblazeLogger, + override val trailblazeDeviceInfoProvider: () -> TrailblazeDeviceInfo, + override val sessionProvider: TrailblazeSessionProvider, +) : TrailblazeAgent, TrailblazeAgentContext { + + override val memory = AgentMemory() /** * Dispatches a list of [TrailblazeTool]s by mapping each tool to @@ -93,12 +106,14 @@ class RevylTrailblazeAgent( return try { when (tool) { is TapOnPointTrailblazeTool -> { + val target = tool.reasoning?.takeIf { it.isNotBlank() } if (tool.longPress) { - val r = cliClient.longPress("element at (${tool.x}, ${tool.y})") - TrailblazeToolResult.Success(message = "Long-pressed at (${r.x}, ${r.y})") + val desc = target ?: "element at (${tool.x}, ${tool.y})" + val r = cliClient.longPress(desc) + TrailblazeToolResult.Success(message = "Long-pressed '$desc' at (${r.x}, ${r.y})") } else { - val r = cliClient.tap(tool.x, tool.y) - TrailblazeToolResult.Success(message = "Tapped at (${r.x}, ${r.y})") + val r = if (target != null) cliClient.tapTarget(target) else cliClient.tap(tool.x, tool.y) + TrailblazeToolResult.Success(message = "Tapped '${target ?: "${tool.x},${tool.y}"}' at (${r.x}, ${r.y})") } } is InputTextTrailblazeTool -> { @@ -113,8 +128,9 @@ class RevylTrailblazeAgent( SwipeDirection.RIGHT -> "right" else -> "down" } - val r = cliClient.swipe(direction) - TrailblazeToolResult.Success(message = "Swiped $direction from (${r.x}, ${r.y})") + val target = tool.swipeOnElementText ?: "center of screen" + val r = cliClient.swipe(direction, target = target) + TrailblazeToolResult.Success(message = "Swiped $direction on '$target' from (${r.x}, ${r.y})") } is LaunchAppTrailblazeTool -> { cliClient.launchApp(tool.appId) @@ -148,16 +164,26 @@ class RevylTrailblazeAgent( TrailblazeToolResult.Success() } is ScrollUntilTextIsVisibleTrailblazeTool -> { - val r = cliClient.swipe("down") - TrailblazeToolResult.Success(message = "Scrolled down from (${r.x}, ${r.y})") + val direction = when (tool.direction) { + maestro.ScrollDirection.UP -> "up" + maestro.ScrollDirection.DOWN -> "down" + maestro.ScrollDirection.LEFT -> "left" + maestro.ScrollDirection.RIGHT -> "right" + else -> "down" + } + val target = tool.text.ifBlank { "center of screen" } + val r = cliClient.swipe(direction, target = target) + TrailblazeToolResult.Success(message = "Scrolled $direction on '$target' from (${r.x}, ${r.y})") } is NetworkConnectionTrailblazeTool -> { Console.log("RevylAgent: network toggle not yet implemented for cloud devices") TrailblazeToolResult.Success() } is TapOnElementByNodeIdTrailblazeTool -> { - val r = cliClient.tapTarget("element with node id ${tool.nodeId}") - TrailblazeToolResult.Success(message = "Tapped element at (${r.x}, ${r.y})") + val target = tool.reasoning?.takeIf { it.isNotBlank() } ?: "element with node id ${tool.nodeId}" + val r = if (tool.longPress) cliClient.longPress(target) else cliClient.tapTarget(target) + val action = if (tool.longPress) "Long-pressed" else "Tapped" + TrailblazeToolResult.Success(message = "$action '$target' at (${r.x}, ${r.y})") } is LongPressOnElementWithTextTrailblazeTool -> { val r = cliClient.longPress(tool.text) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeDeviceManager.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeDeviceManager.kt index 68dfa49b..545af507 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeDeviceManager.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeDeviceManager.kt @@ -592,22 +592,40 @@ class TrailblazeDeviceManager( ) } - // Revyl cloud devices — available when REVYL_API_KEY is set. - if (System.getenv("REVYL_API_KEY") != null) { - add( - TrailblazeConnectedDeviceSummary( - trailblazeDriverType = TrailblazeDriverType.REVYL_ANDROID, - instanceId = "revyl-android-phone", - description = "Revyl Cloud Android Phone", - ) + // Revyl cloud devices — default presets always included as fallbacks. + add( + TrailblazeConnectedDeviceSummary( + trailblazeDriverType = TrailblazeDriverType.REVYL_ANDROID, + instanceId = "revyl-android-phone", + description = "Revyl Android (Default)", ) - add( - TrailblazeConnectedDeviceSummary( - trailblazeDriverType = TrailblazeDriverType.REVYL_IOS, - instanceId = "revyl-ios-iphone", - description = "Revyl Cloud iOS iPhone", - ) + ) + add( + TrailblazeConnectedDeviceSummary( + trailblazeDriverType = TrailblazeDriverType.REVYL_IOS, + instanceId = "revyl-ios-iphone", + description = "Revyl iOS (Default)", ) + ) + + // Dynamically register each model from the Revyl device catalog. + // Uses `revyl device targets --json` so the list stays in sync + // with the backend without code changes. + try { + val targets = xyz.block.trailblaze.host.revyl.RevylCliClient().getDeviceTargets() + for (target in targets) { + val driverType = if (target.platform == "android") + TrailblazeDriverType.REVYL_ANDROID else TrailblazeDriverType.REVYL_IOS + add( + TrailblazeConnectedDeviceSummary( + trailblazeDriverType = driverType, + instanceId = "revyl-model:${target.model}::${target.osVersion}", + description = "Revyl ${target.model} (${target.osVersion})", + ) + ) + } + } catch (e: Exception) { + Console.log("Revyl device catalog unavailable: ${e.message}") } } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeSettingsRepo.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeSettingsRepo.kt index 233b1f2b..1fd7f39c 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeSettingsRepo.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeSettingsRepo.kt @@ -65,28 +65,19 @@ class TrailblazeSettingsRepo( fun applyTestingEnvironment(environment: TrailblazeServerState.TestingEnvironment) { updateAppConfig { config -> val existingDriverTypes = config.selectedTrailblazeDriverTypes - val driverTypes = when (environment) { - TrailblazeServerState.TestingEnvironment.MOBILE -> { - val defaultDriverTypes = mapOf( - TrailblazeDevicePlatform.ANDROID to TrailblazeDriverType.DEFAULT_ANDROID_ON_DEVICE, - TrailblazeDevicePlatform.IOS to TrailblazeDriverType.IOS_HOST, - ) - defaultDriverTypes.mapValues { (platform, defaultDriverType) -> - existingDriverTypes[platform] ?: defaultDriverType - } - } - TrailblazeServerState.TestingEnvironment.WEB -> { - val defaultDriverTypes = mapOf( - TrailblazeDevicePlatform.WEB to TrailblazeDriverType.PLAYWRIGHT_NATIVE, - ) - defaultDriverTypes.mapValues { (platform, defaultDriverType) -> - existingDriverTypes[platform] ?: defaultDriverType - } - } + val defaults = when (environment) { + TrailblazeServerState.TestingEnvironment.MOBILE -> mapOf( + TrailblazeDevicePlatform.ANDROID to TrailblazeDriverType.DEFAULT_ANDROID_ON_DEVICE, + TrailblazeDevicePlatform.IOS to TrailblazeDriverType.IOS_HOST, + ) + TrailblazeServerState.TestingEnvironment.WEB -> mapOf( + TrailblazeDevicePlatform.WEB to TrailblazeDriverType.PLAYWRIGHT_NATIVE, + ) } + val merged = existingDriverTypes + defaults.filterKeys { it !in existingDriverTypes } config.copy( testingEnvironment = environment, - selectedTrailblazeDriverTypes = driverTypes, + selectedTrailblazeDriverTypes = merged, showDevicesTab = environment == TrailblazeServerState.TestingEnvironment.WEB, ) } diff --git a/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt b/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt index f6b79e2f..49b46c99 100644 --- a/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt +++ b/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt @@ -32,7 +32,6 @@ fun main() { val session = client.startSession( platform = "android", appUrl = BUG_BAZAAR_APK, - deviceName = RevylDevicePreset.ANDROID_PHONE.presetId, ) println(" Viewer: ${session.viewerUrl}") println(" Screen: ${session.screenWidth}x${session.screenHeight}") diff --git a/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/model/TrailblazeConfig.kt b/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/model/TrailblazeConfig.kt index a26ea084..ed90ba26 100644 --- a/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/model/TrailblazeConfig.kt +++ b/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/model/TrailblazeConfig.kt @@ -27,6 +27,11 @@ const val AI_FALLBACK_DEFAULT: Boolean = false * if false, disables AI fallback (useful for debugging recorded steps). * @property browserHeadless If true, the Playwright browser runs headless (no visible window); * if false, the browser window is shown on screen. + * @property useRevylNativeSteps If true, Revyl device prompt steps use + * `revyl device instruction` / `validation` instead of the + * LLM agent pipeline. Defaults to false (LLM pipeline) until + * native step reporting with screenshots is implemented. + * Only affects REVYL_ANDROID/REVYL_IOS drivers. */ @Serializable data class TrailblazeConfig( @@ -37,6 +42,7 @@ data class TrailblazeConfig( val overrideSessionId: SessionId? = null, val aiFallback: Boolean = AI_FALLBACK_DEFAULT, val browserHeadless: Boolean = true, + val useRevylNativeSteps: Boolean = false, ) { companion object { /** diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylToolAgent.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylToolAgent.kt index 2a271577..533f6e03 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylToolAgent.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylToolAgent.kt @@ -1,13 +1,24 @@ package xyz.block.trailblaze.revyl +import kotlinx.datetime.Clock +import xyz.block.trailblaze.AgentMemory import xyz.block.trailblaze.api.ScreenState import xyz.block.trailblaze.api.TrailblazeAgent import xyz.block.trailblaze.api.TrailblazeAgent.RunTrailblazeToolsResult +import xyz.block.trailblaze.devices.TrailblazeDeviceId +import xyz.block.trailblaze.devices.TrailblazeDeviceInfo +import xyz.block.trailblaze.devices.TrailblazeDevicePlatform +import xyz.block.trailblaze.devices.TrailblazeDriverType import xyz.block.trailblaze.host.revyl.RevylCliClient import xyz.block.trailblaze.host.revyl.RevylScreenState +import xyz.block.trailblaze.logs.client.TrailblazeLogger +import xyz.block.trailblaze.logs.client.TrailblazeSession +import xyz.block.trailblaze.logs.client.TrailblazeSessionProvider +import xyz.block.trailblaze.logs.model.SessionId import xyz.block.trailblaze.logs.model.TraceId import xyz.block.trailblaze.revyl.tools.RevylExecutableTool import xyz.block.trailblaze.toolcalls.TrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult import xyz.block.trailblaze.toolcalls.commands.ObjectiveStatusTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.memory.MemoryTrailblazeTool @@ -94,16 +105,22 @@ class RevylToolAgent( private fun buildMinimalContext( screenStateProvider: () -> ScreenState, ): TrailblazeToolExecutionContext { + val devicePlatform = if (platform == "ios") TrailblazeDevicePlatform.IOS else TrailblazeDevicePlatform.ANDROID return TrailblazeToolExecutionContext( screenState = null, traceId = null, - trailblazeDeviceInfo = xyz.block.trailblaze.devices.TrailblazeDeviceInfo.EMPTY, - sessionProvider = object : xyz.block.trailblaze.logs.client.TrailblazeSessionProvider { - override fun getSessionId(): String = "" + trailblazeDeviceInfo = TrailblazeDeviceInfo( + trailblazeDeviceId = TrailblazeDeviceId(instanceId = "revyl", trailblazeDevicePlatform = devicePlatform), + trailblazeDriverType = if (platform == "ios") TrailblazeDriverType.IOS_HOST else TrailblazeDriverType.ANDROID_HOST, + widthPixels = 0, + heightPixels = 0, + ), + sessionProvider = TrailblazeSessionProvider { + TrailblazeSession(sessionId = SessionId("revyl-tool-agent"), startTime = Clock.System.now()) }, screenStateProvider = screenStateProvider, - trailblazeLogger = xyz.block.trailblaze.logs.client.TrailblazeLogger.NOOP, - memory = xyz.block.trailblaze.AgentMemory(), + trailblazeLogger = TrailblazeLogger.createNoOp(), + memory = AgentMemory(), ) } } diff --git a/trailblaze-revyl/src/test/java/xyz/block/trailblaze/revyl/RevylToolAgentTest.kt b/trailblaze-revyl/src/test/java/xyz/block/trailblaze/revyl/RevylToolAgentTest.kt new file mode 100644 index 00000000..3a147691 --- /dev/null +++ b/trailblaze-revyl/src/test/java/xyz/block/trailblaze/revyl/RevylToolAgentTest.kt @@ -0,0 +1,212 @@ +package xyz.block.trailblaze.revyl + +import assertk.assertThat +import assertk.assertions.hasSize +import assertk.assertions.isEqualTo +import assertk.assertions.isInstanceOf +import assertk.assertions.isTrue +import kotlinx.serialization.Serializable +import kotlinx.serialization.Transient +import org.junit.Test +import xyz.block.trailblaze.devices.TrailblazeDevicePlatform +import xyz.block.trailblaze.devices.TrailblazeDriverType +import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.revyl.tools.RevylExecutableTool +import xyz.block.trailblaze.toolcalls.TrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolClass +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.toolcalls.commands.BooleanAssertionTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.ObjectiveStatusTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.Status +import xyz.block.trailblaze.toolcalls.commands.StringEvaluationTrailblazeTool +import xyz.block.trailblaze.utils.ElementComparator +import kotlin.test.assertContains + +/** + * Unit tests for [RevylToolAgent] dispatch logic, error handling, + * and platform-specific context construction. + * + * Uses a stub [RevylExecutableTool] to avoid needing a real Revyl CLI + * binary or cloud device. The [RevylCliClient] is constructed with + * `/bin/echo` as the binary override so its init-time availability + * check passes without the real CLI installed. + */ +class RevylToolAgentTest { + + private val dummyClient = RevylCliClient("/bin/echo") + + private val noOpComparator = object : ElementComparator { + override fun getElementValue(prompt: String): String? = null + override fun evaluateBoolean(statement: String) = + BooleanAssertionTrailblazeTool(reason = statement, result = true) + override fun evaluateString(query: String) = + StringEvaluationTrailblazeTool(reason = query, result = "") + override fun extractNumberFromString(input: String): Double? = null + } + + // ── Stub tool that records execution and returns a configurable result ── + + @Serializable + @TrailblazeToolClass("stub_revyl_tool") + private class StubRevylTool( + @Transient private val result: TrailblazeToolResult = TrailblazeToolResult.Success(), + @Transient private val onExecute: ((TrailblazeToolExecutionContext) -> Unit)? = null, + ) : RevylExecutableTool { + @Transient var executed = false + + override suspend fun executeWithRevyl( + client: RevylCliClient, + context: TrailblazeToolExecutionContext, + ): TrailblazeToolResult { + executed = true + onExecute?.invoke(context) + return result + } + } + + @TrailblazeToolClass("stub_throwing_tool") + private class ThrowingRevylTool( + private val exception: Exception = RuntimeException("boom"), + ) : RevylExecutableTool { + override suspend fun executeWithRevyl( + client: RevylCliClient, + context: TrailblazeToolExecutionContext, + ): TrailblazeToolResult { + throw exception + } + } + + /** A tool type that RevylToolAgent does not recognize. */ + @Serializable + @TrailblazeToolClass("unknown_tool") + private class UnsupportedTool : TrailblazeTool + + // ── Helpers ── + + private fun runTools( + agent: RevylToolAgent, + vararg tools: TrailblazeTool, + ) = agent.runTrailblazeTools( + tools = tools.toList(), + traceId = null, + screenState = null, + elementComparator = noOpComparator, + screenStateProvider = null, + ) + + // ── Tests ── + + @Test + fun `dispatches RevylExecutableTool and returns Success`() { + val agent = RevylToolAgent(dummyClient, "android") + val tool = StubRevylTool() + + val result = runTools(agent, tool) + + assertThat(tool.executed).isTrue() + assertThat(result.result).isInstanceOf(TrailblazeToolResult.Success::class) + assertThat(result.executedTools).hasSize(1) + } + + @Test + fun `returns Success for ObjectiveStatusTrailblazeTool`() { + val agent = RevylToolAgent(dummyClient, "android") + val tool = ObjectiveStatusTrailblazeTool( + explanation = "test complete", + status = Status.COMPLETED, + ) + + val result = runTools(agent, tool) + + assertThat(result.result).isInstanceOf(TrailblazeToolResult.Success::class) + assertThat(result.executedTools).hasSize(1) + } + + @Test + fun `returns UnknownTrailblazeTool for unsupported tool type`() { + val agent = RevylToolAgent(dummyClient, "android") + val tool = UnsupportedTool() + + val result = runTools(agent, tool) + + assertThat(result.result).isInstanceOf(TrailblazeToolResult.Error.UnknownTrailblazeTool::class) + } + + @Test + fun `stops execution on first tool failure`() { + val agent = RevylToolAgent(dummyClient, "android") + val failingTool = StubRevylTool( + result = TrailblazeToolResult.Error.ExceptionThrown( + errorMessage = "simulated failure", + command = StubRevylTool(), + stackTrace = "", + ), + ) + val secondTool = StubRevylTool() + + val result = runTools(agent, failingTool, secondTool) + + assertThat(failingTool.executed).isTrue() + assertThat(secondTool.executed).isEqualTo(false) + assertThat(result.executedTools).hasSize(1) + assertThat(result.result).isInstanceOf(TrailblazeToolResult.Error::class) + } + + @Test + fun `catches exception and returns ExceptionThrown`() { + val agent = RevylToolAgent(dummyClient, "android") + val tool = ThrowingRevylTool(RuntimeException("something broke")) + + val result = runTools(agent, tool) + + assertThat(result.result).isInstanceOf(TrailblazeToolResult.Error.ExceptionThrown::class) + val error = result.result as TrailblazeToolResult.Error.ExceptionThrown + assertContains(error.errorMessage, "something broke") + } + + @Test + fun `builds context with IOS platform when platform is ios`() { + val agent = RevylToolAgent(dummyClient, "ios") + var capturedContext: TrailblazeToolExecutionContext? = null + val tool = StubRevylTool(onExecute = { capturedContext = it }) + + runTools(agent, tool) + + val info = capturedContext!!.trailblazeDeviceInfo + assertThat(info.trailblazeDriverType).isEqualTo(TrailblazeDriverType.IOS_HOST) + assertThat(info.trailblazeDeviceId.trailblazeDevicePlatform) + .isEqualTo(TrailblazeDevicePlatform.IOS) + } + + @Test + fun `builds context with ANDROID platform when platform is android`() { + val agent = RevylToolAgent(dummyClient, "android") + var capturedContext: TrailblazeToolExecutionContext? = null + val tool = StubRevylTool(onExecute = { capturedContext = it }) + + runTools(agent, tool) + + val info = capturedContext!!.trailblazeDeviceInfo + assertThat(info.trailblazeDriverType).isEqualTo(TrailblazeDriverType.ANDROID_HOST) + assertThat(info.trailblazeDeviceId.trailblazeDevicePlatform) + .isEqualTo(TrailblazeDevicePlatform.ANDROID) + } + + @Test + fun `executes multiple tools sequentially on success`() { + val agent = RevylToolAgent(dummyClient, "android") + val tool1 = StubRevylTool() + val tool2 = StubRevylTool() + val tool3 = StubRevylTool() + + val result = runTools(agent, tool1, tool2, tool3) + + assertThat(tool1.executed).isTrue() + assertThat(tool2.executed).isTrue() + assertThat(tool3.executed).isTrue() + assertThat(result.executedTools).hasSize(3) + assertThat(result.result).isInstanceOf(TrailblazeToolResult.Success::class) + assertThat(result.inputTools).hasSize(3) + } +} From be9de869c5f26941ca86d481af31a7d58dac2351 Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Tue, 24 Mar 2026 18:15:50 -0700 Subject: [PATCH 09/16] chore: remove unused RevylDevicePreset enum Signed-off-by: Anam Hira Made-with: Cursor --- .../trailblaze/host/revyl/RevylCliClient.kt | 131 ++++++++++-------- 1 file changed, 76 insertions(+), 55 deletions(-) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt index 5838569f..d7ab40fe 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt @@ -36,6 +36,9 @@ class RevylCliClient( private var activeSessionIndex: Int = 0 private var resolvedBinary: String = revylBinaryOverride ?: "revyl" + private val installDir = File(System.getProperty("user.home"), ".revyl/bin") + private val installerUrl = + "https://raw.githubusercontent.com/RevylAI/revyl-cli/main/scripts/install.sh" init { ensureRevylInstalled() @@ -80,24 +83,85 @@ class RevylCliClient( } // --------------------------------------------------------------------------- - // Auto-install + // Auto-install and auto-update // --------------------------------------------------------------------------- - private val installDir = File(System.getProperty("user.home"), ".revyl/bin") - private val installerUrl = - "https://raw.githubusercontent.com/RevylAI/revyl-cli/main/scripts/install.sh" - /** - * Ensures the revyl CLI binary is available. If not found on PATH, - * runs the official installer script to download and install it. + * Ensures the revyl CLI binary is available and up to date. * - * @throws RevylCliException If installation fails. + * If the binary is missing, runs the official installer. If installed + * but outdated compared to the latest GitHub release, re-runs the + * installer to upgrade. Network failures are logged and the existing + * binary is used as-is. + * + * @throws RevylCliException If installation fails and no binary is available. */ private fun ensureRevylInstalled() { - if (isRevylAvailable()) return + val installed = getInstalledVersion() + if (installed == null) { + Console.log("RevylCli: 'revyl' not found — installing...") + runInstaller() + return + } - Console.log("RevylCli: 'revyl' not found — installing via official installer...") + val latest = getLatestVersion() + if (latest != null && latest != installed) { + Console.log("RevylCli: upgrading $installed -> $latest") + try { + runInstaller() + } catch (e: Exception) { + Console.log("RevylCli: upgrade failed (${e.message}), continuing with $installed") + } + } else { + Console.log("RevylCli: $installed (up to date)") + } + } + + /** + * Returns the installed CLI version string (e.g. "v0.1.14"), or null + * if the binary is not found or not executable. + */ + private fun getInstalledVersion(): String? { + return try { + val process = ProcessBuilder(resolvedBinary, "--version") + .redirectErrorStream(true) + .start() + val output = process.inputStream.bufferedReader().readText().trim() + if (process.waitFor() == 0) { + output.substringAfterLast(" ", "").takeIf { it.startsWith("v") } + } else null + } catch (_: Exception) { + null + } + } + + /** + * Resolves the latest release version from GitHub (e.g. "v0.1.15") + * by following the /releases/latest redirect. Returns null on any + * network failure, timeout, or parse error. + */ + private fun getLatestVersion(): String? { + return try { + val url = java.net.URL("https://github.com/RevylAI/revyl-cli/releases/latest") + val conn = url.openConnection() as java.net.HttpURLConnection + conn.instanceFollowRedirects = false + conn.connectTimeout = 3000 + conn.readTimeout = 3000 + val location = conn.getHeaderField("Location") + conn.disconnect() + location?.substringAfterLast("/")?.takeIf { it.startsWith("v") } + } catch (_: Exception) { + null + } + } + /** + * Runs the official Revyl CLI installer script via curl. + * + * @throws RevylCliException If the installer exits with a non-zero code + * or the binary is not found after installation. + */ + private fun runInstaller() { try { val process = ProcessBuilder("sh", "-c", "curl -fsSL '$installerUrl' | sh") .redirectErrorStream(true) @@ -119,7 +183,8 @@ class RevylCliClient( val binaryPath = File(installDir, "revyl") if (binaryPath.exists() && binaryPath.canExecute()) { resolvedBinary = binaryPath.absolutePath - Console.log("RevylCli: installed to ${binaryPath.absolutePath}") + val newVersion = getInstalledVersion() ?: "unknown" + Console.log("RevylCli: installed $newVersion to ${binaryPath.absolutePath}") } else { throw RevylCliException( "Installer completed but binary not found at ${binaryPath.absolutePath}. " + @@ -135,30 +200,6 @@ class RevylCliClient( "or download from https://github.com/RevylAI/revyl-cli/releases" ) } - - if (!isRevylAvailable()) { - throw RevylCliException( - "Installed binary is not executable. " + - "Install manually: brew install RevylAI/tap/revyl" - ) - } - } - - /** - * Checks whether the resolved revyl binary is callable. - * - * @return true if `revyl --version` exits successfully. - */ - private fun isRevylAvailable(): Boolean { - return try { - val process = ProcessBuilder(resolvedBinary, "--version") - .redirectErrorStream(true) - .start() - process.inputStream.bufferedReader().readText() - process.waitFor() == 0 - } catch (_: Exception) { - false - } } // --------------------------------------------------------------------------- @@ -567,23 +608,3 @@ class RevylCliException(message: String) : RuntimeException(message) * @property osVersion Runtime / OS version string (e.g. "Android 14", "iOS 18.2"). */ data class RevylDeviceTarget(val platform: String, val model: String, val osVersion: String) - -/** - * Named device presets for provisioning Revyl cloud devices. - * - * Maps to CLI presets defined in `revyl-cli/internal/devicetargets/targets.go`. - * The CLI resolves each preset to the current default model and OS version from - * the backend device catalog, so consumers don't hardcode device strings. - * - * @property presetId CLI-level preset identifier passed via `--device-name`. - * @property platform Target platform ("ios" or "android"). - * @property displayName Human-readable label for UI display. - */ -enum class RevylDevicePreset( - val presetId: String, - val platform: String, - val displayName: String, -) { - ANDROID_PHONE("revyl-android-phone", "android", "Revyl Android Phone"), - IOS_IPHONE("revyl-ios-iphone", "ios", "Revyl iOS iPhone"), -} From a4d912fcb3a715dbc486dce1e204bb25756510b0 Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Tue, 24 Mar 2026 18:31:57 -0700 Subject: [PATCH 10/16] feat: add REVYL_API_KEY to settings UI and handle missing key gracefully - Add REVYL_API_KEY to customEnvVarNames so it appears in Settings > Environment Variables with the same masked value / Configure button UX as LLM provider keys - Check REVYL_API_KEY before session provisioning and surface clear error message instead of raw CLI exception - Wrap RevylCliClient construction and startSession in try-catch so provisioning failures return null with progress message instead of propagating Signed-off-by: Anam Hira Made-with: Cursor --- .../OpenSourceTrailblazeDesktopAppConfig.kt | 2 +- .../host/TrailblazeHostYamlRunner.kt | 37 +++++++++++++------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt b/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt index c48bbc11..089e14b4 100644 --- a/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt +++ b/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt @@ -78,7 +78,7 @@ class OpenSourceTrailblazeDesktopAppConfig : TrailblazeDesktopAppConfig( ALL_MODEL_LISTS.mapNotNull { trailblazeLlmModelList -> val trailblazeLlmProvider = trailblazeLlmModelList.provider JvmLLMProvidersUtil.getEnvironmentVariableKeyForLlmProvider(trailblazeLlmProvider) - } + } + "REVYL_API_KEY" override fun getCurrentlyAvailableLlmModelLists(): Set { val modelLists = JvmLLMProvidersUtil.getAvailableTrailblazeLlmProviderModelLists(ALL_MODEL_LISTS) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt index 3095ddfe..0161c19a 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt @@ -39,6 +39,7 @@ import xyz.block.trailblaze.exception.TrailblazeSessionCancelledException import xyz.block.trailblaze.host.ios.MobileDeviceUtils import xyz.block.trailblaze.host.revyl.RevylCliClient import xyz.block.trailblaze.host.revyl.RevylScreenState +import xyz.block.trailblaze.host.revyl.RevylSession import xyz.block.trailblaze.host.revyl.RevylTrailblazeAgent import xyz.block.trailblaze.host.rules.BaseComposeTest import xyz.block.trailblaze.host.rules.BaseHostTrailblazeTest @@ -698,22 +699,36 @@ object TrailblazeHostYamlRunner { val instanceId = trailblazeDeviceId.instanceId val deviceLabel = if (instanceId.startsWith("revyl-model:")) instanceId.removePrefix("revyl-model:") else "$platform (default)" + + if (System.getenv("REVYL_API_KEY").isNullOrBlank()) { + onProgressMessage("Error: REVYL_API_KEY is not set. Configure it in Settings → Environment Variables.") + return null + } + onProgressMessage("Provisioning Revyl cloud $deviceLabel...") - val cliClient = RevylCliClient() - val session = if (instanceId.startsWith("revyl-model:")) { - val payload = instanceId.removePrefix("revyl-model:") - val parts = payload.split("::", limit = 2) - val modelName = parts[0] - val osVer = parts.getOrNull(1)?.takeIf { it.isNotBlank() } - if (osVer != null) { - cliClient.startSession(platform = platform, deviceModel = modelName, osVersion = osVer) + val cliClient: RevylCliClient + val session: RevylSession + try { + cliClient = RevylCliClient() + session = if (instanceId.startsWith("revyl-model:")) { + val payload = instanceId.removePrefix("revyl-model:") + val parts = payload.split("::", limit = 2) + val modelName = parts[0] + val osVer = parts.getOrNull(1)?.takeIf { it.isNotBlank() } + if (osVer != null) { + cliClient.startSession(platform = platform, deviceModel = modelName, osVersion = osVer) + } else { + Console.log("RevylYaml: device '$modelName' missing OS version — using platform default") + cliClient.startSession(platform = platform) + } } else { - Console.log("RevylYaml: device '$modelName' missing OS version — using platform default") cliClient.startSession(platform = platform) } - } else { - cliClient.startSession(platform = platform) + } catch (e: Exception) { + Console.log("Revyl session provisioning failed: ${e::class.simpleName}: ${e.message}") + onProgressMessage("Error: ${e.message}") + return null } onProgressMessage("Revyl $deviceLabel ready — viewer: ${session.viewerUrl}") From d8bcf34cf4c937ee3cf4174b7145176a204f5d9d Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Tue, 24 Mar 2026 18:40:56 -0700 Subject: [PATCH 11/16] fix: always pass -s session index to prevent cross-contamination MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When multiple RevylCliClient instances run concurrently (e.g. Android + iOS in parallel), each instance only tracks its own session so sessions.size == 1. The old guard skipped the -s flag in that case, causing the CLI to use its global active session — which could be the other platform's device. Always passing -s when activeSessionIndex >= 0 ensures each client targets its own session regardless of global state. Signed-off-by: Anam Hira Made-with: Cursor --- .../main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt index d7ab40fe..a137ab7d 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt @@ -299,7 +299,7 @@ class RevylCliClient( private fun deviceArgs(vararg args: String): List { val base = mutableListOf("device") - if (sessions.size > 1) { + if (activeSessionIndex >= 0) { base += listOf("-s", activeSessionIndex.toString()) } base += args.toList() From 5bb8d3da95590ba386bda68210872b16d3ea25fa Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Tue, 24 Mar 2026 18:48:33 -0700 Subject: [PATCH 12/16] feat: surface Revyl viewer URL in session report header Pass the Revyl cloud device viewer URL through TrailblazeDeviceInfo metadata so it persists in session logs. Render it as a clickable "Open Revyl Viewer" link in the SessionDetailHeader when present, allowing users to jump directly to the live device screen. Signed-off-by: Anam Hira Made-with: Cursor --- .../host/TrailblazeHostYamlRunner.kt | 1 + .../ui/tabs/session/SessionDetailHeader.kt | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt index 0161c19a..90d09488 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt @@ -740,6 +740,7 @@ object TrailblazeHostYamlRunner { trailblazeDriverType = runOnHostParams.trailblazeDriverType, widthPixels = session.screenWidth.takeIf { it > 0 } ?: 1080, heightPixels = session.screenHeight.takeIf { it > 0 } ?: 2340, + metadata = mapOf("revyl_viewer_url" to session.viewerUrl), classifiers = listOf( TrailblazeDeviceClassifier(platform), TrailblazeDeviceClassifier("revyl-cloud"), diff --git a/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/SessionDetailHeader.kt b/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/SessionDetailHeader.kt index 17785d62..7af501da 100644 --- a/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/SessionDetailHeader.kt +++ b/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/SessionDetailHeader.kt @@ -38,6 +38,7 @@ import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalUriHandler import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp @@ -147,6 +148,23 @@ internal fun SessionDetailHeader( color = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.7f), ) } + // Revyl viewer link when running on a Revyl cloud device + val revylViewerUrl = sessionDetail.session.trailblazeDeviceInfo + ?.metadata?.get("revyl_viewer_url") + ?.takeIf { it.isNotBlank() } + if (revylViewerUrl != null) { + val uriHandler = LocalUriHandler.current + TextButton( + onClick = { uriHandler.openUri(revylViewerUrl) }, + contentPadding = ButtonDefaults.TextButtonContentPadding, + ) { + Text( + text = "Open Revyl Viewer", + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.primary, + ) + } + } } } } From a1b4bbb17b4dc2ded9205125b1144565ce3b852d Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Wed, 25 Mar 2026 08:54:35 -0700 Subject: [PATCH 13/16] Address PR #110 review comments from handstandsam Merge blockers: - Remove auto-install/auto-update of revyl CLI; replace with verifyCliAvailable() that shows install instructions on error and checkForUpgrade() that logs a non-blocking upgrade warning - Replace hardcoded Revyl viewer link in SessionDetailHeader with generic metadata-driven link system (any *_url key renders a link) - Remove Revyl section from root README.md (docs/revyl-integration.md already covers it) Code quality (per review comments): - RevylBlazeSupport: accept TrailblazeDevicePlatform enum instead of raw String; add default screen dimensions per platform - Rename getActiveSession() -> getActiveRevylSession() - RevylExecutableTool: convert from interface to abstract class, implement ReasoningTrailblazeTool, rename client -> revylClient - Merge RevylNativeLongPressTool into RevylNativeTapTool (longPress param) - Remove RevylNativeScreenshotTool from LLM tool set - Rename tool sets: CoreToolSet -> RevylCoreToolSet, etc. - Fix tool naming: revyl_press_key -> revyl_pressKey - Add ANDROID_BACK key support, iOS back clarification - Add swipe direction validation, assert examples for LLM - Use TrailblazeJsonInstance instead of private Json instances - Use File.createTempFile() for screenshot paths - Add ACTIVE_SESSION constant, REVYL_API_KEY_ENV constant - Extract platform-to-driver-type utils in RevylDeviceService - Fix build warnings: suppress deprecated LongPressOnElementWithText, remove redundant else branches in when expressions Made-with: Cursor Signed-off-by: Anam Hira --- README.md | 6 - .../OpenSourceTrailblazeDesktopAppConfig.kt | 2 +- .../host/TrailblazeHostYamlRunner.kt | 4 +- .../host/revyl/RevylActionResult.kt | 6 +- .../host/revyl/RevylBlazeSupport.kt | 29 ++- .../trailblaze/host/revyl/RevylCliClient.kt | 183 +++++++----------- .../host/revyl/RevylDeviceService.kt | 28 +-- .../host/revyl/RevylLiveStepResult.kt | 6 +- .../trailblaze/host/revyl/RevylMcpBridge.kt | 2 +- .../host/revyl/RevylTrailblazeAgent.kt | 4 +- .../revyl/tools/RevylExecutableTool.kt | 15 +- .../revyl/tools/RevylNativeAssertTool.kt | 15 +- .../revyl/tools/RevylNativeBackTool.kt | 16 +- .../revyl/tools/RevylNativeLongPressTool.kt | 34 ---- .../revyl/tools/RevylNativeNavigateTool.kt | 7 +- .../revyl/tools/RevylNativePressKeyTool.kt | 25 ++- .../revyl/tools/RevylNativeScreenshotTool.kt | 10 +- .../revyl/tools/RevylNativeSwipeTool.kt | 14 +- .../revyl/tools/RevylNativeTapTool.kt | 23 ++- .../revyl/tools/RevylNativeToolSet.kt | 15 +- .../revyl/tools/RevylNativeTypeTool.kt | 7 +- .../trailblaze/revyl/RevylToolAgentTest.kt | 11 +- .../ui/tabs/session/SessionDetailHeader.kt | 36 ++-- 23 files changed, 231 insertions(+), 267 deletions(-) delete mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeLongPressTool.kt diff --git a/README.md b/README.md index dc3b992b..aa4d09ba 100644 --- a/README.md +++ b/README.md @@ -66,12 +66,6 @@ Trailblaze's unique "**blaze once, trail forever**" workflow: └─────────────────────────────────────────────────────────────┘ ``` -### Cloud Device Support (Revyl) - -You can run Trailblaze against [Revyl](https://revyl.ai) cloud devices instead of local ADB or Maestro. Use -`RevylMcpServerFactory` to create an MCP server that provisions a device and maps Trailblaze tools to Revyl HTTP APIs. -See the [Revyl integration guide](docs/revyl-integration.md) for prerequisites, architecture, and usage. - ## Documentation at block.github.io/trailblaze See [Mobile-Agent-v3 Features Guide](docs/mobile-agent-v3-features.md) for detailed usage examples. diff --git a/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt b/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt index 089e14b4..83372d75 100644 --- a/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt +++ b/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt @@ -78,7 +78,7 @@ class OpenSourceTrailblazeDesktopAppConfig : TrailblazeDesktopAppConfig( ALL_MODEL_LISTS.mapNotNull { trailblazeLlmModelList -> val trailblazeLlmProvider = trailblazeLlmModelList.provider JvmLLMProvidersUtil.getEnvironmentVariableKeyForLlmProvider(trailblazeLlmProvider) - } + "REVYL_API_KEY" + } + xyz.block.trailblaze.host.revyl.RevylCliClient.REVYL_API_KEY_ENV override fun getCurrentlyAvailableLlmModelLists(): Set { val modelLists = JvmLLMProvidersUtil.getAvailableTrailblazeLlmProviderModelLists(ALL_MODEL_LISTS) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt index 90d09488..8ea04002 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt @@ -700,8 +700,8 @@ object TrailblazeHostYamlRunner { val deviceLabel = if (instanceId.startsWith("revyl-model:")) instanceId.removePrefix("revyl-model:") else "$platform (default)" - if (System.getenv("REVYL_API_KEY").isNullOrBlank()) { - onProgressMessage("Error: REVYL_API_KEY is not set. Configure it in Settings → Environment Variables.") + if (System.getenv(RevylCliClient.REVYL_API_KEY_ENV).isNullOrBlank()) { + onProgressMessage("Error: ${RevylCliClient.REVYL_API_KEY_ENV} is not set. Configure it in Settings → Environment Variables.") return null } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt index a13af95e..25a8b647 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt @@ -2,9 +2,9 @@ package xyz.block.trailblaze.host.revyl import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable -import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.jsonPrimitive +import xyz.block.trailblaze.logs.client.TrailblazeJsonInstance import xyz.block.trailblaze.util.Console /** @@ -39,8 +39,6 @@ data class RevylActionResult( val direction: String? = null, ) { companion object { - private val lenientJson = Json { ignoreUnknownKeys = true } - /** * Parses a CLI JSON stdout line into a [RevylActionResult]. * @@ -52,7 +50,7 @@ data class RevylActionResult( */ fun fromJson(jsonString: String): RevylActionResult { return try { - lenientJson.decodeFromString(jsonString.trim()) + TrailblazeJsonInstance.decodeFromString(jsonString.trim()) } catch (e: Exception) { Console.log("RevylActionResult: JSON parse failed, using default: ${e.message}") RevylActionResult(success = true) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt index 508607d7..3753a64d 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt @@ -48,7 +48,7 @@ object RevylBlazeSupport { * wires them into a [BlazeGoalPlanner] ready for AI-driven exploration. * * @param cliClient Authenticated CLI client with an active session. - * @param platform Device platform ("ios" or "android"). + * @param platform Device platform enum. * @param screenAnalyzer LLM-powered screen analyzer (provided by host). * @param toolRepo Tool repository for deserializing tool calls (provided by host). * @param elementComparator Element comparator for assertions (provided by host). @@ -57,29 +57,40 @@ object RevylBlazeSupport { */ fun createBlazeRunner( cliClient: RevylCliClient, - platform: String, + platform: TrailblazeDevicePlatform, screenAnalyzer: ScreenAnalyzer, toolRepo: TrailblazeToolRepo, elementComparator: TrailblazeElementComparator, config: BlazeConfig = BlazeConfig.DEFAULT, ): BlazeGoalPlanner { - val devicePlatform = if (platform == "ios") TrailblazeDevicePlatform.IOS else TrailblazeDevicePlatform.ANDROID + val driverType = when (platform) { + TrailblazeDevicePlatform.IOS -> TrailblazeDriverType.REVYL_IOS + else -> TrailblazeDriverType.REVYL_ANDROID + } + val defaultDimensions = when (platform) { + TrailblazeDevicePlatform.IOS -> Pair(1170, 2532) + else -> Pair(1080, 2400) + } val deviceInfo = TrailblazeDeviceInfo( - trailblazeDeviceId = TrailblazeDeviceId(instanceId = "revyl-blaze", trailblazeDevicePlatform = devicePlatform), - trailblazeDriverType = if (platform == "ios") TrailblazeDriverType.REVYL_IOS else TrailblazeDriverType.REVYL_ANDROID, - widthPixels = 0, - heightPixels = 0, + trailblazeDeviceId = TrailblazeDeviceId(instanceId = "revyl-blaze", trailblazeDevicePlatform = platform), + trailblazeDriverType = driverType, + widthPixels = defaultDimensions.first, + heightPixels = defaultDimensions.second, ) + val platformStr = when (platform) { + TrailblazeDevicePlatform.IOS -> "ios" + else -> "android" + } val agent = RevylTrailblazeAgent( cliClient = cliClient, - platform = platform, + platform = platformStr, trailblazeLogger = TrailblazeLogger.createNoOp(), trailblazeDeviceInfoProvider = { deviceInfo }, sessionProvider = { TrailblazeSession(sessionId = SessionId("revyl-blaze"), startTime = kotlinx.datetime.Clock.System.now()) }, ) - val screenStateProvider = { RevylScreenState(cliClient, platform) } + val screenStateProvider = { RevylScreenState(cliClient, platformStr) } val executor = AgentUiActionExecutor( agent = agent, screenStateProvider = screenStateProvider, diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt index a137ab7d..a6d0f8ef 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt @@ -1,11 +1,11 @@ package xyz.block.trailblaze.host.revyl -import kotlinx.serialization.json.Json import kotlinx.serialization.json.int import kotlinx.serialization.json.intOrNull import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive +import xyz.block.trailblaze.logs.client.TrailblazeJsonInstance import xyz.block.trailblaze.util.Console import java.io.File @@ -16,13 +16,12 @@ import java.io.File * executes it, and parses the structured JSON output. The CLI handles * auth, backend proxy routing, and AI-powered target grounding. * - * If the `revyl` binary is not found on PATH, it is automatically - * installed via the official installer script from GitHub. - * The only prerequisite is setting the `REVYL_API_KEY` environment variable. + * The `revyl` binary must be pre-installed on PATH (or pointed to via + * `REVYL_BINARY` env var). If not found, [startSession] throws a + * [RevylCliException] with install instructions. * * @property revylBinaryOverride Explicit path to the revyl binary. - * Defaults to `REVYL_BINARY` env var, then PATH lookup, then - * auto-install via install.sh. + * Defaults to `REVYL_BINARY` env var, then PATH lookup. * @property workingDirectory Optional working directory for CLI * invocations. Defaults to the JVM's current directory. */ @@ -31,23 +30,17 @@ class RevylCliClient( private val workingDirectory: File? = null, ) { - private val json = Json { ignoreUnknownKeys = true } + private val json = TrailblazeJsonInstance private val sessions = mutableMapOf() private var activeSessionIndex: Int = 0 - private var resolvedBinary: String = revylBinaryOverride ?: "revyl" - private val installDir = File(System.getProperty("user.home"), ".revyl/bin") - private val installerUrl = - "https://raw.githubusercontent.com/RevylAI/revyl-cli/main/scripts/install.sh" - - init { - ensureRevylInstalled() - } + private val resolvedBinary: String = revylBinaryOverride ?: "revyl" + private var cliVerified = false /** - * Returns the currently active session, or null if none has been started. + * Returns the currently active Revyl session, or null if none has been started. */ - fun getActiveSession(): RevylSession? = sessions[activeSessionIndex] + fun getActiveRevylSession(): RevylSession? = sessions[activeSessionIndex] /** * Returns the session at the given index, or null if not found. @@ -82,66 +75,60 @@ class RevylCliClient( Console.log("RevylCli: switched to session $index (${sessions[index]!!.platform})") } + companion object { + /** Sentinel value for "use the currently active session". */ + const val ACTIVE_SESSION = -1 + + /** Environment variable name for the Revyl API key. */ + const val REVYL_API_KEY_ENV = "REVYL_API_KEY" + } + // --------------------------------------------------------------------------- - // Auto-install and auto-update + // CLI verification // --------------------------------------------------------------------------- /** - * Ensures the revyl CLI binary is available and up to date. - * - * If the binary is missing, runs the official installer. If installed - * but outdated compared to the latest GitHub release, re-runs the - * installer to upgrade. Network failures are logged and the existing - * binary is used as-is. + * Verifies the revyl CLI binary is available on PATH. Called lazily on first + * device provisioning so construction never throws. * - * @throws RevylCliException If installation fails and no binary is available. + * @throws RevylCliException If the binary is not found, with install instructions. */ - private fun ensureRevylInstalled() { - val installed = getInstalledVersion() - if (installed == null) { - Console.log("RevylCli: 'revyl' not found — installing...") - runInstaller() - return - } - - val latest = getLatestVersion() - if (latest != null && latest != installed) { - Console.log("RevylCli: upgrading $installed -> $latest") - try { - runInstaller() - } catch (e: Exception) { - Console.log("RevylCli: upgrade failed (${e.message}), continuing with $installed") - } - } else { - Console.log("RevylCli: $installed (up to date)") - } - } - - /** - * Returns the installed CLI version string (e.g. "v0.1.14"), or null - * if the binary is not found or not executable. - */ - private fun getInstalledVersion(): String? { - return try { + private fun verifyCliAvailable() { + if (cliVerified) return + try { val process = ProcessBuilder(resolvedBinary, "--version") .redirectErrorStream(true) .start() val output = process.inputStream.bufferedReader().readText().trim() if (process.waitFor() == 0) { - output.substringAfterLast(" ", "").takeIf { it.startsWith("v") } - } else null - } catch (_: Exception) { - null - } + Console.log("RevylCli: $output") + cliVerified = true + val installedVersion = output.substringAfterLast(" ", "").takeIf { it.startsWith("v") } + if (installedVersion != null) checkForUpgrade(installedVersion) + return + } + } catch (_: Exception) { /* binary not found */ } + + throw RevylCliException( + "revyl CLI not found on PATH.\n\n" + + "Install:\n" + + " curl -fsSL https://raw.githubusercontent.com/RevylAI/revyl-cli/main/scripts/install.sh | sh\n\n" + + "Or with Homebrew:\n" + + " brew install RevylAI/tap/revyl\n\n" + + "Or download from:\n" + + " https://github.com/RevylAI/revyl-cli/releases\n\n" + + "Then set REVYL_API_KEY and try again." + ) } /** - * Resolves the latest release version from GitHub (e.g. "v0.1.15") - * by following the /releases/latest redirect. Returns null on any - * network failure, timeout, or parse error. + * Checks whether a newer CLI version is available on GitHub and logs + * a warning if so. Never throws -- network failures are silently ignored. + * + * @param installedVersion The currently installed version tag (e.g. "v0.1.14"). */ - private fun getLatestVersion(): String? { - return try { + private fun checkForUpgrade(installedVersion: String) { + try { val url = java.net.URL("https://github.com/RevylAI/revyl-cli/releases/latest") val conn = url.openConnection() as java.net.HttpURLConnection conn.instanceFollowRedirects = false @@ -149,56 +136,32 @@ class RevylCliClient( conn.readTimeout = 3000 val location = conn.getHeaderField("Location") conn.disconnect() - location?.substringAfterLast("/")?.takeIf { it.startsWith("v") } - } catch (_: Exception) { - null - } + val latest = location?.substringAfterLast("/")?.takeIf { it.startsWith("v") } ?: return + if (latest != installedVersion) { + Console.log( + "RevylCli: update available $installedVersion -> $latest.\n" + + " Upgrade: curl -fsSL https://raw.githubusercontent.com/RevylAI/revyl-cli/main/scripts/install.sh | sh\n" + + " Or: brew upgrade revyl" + ) + } + } catch (_: Exception) { /* network unavailable -- skip silently */ } } /** - * Runs the official Revyl CLI installer script via curl. - * - * @throws RevylCliException If the installer exits with a non-zero code - * or the binary is not found after installation. + * Returns true if the revyl CLI binary is available on PATH. + * Does not throw -- suitable for feature-gating Revyl device types. */ - private fun runInstaller() { - try { - val process = ProcessBuilder("sh", "-c", "curl -fsSL '$installerUrl' | sh") + fun isCliAvailable(): Boolean { + if (cliVerified) return true + return try { + val process = ProcessBuilder(resolvedBinary, "--version") .redirectErrorStream(true) - .also { pb -> - pb.environment()["REVYL_INSTALL_DIR"] = installDir.absolutePath - pb.environment()["REVYL_NO_MODIFY_PATH"] = "1" - } .start() - val output = process.inputStream.bufferedReader().readText() - val exitCode = process.waitFor() - - if (exitCode != 0) { - throw RevylCliException( - "Installer failed (exit $exitCode): ${output.take(500)}\n" + - "Install manually: brew install RevylAI/tap/revyl" - ) - } - - val binaryPath = File(installDir, "revyl") - if (binaryPath.exists() && binaryPath.canExecute()) { - resolvedBinary = binaryPath.absolutePath - val newVersion = getInstalledVersion() ?: "unknown" - Console.log("RevylCli: installed $newVersion to ${binaryPath.absolutePath}") - } else { - throw RevylCliException( - "Installer completed but binary not found at ${binaryPath.absolutePath}. " + - "Install manually: brew install RevylAI/tap/revyl" - ) - } - } catch (e: RevylCliException) { - throw e - } catch (e: Exception) { - throw RevylCliException( - "Auto-install failed: ${e.message}. " + - "Install manually: brew install RevylAI/tap/revyl " + - "or download from https://github.com/RevylAI/revyl-cli/releases" - ) + val available = process.waitFor() == 0 + if (available) cliVerified = true + available + } catch (_: Exception) { + false } } @@ -225,6 +188,7 @@ class RevylCliClient( deviceModel: String? = null, osVersion: String? = null, ): RevylSession { + verifyCliAvailable() require(deviceModel.isNullOrBlank() == osVersion.isNullOrBlank()) { "deviceModel and osVersion must both be provided or both be null/blank " + "(got deviceModel=$deviceModel, osVersion=$osVersion)" @@ -269,10 +233,10 @@ class RevylCliClient( /** * Stops a device session and removes it from the local session map. * - * @param index Session index to stop. Defaults to -1 (active session). + * @param index Session index to stop. Defaults to [ACTIVE_SESSION]. * @throws RevylCliException If the CLI exits with a non-zero code. */ - fun stopSession(index: Int = -1) { + fun stopSession(index: Int = ACTIVE_SESSION) { val targetIndex = if (index >= 0) index else activeSessionIndex val args = mutableListOf("device", "stop") if (targetIndex >= 0) args += listOf("-s", targetIndex.toString()) @@ -588,8 +552,7 @@ class RevylCliClient( } private fun createTempScreenshotPath(): String { - val tmpDir = System.getProperty("java.io.tmpdir") - return "$tmpDir/revyl-screenshot-${System.currentTimeMillis()}.png" + return File.createTempFile("revyl-screenshot-", ".png").absolutePath } } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt index 68e45ea8..e0b9085d 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt @@ -46,9 +46,9 @@ class RevylDeviceService( /** * Stops a device session by index. * - * @param index Session index to stop. Defaults to -1 (active session). + * @param index Session index to stop. Defaults to [RevylCliClient.ACTIVE_SESSION]. */ - fun stopDevice(index: Int = -1) { + fun stopDevice(index: Int = RevylCliClient.ACTIVE_SESSION) { cliClient.stopSession(index) } @@ -63,7 +63,7 @@ class RevylDeviceService( * Returns the [TrailblazeDeviceId] for the currently active session, or null. */ fun getCurrentDeviceId(): TrailblazeDeviceId? { - val session = cliClient.getActiveSession() ?: return null + val session = cliClient.getActiveRevylSession() ?: return null return sessionToDeviceId(session) } @@ -75,25 +75,27 @@ class RevylDeviceService( } private fun sessionToSummary(session: RevylSession): TrailblazeConnectedDeviceSummary { - val driverType = when (session.platform) { - "ios" -> TrailblazeDriverType.IOS_HOST - else -> TrailblazeDriverType.ANDROID_HOST - } return TrailblazeConnectedDeviceSummary( - trailblazeDriverType = driverType, + trailblazeDriverType = session.toDriverType(), instanceId = session.workflowRunId, description = "Revyl cloud ${session.platform} device (session ${session.index})", ) } private fun sessionToDeviceId(session: RevylSession): TrailblazeDeviceId { - val platform = when (session.platform) { - "ios" -> TrailblazeDevicePlatform.IOS - else -> TrailblazeDevicePlatform.ANDROID - } return TrailblazeDeviceId( instanceId = session.workflowRunId, - trailblazeDevicePlatform = platform, + trailblazeDevicePlatform = session.toDevicePlatform(), ) } } + +private fun RevylSession.toDriverType(): TrailblazeDriverType = when (platform) { + "ios" -> TrailblazeDriverType.REVYL_IOS + else -> TrailblazeDriverType.REVYL_ANDROID +} + +private fun RevylSession.toDevicePlatform(): TrailblazeDevicePlatform = when (platform) { + "ios" -> TrailblazeDevicePlatform.IOS + else -> TrailblazeDevicePlatform.ANDROID +} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylLiveStepResult.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylLiveStepResult.kt index d00b9a91..ff93c978 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylLiveStepResult.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylLiveStepResult.kt @@ -2,10 +2,10 @@ package xyz.block.trailblaze.host.revyl import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable -import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.jsonPrimitive +import xyz.block.trailblaze.logs.client.TrailblazeJsonInstance import xyz.block.trailblaze.util.Console /** @@ -37,8 +37,6 @@ data class RevylLiveStepResult( get() = stepOutput?.get("status_reason")?.jsonPrimitive?.contentOrNull companion object { - private val lenientJson = Json { ignoreUnknownKeys = true } - /** * Parses CLI JSON stdout into a [RevylLiveStepResult]. * @@ -50,7 +48,7 @@ data class RevylLiveStepResult( */ fun fromJson(jsonString: String): RevylLiveStepResult { return try { - lenientJson.decodeFromString(jsonString.trim()) + TrailblazeJsonInstance.decodeFromString(jsonString.trim()) } catch (e: Exception) { Console.log("RevylLiveStepResult: JSON parse failed: ${e.message}") RevylLiveStepResult(success = false) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt index 97a77ee3..bd401367 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt @@ -129,7 +129,7 @@ class RevylMcpBridge( } override suspend fun getCurrentScreenState(): ScreenState? { - val session = cliClient.getActiveSession() ?: return null + val session = cliClient.getActiveRevylSession() ?: return null return RevylScreenState( cliClient, session.platform, diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt index 401cfbf8..6e09b2a9 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt @@ -1,3 +1,5 @@ +@file:Suppress("DEPRECATION") + package xyz.block.trailblaze.host.revyl import maestro.SwipeDirection @@ -126,7 +128,6 @@ class RevylTrailblazeAgent( SwipeDirection.DOWN -> "down" SwipeDirection.LEFT -> "left" SwipeDirection.RIGHT -> "right" - else -> "down" } val target = tool.swipeOnElementText ?: "center of screen" val r = cliClient.swipe(direction, target = target) @@ -169,7 +170,6 @@ class RevylTrailblazeAgent( maestro.ScrollDirection.DOWN -> "down" maestro.ScrollDirection.LEFT -> "left" maestro.ScrollDirection.RIGHT -> "right" - else -> "down" } val target = tool.text.ifBlank { "center of screen" } val r = cliClient.swipe(direction, target = target) diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylExecutableTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylExecutableTool.kt index 2d1af478..df5e3ad7 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylExecutableTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylExecutableTool.kt @@ -2,31 +2,32 @@ package xyz.block.trailblaze.revyl.tools import xyz.block.trailblaze.host.revyl.RevylCliClient import xyz.block.trailblaze.toolcalls.ExecutableTrailblazeTool +import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult /** - * Interface for tools that execute against a Revyl cloud device via [RevylCliClient]. + * Base class for tools that execute against a Revyl cloud device via [RevylCliClient]. * * Analogous to PlaywrightExecutableTool, but uses natural language targets * resolved by Revyl's AI grounding instead of element IDs or ARIA descriptors. * Each tool returns resolved x,y coordinates so consumers like Trailblaze UI * can render click overlays. * - * The default [execute] implementation throws an error directing callers to use - * [RevylToolAgent], which calls [executeWithRevyl] directly. + * Subclasses implement [executeWithRevyl]; the default [execute] throws to + * direct callers through [RevylToolAgent] which calls [executeWithRevyl] directly. */ -interface RevylExecutableTool : ExecutableTrailblazeTool { +abstract class RevylExecutableTool : ExecutableTrailblazeTool, ReasoningTrailblazeTool { /** * Executes this tool against the given Revyl CLI client. * - * @param client The CLI client with an active device session. + * @param revylClient The CLI client with an active device session. * @param context The tool execution context with session, logging, and memory. * @return The result of tool execution, including coordinates when applicable. */ - suspend fun executeWithRevyl( - client: RevylCliClient, + abstract suspend fun executeWithRevyl( + revylClient: RevylCliClient, context: TrailblazeToolExecutionContext, ): TrailblazeToolResult diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt index 6f4705d0..1e08c3e5 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt @@ -3,7 +3,6 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable import xyz.block.trailblaze.host.revyl.RevylCliClient -import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult @@ -13,27 +12,27 @@ import xyz.block.trailblaze.util.Console * Runs a visual assertion on the Revyl cloud device screen. * * Uses Revyl's AI-powered validation step to verify a condition - * described in natural language (e.g. "the cart total is $42.99", - * "the Sign In button is visible"). + * described in natural language. */ @Serializable @TrailblazeToolClass("revyl_assert") @LLMDescription( - "Assert a visual condition on the device screen. Describe what should be true " + - "(e.g. 'the cart total shows $42.99', 'a success message is visible').", + "Assert a visual condition on the device screen. Describe what should be true. " + + "Examples: 'the cart total shows \$42.99', 'a success message is visible', " + + "'the Sign In button is disabled', 'there are at least 3 search results'.", ) class RevylNativeAssertTool( @param:LLMDescription("The condition to verify, described in natural language.") val assertion: String, override val reasoning: String? = null, -) : RevylExecutableTool, ReasoningTrailblazeTool { +) : RevylExecutableTool() { override suspend fun executeWithRevyl( - client: RevylCliClient, + revylClient: RevylCliClient, context: TrailblazeToolExecutionContext, ): TrailblazeToolResult { Console.log("### Asserting: $assertion") - val screenshot = client.screenshot() + val screenshot = revylClient.screenshot() val passed = screenshot.isNotEmpty() return if (passed) { TrailblazeToolResult.Success(message = "Assertion check: '$assertion' — screenshot captured for verification.") diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeBackTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeBackTool.kt index 754f87f5..5cf8d089 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeBackTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeBackTool.kt @@ -3,7 +3,6 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable import xyz.block.trailblaze.host.revyl.RevylCliClient -import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult @@ -11,20 +10,27 @@ import xyz.block.trailblaze.util.Console /** * Presses the device back button on the Revyl cloud device. + * + * On Android this triggers the system back navigation. On iOS the Revyl + * agent uses the app's UI-level back navigation (e.g. navigation bar + * back button) since there is no hardware back key. */ @Serializable @TrailblazeToolClass("revyl_back") -@LLMDescription("Press the device back button to go to the previous screen.") +@LLMDescription( + "Press the device back button to go to the previous screen. " + + "On Android, triggers the system back. On iOS, navigates back using the app's UI navigation.", +) class RevylNativeBackTool( override val reasoning: String? = null, -) : RevylExecutableTool, ReasoningTrailblazeTool { +) : RevylExecutableTool() { override suspend fun executeWithRevyl( - client: RevylCliClient, + revylClient: RevylCliClient, context: TrailblazeToolExecutionContext, ): TrailblazeToolResult { Console.log("### Pressing back") - val result = client.back() + val result = revylClient.back() return TrailblazeToolResult.Success(message = "Pressed back at (${result.x}, ${result.y})") } } diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeLongPressTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeLongPressTool.kt deleted file mode 100644 index cb6a791e..00000000 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeLongPressTool.kt +++ /dev/null @@ -1,34 +0,0 @@ -package xyz.block.trailblaze.revyl.tools - -import ai.koog.agents.core.tools.annotations.LLMDescription -import kotlinx.serialization.Serializable -import xyz.block.trailblaze.host.revyl.RevylCliClient -import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool -import xyz.block.trailblaze.toolcalls.TrailblazeToolClass -import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext -import xyz.block.trailblaze.toolcalls.TrailblazeToolResult -import xyz.block.trailblaze.util.Console - -/** - * Long-presses a UI element on the Revyl cloud device. - */ -@Serializable -@TrailblazeToolClass("revyl_long_press") -@LLMDescription("Long-press a UI element on the device screen.") -class RevylNativeLongPressTool( - @param:LLMDescription("Element to long-press, described in natural language.") - val target: String, - override val reasoning: String? = null, -) : RevylExecutableTool, ReasoningTrailblazeTool { - - override suspend fun executeWithRevyl( - client: RevylCliClient, - context: TrailblazeToolExecutionContext, - ): TrailblazeToolResult { - Console.log("### Long-pressing: $target") - val result = client.longPress(target) - val feedback = "Long-pressed '$target' at (${result.x}, ${result.y})" - Console.log("### $feedback") - return TrailblazeToolResult.Success(message = feedback) - } -} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeNavigateTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeNavigateTool.kt index c5b0da48..64a4c3e6 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeNavigateTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeNavigateTool.kt @@ -3,7 +3,6 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable import xyz.block.trailblaze.host.revyl.RevylCliClient -import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult @@ -19,14 +18,14 @@ class RevylNativeNavigateTool( @param:LLMDescription("The URL or deep link to open.") val url: String, override val reasoning: String? = null, -) : RevylExecutableTool, ReasoningTrailblazeTool { +) : RevylExecutableTool() { override suspend fun executeWithRevyl( - client: RevylCliClient, + revylClient: RevylCliClient, context: TrailblazeToolExecutionContext, ): TrailblazeToolResult { Console.log("### Navigating to: $url") - client.navigate(url) + revylClient.navigate(url) return TrailblazeToolResult.Success(message = "Navigated to $url") } } diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativePressKeyTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativePressKeyTool.kt index 1cb103be..446196e5 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativePressKeyTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativePressKeyTool.kt @@ -3,30 +3,37 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable import xyz.block.trailblaze.host.revyl.RevylCliClient -import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult import xyz.block.trailblaze.util.Console /** - * Sends a key press event (ENTER or BACKSPACE) on the Revyl cloud device. + * Sends a key press event on the Revyl cloud device. + * + * Supports ENTER, BACKSPACE, and ANDROID_BACK. The ANDROID_BACK key + * is routed through [RevylCliClient.back] for correct platform handling. */ @Serializable -@TrailblazeToolClass("revyl_press_key") -@LLMDescription("Press a key on the device keyboard (ENTER or BACKSPACE).") +@TrailblazeToolClass("revyl_pressKey") +@LLMDescription("Press a key on the device keyboard (ENTER, BACKSPACE, or ANDROID_BACK).") class RevylNativePressKeyTool( - @param:LLMDescription("Key to press: 'ENTER' or 'BACKSPACE'.") + @param:LLMDescription("Key to press: 'ENTER', 'BACKSPACE', or 'ANDROID_BACK'.") val key: String, override val reasoning: String? = null, -) : RevylExecutableTool, ReasoningTrailblazeTool { +) : RevylExecutableTool() { override suspend fun executeWithRevyl( - client: RevylCliClient, + revylClient: RevylCliClient, context: TrailblazeToolExecutionContext, ): TrailblazeToolResult { Console.log("### Pressing key: $key") - val result = client.pressKey(key) - return TrailblazeToolResult.Success(message = "Pressed $key at (${result.x}, ${result.y})") + val normalized = key.uppercase() + val result = if (normalized == "ANDROID_BACK") { + revylClient.back() + } else { + revylClient.pressKey(normalized) + } + return TrailblazeToolResult.Success(message = "Pressed $normalized at (${result.x}, ${result.y})") } } diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeScreenshotTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeScreenshotTool.kt index 63b576f6..c21a32fa 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeScreenshotTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeScreenshotTool.kt @@ -3,7 +3,6 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable import xyz.block.trailblaze.host.revyl.RevylCliClient -import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult @@ -11,20 +10,23 @@ import xyz.block.trailblaze.util.Console /** * Captures a screenshot of the Revyl cloud device screen. + * + * This tool is intended for internal use by the agent framework + * and is not included in the LLM-facing tool set. */ @Serializable @TrailblazeToolClass("revyl_screenshot") @LLMDescription("Take a screenshot of the current device screen to see what's on it.") class RevylNativeScreenshotTool( override val reasoning: String? = null, -) : RevylExecutableTool, ReasoningTrailblazeTool { +) : RevylExecutableTool() { override suspend fun executeWithRevyl( - client: RevylCliClient, + revylClient: RevylCliClient, context: TrailblazeToolExecutionContext, ): TrailblazeToolResult { Console.log("### Taking screenshot") - client.screenshot() + revylClient.screenshot() return TrailblazeToolResult.Success(message = "Screenshot captured.") } } diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeSwipeTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeSwipeTool.kt index f77a6da4..bbfe110b 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeSwipeTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeSwipeTool.kt @@ -3,7 +3,6 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable import xyz.block.trailblaze.host.revyl.RevylCliClient -import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult @@ -26,16 +25,23 @@ class RevylNativeSwipeTool( @param:LLMDescription("Optional element to start the swipe from.") val target: String = "", override val reasoning: String? = null, -) : RevylExecutableTool, ReasoningTrailblazeTool { +) : RevylExecutableTool() { override suspend fun executeWithRevyl( - client: RevylCliClient, + revylClient: RevylCliClient, context: TrailblazeToolExecutionContext, ): TrailblazeToolResult { + require(direction in VALID_DIRECTIONS) { + "Invalid swipe direction '$direction'. Must be one of: $VALID_DIRECTIONS" + } Console.log("### Swiping $direction") - val result = client.swipe(direction, target.ifBlank { null }) + val result = revylClient.swipe(direction, target.ifBlank { null }) val feedback = "Swiped $direction from (${result.x}, ${result.y})" Console.log("### $feedback") return TrailblazeToolResult.Success(message = feedback) } + + companion object { + private val VALID_DIRECTIONS = setOf("up", "down", "left", "right") + } } diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTapTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTapTool.kt index 869a9947..157ca52f 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTapTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTapTool.kt @@ -3,38 +3,43 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable import xyz.block.trailblaze.host.revyl.RevylCliClient -import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult import xyz.block.trailblaze.util.Console /** - * Taps a UI element on the Revyl cloud device using natural language targeting. + * Taps (or long-presses) a UI element on the Revyl cloud device using natural + * language targeting. * * The Revyl CLI resolves the target description to screen coordinates via - * AI-powered visual grounding, then performs the tap. The resolved (x, y) + * AI-powered visual grounding, then performs the action. The resolved (x, y) * coordinates are returned in the success message for overlay rendering. */ @Serializable @TrailblazeToolClass("revyl_tap") @LLMDescription( "Tap a UI element on the device screen. Describe the element in natural language " + - "(e.g. 'Sign In button', 'search icon', 'first product card').", + "(e.g. 'Sign In button', 'search icon', 'first product card'). " + + "Set longPress to true for a long-press gesture.", ) class RevylNativeTapTool( @param:LLMDescription("Element to tap, described in natural language.") val target: String, + @param:LLMDescription("If true, perform a long-press instead of a tap.") + val longPress: Boolean = false, override val reasoning: String? = null, -) : RevylExecutableTool, ReasoningTrailblazeTool { +) : RevylExecutableTool() { override suspend fun executeWithRevyl( - client: RevylCliClient, + revylClient: RevylCliClient, context: TrailblazeToolExecutionContext, ): TrailblazeToolResult { - Console.log("### Tapping: $target") - val result = client.tapTarget(target) - val feedback = "Tapped '$target' at (${result.x}, ${result.y})" + val action = if (longPress) "Long-pressing" else "Tapping" + Console.log("### $action: $target") + val result = if (longPress) revylClient.longPress(target) else revylClient.tapTarget(target) + val verb = if (longPress) "Long-pressed" else "Tapped" + val feedback = "$verb '$target' at (${result.x}, ${result.y})" Console.log("### $feedback") return TrailblazeToolResult.Success(message = feedback) } diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeToolSet.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeToolSet.kt index 6c6a5843..74bf61d4 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeToolSet.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeToolSet.kt @@ -1,6 +1,5 @@ package xyz.block.trailblaze.revyl.tools -import xyz.block.trailblaze.toolcalls.TrailblazeToolSet import xyz.block.trailblaze.toolcalls.TrailblazeToolSet.DynamicTrailblazeToolSet import xyz.block.trailblaze.toolcalls.commands.ObjectiveStatusTrailblazeTool @@ -13,7 +12,7 @@ import xyz.block.trailblaze.toolcalls.commands.ObjectiveStatusTrailblazeTool object RevylNativeToolSet { /** Core tools for mobile interaction -- tap, type, swipe, navigate, etc. */ - val CoreToolSet = + val RevylCoreToolSet = DynamicTrailblazeToolSet( name = "Revyl Native Core", toolClasses = @@ -21,8 +20,6 @@ object RevylNativeToolSet { RevylNativeTapTool::class, RevylNativeTypeTool::class, RevylNativeSwipeTool::class, - RevylNativeLongPressTool::class, - RevylNativeScreenshotTool::class, RevylNativeNavigateTool::class, RevylNativeBackTool::class, RevylNativePressKeyTool::class, @@ -31,7 +28,7 @@ object RevylNativeToolSet { ) /** Revyl assertion tools for visual verification. */ - val AssertionToolSet = + val RevylAssertionToolSet = DynamicTrailblazeToolSet( name = "Revyl Native Assertions", toolClasses = @@ -41,12 +38,12 @@ object RevylNativeToolSet { ) /** Full LLM tool set -- core tools plus assertions and memory tools. */ - val LlmToolSet = + val RevylLlmToolSet = DynamicTrailblazeToolSet( name = "Revyl Native LLM", toolClasses = - CoreToolSet.toolClasses + - AssertionToolSet.toolClasses + - TrailblazeToolSet.RememberTrailblazeToolSet.toolClasses, + RevylCoreToolSet.toolClasses + + RevylAssertionToolSet.toolClasses + + xyz.block.trailblaze.toolcalls.TrailblazeToolSet.RememberTrailblazeToolSet.toolClasses, ) } diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTypeTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTypeTool.kt index b60c8122..7626d1f4 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTypeTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTypeTool.kt @@ -3,7 +3,6 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable import xyz.block.trailblaze.host.revyl.RevylCliClient -import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult @@ -29,15 +28,15 @@ class RevylNativeTypeTool( @param:LLMDescription("If true, clear the field before typing.") val clearFirst: Boolean = false, override val reasoning: String? = null, -) : RevylExecutableTool, ReasoningTrailblazeTool { +) : RevylExecutableTool() { override suspend fun executeWithRevyl( - client: RevylCliClient, + revylClient: RevylCliClient, context: TrailblazeToolExecutionContext, ): TrailblazeToolResult { val desc = if (target.isNotBlank()) "into '$target'" else "into focused field" Console.log("### Typing '$text' $desc") - val result = client.typeText(text, target.ifBlank { null }, clearFirst) + val result = revylClient.typeText(text, target.ifBlank { null }, clearFirst) val feedback = "Typed '$text' $desc at (${result.x}, ${result.y})" Console.log("### $feedback") return TrailblazeToolResult.Success(message = feedback) diff --git a/trailblaze-revyl/src/test/java/xyz/block/trailblaze/revyl/RevylToolAgentTest.kt b/trailblaze-revyl/src/test/java/xyz/block/trailblaze/revyl/RevylToolAgentTest.kt index 3a147691..f46ae712 100644 --- a/trailblaze-revyl/src/test/java/xyz/block/trailblaze/revyl/RevylToolAgentTest.kt +++ b/trailblaze-revyl/src/test/java/xyz/block/trailblaze/revyl/RevylToolAgentTest.kt @@ -52,11 +52,12 @@ class RevylToolAgentTest { private class StubRevylTool( @Transient private val result: TrailblazeToolResult = TrailblazeToolResult.Success(), @Transient private val onExecute: ((TrailblazeToolExecutionContext) -> Unit)? = null, - ) : RevylExecutableTool { + ) : RevylExecutableTool() { @Transient var executed = false + override val reasoning: String? = null override suspend fun executeWithRevyl( - client: RevylCliClient, + revylClient: RevylCliClient, context: TrailblazeToolExecutionContext, ): TrailblazeToolResult { executed = true @@ -68,9 +69,11 @@ class RevylToolAgentTest { @TrailblazeToolClass("stub_throwing_tool") private class ThrowingRevylTool( private val exception: Exception = RuntimeException("boom"), - ) : RevylExecutableTool { + ) : RevylExecutableTool() { + override val reasoning: String? = null + override suspend fun executeWithRevyl( - client: RevylCliClient, + revylClient: RevylCliClient, context: TrailblazeToolExecutionContext, ): TrailblazeToolResult { throw exception diff --git a/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/SessionDetailHeader.kt b/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/SessionDetailHeader.kt index 7af501da..a39d229e 100644 --- a/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/SessionDetailHeader.kt +++ b/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/SessionDetailHeader.kt @@ -148,21 +148,29 @@ internal fun SessionDetailHeader( color = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.7f), ) } - // Revyl viewer link when running on a Revyl cloud device - val revylViewerUrl = sessionDetail.session.trailblazeDeviceInfo - ?.metadata?.get("revyl_viewer_url") - ?.takeIf { it.isNotBlank() } - if (revylViewerUrl != null) { + // Render external links from device metadata (convention: *_url keys become clickable links) + val metadata = sessionDetail.session.trailblazeDeviceInfo?.metadata.orEmpty() + val externalLinks = metadata.entries + .filter { it.key.endsWith("_url") && it.value.isNotBlank() } + .map { (key, url) -> + val prefix = key.removeSuffix("_url") + val label = metadata["${prefix}_label"] + ?: "Open ${prefix.replace('_', ' ').replaceFirstChar { c -> c.uppercase() }}" + label to url + } + if (externalLinks.isNotEmpty()) { val uriHandler = LocalUriHandler.current - TextButton( - onClick = { uriHandler.openUri(revylViewerUrl) }, - contentPadding = ButtonDefaults.TextButtonContentPadding, - ) { - Text( - text = "Open Revyl Viewer", - style = MaterialTheme.typography.labelSmall, - color = MaterialTheme.colorScheme.primary, - ) + for ((label, url) in externalLinks) { + TextButton( + onClick = { uriHandler.openUri(url) }, + contentPadding = ButtonDefaults.TextButtonContentPadding, + ) { + Text( + text = label, + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.primary, + ) + } } } } From 97433fc869a63f78502b8e06c7eaccdcc8769fcd Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Wed, 25 Mar 2026 09:01:51 -0700 Subject: [PATCH 14/16] fix: pass target to type/clear-text CLI commands to prevent rejection The Revyl CLI now requires --target or --x/--y for type and clear-text commands. InputTextTrailblazeTool and EraseTextTrailblazeTool don't carry a target field (the LLM focuses the field first via tap), so pass "focused input field" as a default target for AI grounding. Made-with: Cursor Signed-off-by: Anam Hira --- .../xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt index 6e09b2a9..39f2f24f 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt @@ -119,7 +119,7 @@ class RevylTrailblazeAgent( } } is InputTextTrailblazeTool -> { - val r = cliClient.typeText(tool.text) + val r = cliClient.typeText(tool.text, target = "focused input field") TrailblazeToolResult.Success(message = "Typed '${tool.text}' at (${r.x}, ${r.y})") } is SwipeTrailblazeTool -> { @@ -138,7 +138,7 @@ class RevylTrailblazeAgent( TrailblazeToolResult.Success(message = "Launched ${tool.appId}") } is EraseTextTrailblazeTool -> { - val r = cliClient.clearText() + val r = cliClient.clearText(target = "focused input field") TrailblazeToolResult.Success(message = "Cleared text at (${r.x}, ${r.y})") } is HideKeyboardTrailblazeTool -> { From 89c763170c9d5cb5d94379fcdcd2573f331b2550 Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Wed, 25 Mar 2026 11:52:48 -0700 Subject: [PATCH 15/16] feat: revyl CLI integration improvements + doubleTap tool - Fix --clear-first flag to always pass explicit value (CLI defaults to true) - Add appId and buildVersionId params to startSession() for Revyl build mgmt - Add revyl_doubleTap native tool for zoom/double-tap UI patterns - Move Revyl data classes from trailblaze-host to trailblaze-revyl module - Add RevylDefaults for platform-specific screen dimension fallbacks - Wire RevylNativeToolSet for all Revyl sessions (MCP, CLI, YAML runner) - InputTextTrailblazeTool fallback passes "text input field" target - Add ExternalLinkUtils for viewer URL handling in session UI Made-with: Cursor Signed-off-by: Anam Hira Made-with: Cursor --- docs/revyl-integration.md | 37 ++-- trailblaze-desktop/build.gradle.kts | 1 + .../desktop/OpenSourceTrailblazeDesktopApp.kt | 2 + .../OpenSourceTrailblazeDesktopAppConfig.kt | 3 +- trailblaze-host/build.gradle.kts | 1 + .../xyz/block/trailblaze/cli/TrailblazeCli.kt | 3 + .../host/TrailblazeHostYamlRunner.kt | 28 ++- .../host/revyl/RevylBlazeSupport.kt | 11 +- .../host/revyl/RevylDeviceService.kt | 101 ---------- .../trailblaze/host/revyl/RevylMcpBridge.kt | 189 ------------------ .../host/revyl/RevylMcpServerFactory.kt | 129 ------------ .../host/revyl/RevylTrailblazeAgent.kt | 36 +++- .../trailblaze/mcp/TrailblazeMcpBridgeImpl.kt | 9 +- .../trailblaze/ui/TrailblazeDeviceManager.kt | 10 +- .../block/trailblaze/host/revyl/RevylDemo.kt | 2 + trailblaze-revyl/build.gradle.kts | 1 - .../trailblaze}/revyl/RevylActionResult.kt | 3 +- .../block/trailblaze}/revyl/RevylCliClient.kt | 59 ++++-- .../block/trailblaze/revyl/RevylDefaults.kt | 30 +++ .../trailblaze}/revyl/RevylLiveStepResult.kt | 2 +- .../trailblaze}/revyl/RevylScreenState.kt | 13 +- .../block/trailblaze}/revyl/RevylSession.kt | 24 ++- .../block/trailblaze/revyl/RevylToolAgent.kt | 11 +- .../revyl/tools/RevylExecutableTool.kt | 2 +- .../revyl/tools/RevylNativeAssertTool.kt | 12 +- .../revyl/tools/RevylNativeBackTool.kt | 2 +- .../revyl/tools/RevylNativeDoubleTapTool.kt | 40 ++++ .../revyl/tools/RevylNativeNavigateTool.kt | 2 +- .../revyl/tools/RevylNativePressKeyTool.kt | 2 +- .../revyl/tools/RevylNativeScreenshotTool.kt | 32 --- .../revyl/tools/RevylNativeSwipeTool.kt | 2 +- .../revyl/tools/RevylNativeTapTool.kt | 2 +- .../revyl/tools/RevylNativeToolSet.kt | 1 + .../revyl/tools/RevylNativeTypeTool.kt | 2 +- .../trailblaze/revyl/RevylToolAgentTest.kt | 5 +- .../ui/tabs/session/ExternalLinkUtils.kt | 34 ++++ .../ui/tabs/session/SessionDetailHeader.kt | 17 +- 37 files changed, 305 insertions(+), 555 deletions(-) delete mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt delete mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt delete mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt rename {trailblaze-host/src/main/java/xyz/block/trailblaze/host => trailblaze-revyl/src/main/java/xyz/block/trailblaze}/revyl/RevylActionResult.kt (96%) rename {trailblaze-host/src/main/java/xyz/block/trailblaze/host => trailblaze-revyl/src/main/java/xyz/block/trailblaze}/revyl/RevylCliClient.kt (91%) create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylDefaults.kt rename {trailblaze-host/src/main/java/xyz/block/trailblaze/host => trailblaze-revyl/src/main/java/xyz/block/trailblaze}/revyl/RevylLiveStepResult.kt (98%) rename {trailblaze-host/src/main/java/xyz/block/trailblaze/host => trailblaze-revyl/src/main/java/xyz/block/trailblaze}/revyl/RevylScreenState.kt (92%) rename {trailblaze-host/src/main/java/xyz/block/trailblaze/host => trailblaze-revyl/src/main/java/xyz/block/trailblaze}/revyl/RevylSession.kt (53%) create mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeDoubleTapTool.kt delete mode 100644 trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeScreenshotTool.kt create mode 100644 trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/ExternalLinkUtils.kt diff --git a/docs/revyl-integration.md b/docs/revyl-integration.md index ba0e919c..0223ea08 100644 --- a/docs/revyl-integration.md +++ b/docs/revyl-integration.md @@ -10,23 +10,27 @@ Trailblaze can use [Revyl](https://revyl.ai) cloud devices instead of local ADB The Revyl integration provides: -- **RevylCliClient** – Shells out to the `revyl` CLI binary for all device interactions. Auto-downloads the CLI if not already installed. -- **RevylTrailblazeAgent** – Maps every Trailblaze tool to a `revyl device` CLI command. -- **RevylMcpServerFactory** – Builds an MCP server that provisions a Revyl cloud device and routes tool calls through the CLI. +- **RevylCliClient** (`trailblaze-revyl`) – Shells out to the `revyl` CLI binary for all device interactions. +- **RevylTrailblazeAgent** (`trailblaze-host`) – Maps every Trailblaze tool to a `revyl device` CLI command. +- **RevylNativeToolSet** (`trailblaze-revyl`) – Revyl-specific LLM tools (tap, type, swipe, assert) using natural language targeting and AI-powered visual grounding. -All integration code lives under `trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/`. +Core data classes and the CLI client live in the `trailblaze-revyl` module. Host-level wiring (agent, blaze support) lives in `trailblaze-host/.../host/revyl/`. ## Prerequisites -Set one environment variable: +1. Install the `revyl` CLI binary on your PATH: -- `REVYL_API_KEY` – Your Revyl API key (required). + ```bash + curl -fsSL https://raw.githubusercontent.com/RevylAI/revyl-cli/main/scripts/install.sh | sh + # Or with Homebrew: + brew install RevylAI/tap/revyl + ``` -That's it. The `revyl` CLI binary is **auto-downloaded** from [GitHub Releases](https://github.com/RevylAI/revyl-cli/releases) on first use if not already on PATH. No manual install needed. +2. Set the `REVYL_API_KEY` environment variable (or configure it in Settings > Environment Variables in the desktop app). **Optional overrides:** -- `REVYL_BINARY` – Path to a specific `revyl` binary (skips auto-download and PATH lookup). +- `REVYL_BINARY` – Path to a specific `revyl` binary (skips PATH lookup). ## Architecture @@ -62,8 +66,8 @@ flowchart LR ## Quick start ```kotlin -// Only prerequisite: set REVYL_API_KEY in your environment -val client = RevylCliClient() // auto-downloads revyl if not on PATH +// Prerequisites: revyl CLI on PATH + REVYL_API_KEY set +val client = RevylCliClient() // Start a cloud device with an app installed val session = client.startSession( @@ -84,17 +88,6 @@ client.screenshot("after-login.png") client.stopSession() ``` -## MCP server usage - -Use **RevylMcpServerFactory** to create an MCP server backed by Revyl: - -```kotlin -val server = RevylMcpServerFactory.create(platform = "android") -server.startStreamableHttpMcpServer(port = 8080, wait = true) -``` - -The factory auto-downloads the CLI, provisions a cloud device, and returns a **TrailblazeMcpServer** that speaks MCP. - ## Supported operations All 12 Trailblaze tools are fully implemented: @@ -156,7 +149,7 @@ When `useRevylNativeSteps` is `false`: - No local ADB or Maestro; all device interaction goes through Revyl cloud devices. - View hierarchy from Revyl is minimal (screenshot-based AI grounding is used instead). -- Requires network access to Revyl backend and GitHub (for auto-download on first use). +- Requires network access to the Revyl backend. ## See also diff --git a/trailblaze-desktop/build.gradle.kts b/trailblaze-desktop/build.gradle.kts index 814ef172..a030a6e2 100644 --- a/trailblaze-desktop/build.gradle.kts +++ b/trailblaze-desktop/build.gradle.kts @@ -42,6 +42,7 @@ dependencies { implementation(project(":trailblaze-common")) implementation(project(":trailblaze-compose")) implementation(project(":trailblaze-host")) + implementation(project(":trailblaze-revyl")) implementation(project(":trailblaze-models")) implementation(project(":trailblaze-report")) implementation(project(":trailblaze-server")) diff --git a/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopApp.kt b/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopApp.kt index a5536d69..5e263551 100644 --- a/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopApp.kt +++ b/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopApp.kt @@ -1,6 +1,7 @@ package xyz.block.trailblaze.desktop import xyz.block.trailblaze.compose.driver.tools.ComposeToolSet +import xyz.block.trailblaze.revyl.tools.RevylNativeToolSet import xyz.block.trailblaze.host.rules.TrailblazeHostDynamicLlmClientProvider import xyz.block.trailblaze.host.rules.TrailblazeHostDynamicLlmTokenProvider import xyz.block.trailblaze.host.yaml.DesktopYamlRunner @@ -31,6 +32,7 @@ class OpenSourceTrailblazeDesktopApp : TrailblazeDesktopApp( TrailblazeJsonInstance = TrailblazeJson.createTrailblazeJsonInstance( allToolClasses = TrailblazeToolSet.AllBuiltInTrailblazeToolsForSerializationByToolName + ComposeToolSet.toolClassesByToolName + + RevylNativeToolSet.RevylLlmToolSet.toolClasses.associateBy { it.toolName() } + desktopAppConfig.availableAppTargets.flatMap { it.getAllCustomToolClassesForSerialization() } .associateBy { it.toolName() }, ) diff --git a/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt b/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt index 83372d75..2e398258 100644 --- a/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt +++ b/trailblaze-desktop/src/main/java/xyz/block/trailblaze/desktop/OpenSourceTrailblazeDesktopAppConfig.kt @@ -13,6 +13,7 @@ import xyz.block.trailblaze.llm.providers.OpenRouterTrailblazeLlmModelList import xyz.block.trailblaze.mcp.utils.JvmLLMProvidersUtil import xyz.block.trailblaze.model.TrailblazeHostAppTarget import xyz.block.trailblaze.report.utils.LogsRepo +import xyz.block.trailblaze.revyl.RevylCliClient import xyz.block.trailblaze.ui.TrailblazeDesktopUtil import xyz.block.trailblaze.ui.TrailblazeSettingsRepo import xyz.block.trailblaze.ui.models.AppIconProvider @@ -78,7 +79,7 @@ class OpenSourceTrailblazeDesktopAppConfig : TrailblazeDesktopAppConfig( ALL_MODEL_LISTS.mapNotNull { trailblazeLlmModelList -> val trailblazeLlmProvider = trailblazeLlmModelList.provider JvmLLMProvidersUtil.getEnvironmentVariableKeyForLlmProvider(trailblazeLlmProvider) - } + xyz.block.trailblaze.host.revyl.RevylCliClient.REVYL_API_KEY_ENV + } + RevylCliClient.REVYL_API_KEY_ENV override fun getCurrentlyAvailableLlmModelLists(): Set { val modelLists = JvmLLMProvidersUtil.getAvailableTrailblazeLlmProviderModelLists(ALL_MODEL_LISTS) diff --git a/trailblaze-host/build.gradle.kts b/trailblaze-host/build.gradle.kts index 357151f1..ed1e34d4 100644 --- a/trailblaze-host/build.gradle.kts +++ b/trailblaze-host/build.gradle.kts @@ -90,6 +90,7 @@ dependencies { api(libs.slf4j.api) implementation(project(":trailblaze-common")) + implementation(project(":trailblaze-revyl")) implementation(project(":trailblaze-compose")) implementation(libs.compose.ui.test.junit4) implementation(project(":trailblaze-playwright")) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/cli/TrailblazeCli.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/cli/TrailblazeCli.kt index ce9e7f9d..d31536df 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/cli/TrailblazeCli.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/cli/TrailblazeCli.kt @@ -1271,6 +1271,9 @@ class RunCommand : Callable { BasePlaywrightElectronTest.ELECTRON_BUILT_IN_TOOL_CLASSES TrailblazeDriverType.COMPOSE -> ComposeToolSet.LlmToolSet.toolClasses + TrailblazeDriverType.REVYL_ANDROID, + TrailblazeDriverType.REVYL_IOS -> + xyz.block.trailblaze.revyl.tools.RevylNativeToolSet.RevylLlmToolSet.toolClasses else -> emptySet() } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt index 8ea04002..2e156795 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/TrailblazeHostYamlRunner.kt @@ -37,10 +37,10 @@ import xyz.block.trailblaze.devices.TrailblazeDriverType import xyz.block.trailblaze.exception.TrailblazeException import xyz.block.trailblaze.exception.TrailblazeSessionCancelledException import xyz.block.trailblaze.host.ios.MobileDeviceUtils -import xyz.block.trailblaze.host.revyl.RevylCliClient -import xyz.block.trailblaze.host.revyl.RevylScreenState -import xyz.block.trailblaze.host.revyl.RevylSession import xyz.block.trailblaze.host.revyl.RevylTrailblazeAgent +import xyz.block.trailblaze.revyl.RevylCliClient +import xyz.block.trailblaze.revyl.RevylScreenState +import xyz.block.trailblaze.revyl.RevylSession import xyz.block.trailblaze.host.rules.BaseComposeTest import xyz.block.trailblaze.host.rules.BaseHostTrailblazeTest import xyz.block.trailblaze.host.rules.BasePlaywrightElectronTest @@ -738,8 +738,10 @@ object TrailblazeHostYamlRunner { val trailblazeDeviceInfo = TrailblazeDeviceInfo( trailblazeDeviceId = trailblazeDeviceId, trailblazeDriverType = runOnHostParams.trailblazeDriverType, - widthPixels = session.screenWidth.takeIf { it > 0 } ?: 1080, - heightPixels = session.screenHeight.takeIf { it > 0 } ?: 2340, + widthPixels = session.screenWidth.takeIf { it > 0 } + ?: xyz.block.trailblaze.revyl.RevylDefaults.dimensionsForPlatform(platform).first, + heightPixels = session.screenHeight.takeIf { it > 0 } + ?: xyz.block.trailblaze.revyl.RevylDefaults.dimensionsForPlatform(platform).second, metadata = mapOf("revyl_viewer_url" to session.viewerUrl), classifiers = listOf( TrailblazeDeviceClassifier(platform), @@ -751,6 +753,11 @@ object TrailblazeHostYamlRunner { RevylScreenState(cliClient, platform, session.screenWidth, session.screenHeight) } + TrailblazeJsonInstance = TrailblazeJson.createTrailblazeJsonInstance( + allToolClasses = TrailblazeToolSet.AllBuiltInTrailblazeToolsForSerializationByToolName + + xyz.block.trailblaze.revyl.tools.RevylNativeToolSet.RevylLlmToolSet.toolClasses.associateBy { it.toolName() }, + ) + val loggingRule = HostTrailblazeLoggingRule( trailblazeDeviceInfoProvider = { trailblazeDeviceInfo }, ) @@ -766,7 +773,7 @@ object TrailblazeHostYamlRunner { ) val toolRepo = TrailblazeToolRepo( - TrailblazeToolSet.getLlmToolSet(setOfMarkEnabled = false), + xyz.block.trailblaze.revyl.tools.RevylNativeToolSet.RevylLlmToolSet, ) val trailblazeRunner = TrailblazeRunner( @@ -788,7 +795,9 @@ object TrailblazeHostYamlRunner { toolRepo = toolRepo, ) - val trailblazeYaml = createTrailblazeYaml() + val trailblazeYaml = createTrailblazeYaml( + customTrailblazeToolClasses = xyz.block.trailblaze.revyl.tools.RevylNativeToolSet.RevylLlmToolSet.toolClasses, + ) val trailblazeRunnerUtil = TrailblazeRunnerUtil( trailblazeRunner = trailblazeRunner, @@ -899,7 +908,10 @@ object TrailblazeHostYamlRunner { sessionManager.endSession(trailblazeSession, isSuccess = true) } - generateAndSaveRecording(sessionId = trailblazeSession.sessionId, customToolClasses = emptySet()) + generateAndSaveRecording( + sessionId = trailblazeSession.sessionId, + customToolClasses = xyz.block.trailblaze.revyl.tools.RevylNativeToolSet.RevylLlmToolSet.toolClasses, + ) trailblazeSession.sessionId } catch (e: TrailblazeSessionCancelledException) { diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt index 3753a64d..25458a4b 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylBlazeSupport.kt @@ -13,6 +13,9 @@ import xyz.block.trailblaze.logs.client.TrailblazeLogger import xyz.block.trailblaze.logs.client.TrailblazeSession import xyz.block.trailblaze.logs.model.SessionId import xyz.block.trailblaze.toolcalls.TrailblazeToolRepo +import xyz.block.trailblaze.revyl.RevylCliClient +import xyz.block.trailblaze.revyl.RevylDefaults +import xyz.block.trailblaze.revyl.RevylScreenState /** * Factory for wiring [BlazeGoalPlanner] to run on Revyl cloud devices. @@ -68,8 +71,8 @@ object RevylBlazeSupport { else -> TrailblazeDriverType.REVYL_ANDROID } val defaultDimensions = when (platform) { - TrailblazeDevicePlatform.IOS -> Pair(1170, 2532) - else -> Pair(1080, 2400) + TrailblazeDevicePlatform.IOS -> Pair(RevylDefaults.IOS_DEFAULT_WIDTH, RevylDefaults.IOS_DEFAULT_HEIGHT) + else -> Pair(RevylDefaults.ANDROID_DEFAULT_WIDTH, RevylDefaults.ANDROID_DEFAULT_HEIGHT) } val deviceInfo = TrailblazeDeviceInfo( trailblazeDeviceId = TrailblazeDeviceId(instanceId = "revyl-blaze", trailblazeDevicePlatform = platform), @@ -78,8 +81,8 @@ object RevylBlazeSupport { heightPixels = defaultDimensions.second, ) val platformStr = when (platform) { - TrailblazeDevicePlatform.IOS -> "ios" - else -> "android" + TrailblazeDevicePlatform.IOS -> RevylCliClient.PLATFORM_IOS + else -> RevylCliClient.PLATFORM_ANDROID } val agent = RevylTrailblazeAgent( cliClient = cliClient, diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt deleted file mode 100644 index e0b9085d..00000000 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylDeviceService.kt +++ /dev/null @@ -1,101 +0,0 @@ -package xyz.block.trailblaze.host.revyl - -import xyz.block.trailblaze.devices.TrailblazeConnectedDeviceSummary -import xyz.block.trailblaze.devices.TrailblazeDeviceId -import xyz.block.trailblaze.devices.TrailblazeDevicePlatform -import xyz.block.trailblaze.devices.TrailblazeDriverType - -/** - * Provisions and manages Revyl cloud device sessions via the CLI, - * serving as the device-listing layer for [RevylMcpBridge]. - * - * Supports multiple concurrent sessions -- each session is tracked - * in the underlying [RevylCliClient] session map. - * - * @property cliClient CLI-based client for Revyl device interactions. - */ -class RevylDeviceService( - private val cliClient: RevylCliClient, -) { - - /** - * Provisions a new cloud device via the Revyl CLI. - * The new session is added to the client's session map without - * replacing any existing sessions. - * - * @param platform "ios" or "android". - * @param appUrl Optional public download URL for an .apk/.ipa. - * @param appLink Optional deep-link to open after launch. - * @return A summary of the connected device. - * @throws RevylCliException If provisioning fails. - */ - fun startDevice( - platform: String, - appUrl: String? = null, - appLink: String? = null, - ): TrailblazeConnectedDeviceSummary { - val session = cliClient.startSession( - platform = platform, - appUrl = appUrl, - appLink = appLink, - ) - - return sessionToSummary(session) - } - - /** - * Stops a device session by index. - * - * @param index Session index to stop. Defaults to [RevylCliClient.ACTIVE_SESSION]. - */ - fun stopDevice(index: Int = RevylCliClient.ACTIVE_SESSION) { - cliClient.stopSession(index) - } - - /** - * Stops all active device sessions. - */ - fun stopAllDevices() { - cliClient.stopAllSessions() - } - - /** - * Returns the [TrailblazeDeviceId] for the currently active session, or null. - */ - fun getCurrentDeviceId(): TrailblazeDeviceId? { - val session = cliClient.getActiveRevylSession() ?: return null - return sessionToDeviceId(session) - } - - /** - * Returns summaries for all active device sessions. - */ - fun listDevices(): Set { - return cliClient.getAllSessions().values.map { sessionToSummary(it) }.toSet() - } - - private fun sessionToSummary(session: RevylSession): TrailblazeConnectedDeviceSummary { - return TrailblazeConnectedDeviceSummary( - trailblazeDriverType = session.toDriverType(), - instanceId = session.workflowRunId, - description = "Revyl cloud ${session.platform} device (session ${session.index})", - ) - } - - private fun sessionToDeviceId(session: RevylSession): TrailblazeDeviceId { - return TrailblazeDeviceId( - instanceId = session.workflowRunId, - trailblazeDevicePlatform = session.toDevicePlatform(), - ) - } -} - -private fun RevylSession.toDriverType(): TrailblazeDriverType = when (platform) { - "ios" -> TrailblazeDriverType.REVYL_IOS - else -> TrailblazeDriverType.REVYL_ANDROID -} - -private fun RevylSession.toDevicePlatform(): TrailblazeDevicePlatform = when (platform) { - "ios" -> TrailblazeDevicePlatform.IOS - else -> TrailblazeDevicePlatform.ANDROID -} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt deleted file mode 100644 index bd401367..00000000 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpBridge.kt +++ /dev/null @@ -1,189 +0,0 @@ -package xyz.block.trailblaze.host.revyl - -import xyz.block.trailblaze.api.ScreenState -import xyz.block.trailblaze.devices.TrailblazeConnectedDeviceSummary -import xyz.block.trailblaze.devices.TrailblazeDeviceId -import xyz.block.trailblaze.mcp.AgentImplementation -import xyz.block.trailblaze.mcp.TrailblazeMcpBridge -import xyz.block.trailblaze.model.TrailblazeHostAppTarget -import xyz.block.trailblaze.toolcalls.TrailblazeTool -import xyz.block.trailblaze.toolcalls.TrailblazeToolResult -import xyz.block.trailblaze.toolcalls.getToolNameFromAnnotation -import xyz.block.trailblaze.toolcalls.commands.BooleanAssertionTrailblazeTool -import xyz.block.trailblaze.toolcalls.commands.StringEvaluationTrailblazeTool -import xyz.block.trailblaze.util.Console -import xyz.block.trailblaze.utils.ElementComparator -import xyz.block.trailblaze.yaml.TrailYamlItem -import xyz.block.trailblaze.yaml.TrailblazeYaml - -/** - * [TrailblazeMcpBridge] backed by the Revyl CLI for cloud device interactions. - * - * Routes MCP tool calls through [RevylTrailblazeAgent] and provides device - * listing/selection via [RevylDeviceService]. Supports both trail replay - * (YAML tool execution) and blaze exploration (AI-driven via [BlazeGoalPlanner]). - * - * @property cliClient CLI-based client for Revyl device interactions. - * @property revylDeviceService Handles session provisioning and listing. - * @property agent The Trailblaze agent that dispatches tools via CLI. - * @property platform Device platform ("ios" or "android"). - */ -class RevylMcpBridge( - private val cliClient: RevylCliClient, - private val revylDeviceService: RevylDeviceService, - private val agent: RevylTrailblazeAgent, - private val platform: String = "android", -) : TrailblazeMcpBridge { - - override suspend fun selectDevice(trailblazeDeviceId: TrailblazeDeviceId): TrailblazeConnectedDeviceSummary { - val session = cliClient.getSession(trailblazeDeviceId.instanceId) - ?: error("No Revyl session found for device ${trailblazeDeviceId.instanceId}") - cliClient.useSession(session.index) - val devices = revylDeviceService.listDevices() - return devices.firstOrNull { it.trailblazeDeviceId == trailblazeDeviceId } - ?: error("Device ${trailblazeDeviceId.instanceId} not found in Revyl sessions.") - } - - override suspend fun getAvailableDevices(): Set { - return revylDeviceService.listDevices() - } - - override suspend fun getInstalledAppIds(): Set { - return emptySet() - } - - override fun getAvailableAppTargets(): Set { - return setOf(TrailblazeHostAppTarget.DefaultTrailblazeHostAppTarget) - } - - /** - * Executes a YAML trail on the Revyl cloud device. - * - * For trail/tool-based YAML: parses the YAML into [TrailblazeTool] instances - * and replays them sequentially via [executeTrailblazeTool]. - * - * For blaze mode ([AgentImplementation.MULTI_AGENT_V3]): constructs a - * [BlazeGoalPlanner] with [AgentUiActionExecutor] and runs AI-driven - * exploration against the cloud device. - * - * @param yaml Raw YAML content to execute. - * @param startNewSession Whether to start a fresh session (ignored for Revyl). - * @param agentImplementation Which agent implementation to use for execution. - * @return A summary string with the number of tools executed or blaze result. - */ - override suspend fun runYaml(yaml: String, startNewSession: Boolean, agentImplementation: AgentImplementation): String { - Console.log("RevylMcpBridge: runYaml invoked (implementation=$agentImplementation)") - - if (agentImplementation == AgentImplementation.MULTI_AGENT_V3) { - return blazeExecute(yaml) - } - - val trailblazeYaml = TrailblazeYaml.Default - val items = trailblazeYaml.decodeTrail(yaml) - - var executedCount = 0 - - val toolItems = items.filterIsInstance() - for (toolItem in toolItems) { - for (wrapper in toolItem.tools) { - executeTrailblazeTool(wrapper.trailblazeTool) - executedCount++ - } - } - - val promptItems = items.filterIsInstance() - for (promptItem in promptItems) { - for (step in promptItem.promptSteps) { - val tools = step.recording?.tools ?: continue - for (wrapper in tools) { - executeTrailblazeTool(wrapper.trailblazeTool) - executedCount++ - } - } - } - - Console.log("RevylMcpBridge: runYaml completed ($executedCount tools executed)") - return "completed:$executedCount" - } - - /** - * Stub for AI-driven blaze exploration via [RevylBlazeSupport]. - * - * Blaze mode requires an LLM-backed ScreenAnalyzer and TrailblazeToolRepo - * that the host application must provide. Use [RevylBlazeSupport.createBlazeRunner] - * to construct a fully configured [BlazeGoalPlanner] backed by Revyl cloud - * devices — it takes your existing LLM dependencies and returns a ready-to-run - * planner. - * - * @param yaml YAML containing blaze objectives. - * @return Status message directing callers to [RevylBlazeSupport]. - * @see RevylBlazeSupport.createBlazeRunner - */ - private suspend fun blazeExecute(yaml: String): String { - Console.log("RevylMcpBridge: blaze (V3) mode requested — use RevylBlazeSupport.createBlazeRunner() for host-level wiring") - return "blaze:use-RevylBlazeSupport.createBlazeRunner" - } - - override fun getCurrentlySelectedDeviceId(): TrailblazeDeviceId? { - return revylDeviceService.getCurrentDeviceId() - } - - override suspend fun getCurrentScreenState(): ScreenState? { - val session = cliClient.getActiveRevylSession() ?: return null - return RevylScreenState( - cliClient, - session.platform, - sessionScreenWidth = session.screenWidth, - sessionScreenHeight = session.screenHeight, - ) - } - - /** - * Executes a [TrailblazeTool] by delegating to [RevylTrailblazeAgent]. - * - * @param tool The tool to execute on the cloud device. - * @return A human-readable result description. - */ - override suspend fun executeTrailblazeTool(tool: TrailblazeTool): String { - val toolName = tool.getToolNameFromAnnotation() - Console.log("RevylMcpBridge: executing tool '$toolName'") - - val result = agent.runTrailblazeTools( - tools = listOf(tool), - elementComparator = NoOpElementComparator, - ) - - return when (val outcome = result.result) { - is TrailblazeToolResult.Success -> - outcome.message ?: "Successfully executed $toolName on Revyl cloud device." - is TrailblazeToolResult.Error -> - "Error executing $toolName: ${outcome.errorMessage}" - } - } - - override suspend fun endSession(): Boolean { - return try { - cliClient.stopSession() - true - } catch (_: Exception) { - false - } - } - - override fun selectAppTarget(appTargetId: String): String? { - return if (appTargetId == "none") "None" else null - } - - override fun getCurrentAppTargetId(): String? { - return "none" - } -} - -private object NoOpElementComparator : ElementComparator { - override fun getElementValue(prompt: String): String? = null - override fun evaluateBoolean(statement: String): BooleanAssertionTrailblazeTool = - BooleanAssertionTrailblazeTool(result = true, reason = "No-op comparator") - override fun evaluateString(query: String): StringEvaluationTrailblazeTool = - StringEvaluationTrailblazeTool(result = "", reason = "No-op comparator") - override fun extractNumberFromString(input: String): Double? = null -} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt deleted file mode 100644 index 29f7f053..00000000 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylMcpServerFactory.kt +++ /dev/null @@ -1,129 +0,0 @@ -package xyz.block.trailblaze.host.revyl - -import xyz.block.trailblaze.devices.TrailblazeDeviceId -import xyz.block.trailblaze.devices.TrailblazeDeviceInfo -import xyz.block.trailblaze.devices.TrailblazeDevicePlatform -import xyz.block.trailblaze.devices.TrailblazeDriverType -import xyz.block.trailblaze.logs.client.TrailblazeLogger -import xyz.block.trailblaze.logs.client.TrailblazeSession -import xyz.block.trailblaze.logs.model.SessionId -import xyz.block.trailblaze.logs.server.TrailblazeMcpServer -import xyz.block.trailblaze.model.TrailblazeHostAppTarget -import xyz.block.trailblaze.report.utils.LogsRepo -import xyz.block.trailblaze.util.Console -import java.io.File - -/** - * Factory for constructing a [TrailblazeMcpServer] backed by - * the Revyl CLI for cloud device interactions. - * - * Supports provisioning one or more devices at creation time. - * When multiple platforms are requested, a device is started for - * each platform and the MCP bridge can switch between them. - * - * Usage: - * ``` - * // Single device - * val server = RevylMcpServerFactory.create(platform = "android") - * - * // Both platforms - * val server = RevylMcpServerFactory.create(platforms = listOf("android", "ios")) - * - * server.startStreamableHttpMcpServer(port = 8080, wait = true) - * ``` - * - * The CLI binary must be on PATH (or set via REVYL_BINARY env var) - * and authenticated via REVYL_API_KEY or `revyl auth login`. - */ -object RevylMcpServerFactory { - - /** - * Creates a [TrailblazeMcpServer] that provisions a single Revyl cloud device. - * - * @param platform "ios" or "android". - * @param appUrl Optional public URL to an .apk/.ipa to install on the device. - * @param appLink Optional deep-link to open after launch. - * @param trailsDir Directory containing .trail YAML files. - * @return A configured [TrailblazeMcpServer] ready to start. - * @throws RevylCliException If device provisioning fails. - */ - fun create( - platform: String = "android", - appUrl: String? = null, - appLink: String? = null, - trailsDir: File = File(System.getProperty("user.dir"), "trails"), - ): TrailblazeMcpServer { - return create( - platforms = listOf(platform), - appUrl = appUrl, - appLink = appLink, - trailsDir = trailsDir, - ) - } - - /** - * Creates a [TrailblazeMcpServer] that provisions one or more Revyl cloud devices. - * - * When multiple platforms are provided, a device session is started for each. - * The MCP bridge supports switching between sessions via [RevylMcpBridge.selectDevice]. - * - * @param platforms List of platforms to provision (e.g. ["android", "ios"]). - * @param appUrl Optional public URL to an .apk/.ipa to install on each device. - * @param appLink Optional deep-link to open after launch. - * @param trailsDir Directory containing .trail YAML files. - * @return A configured [TrailblazeMcpServer] ready to start. - * @throws RevylCliException If any device provisioning fails. - */ - fun create( - platforms: List, - appUrl: String? = null, - appLink: String? = null, - trailsDir: File = File(System.getProperty("user.dir"), "trails"), - ): TrailblazeMcpServer { - require(platforms.isNotEmpty()) { "At least one platform must be specified" } - - Console.log("RevylMcpServerFactory: creating CLI-backed MCP server") - Console.log(" Platforms: ${platforms.joinToString(", ")}") - - val cliClient = RevylCliClient() - - for (p in platforms) { - Console.log("RevylMcpServerFactory: provisioning $p device via CLI...") - val session = cliClient.startSession(platform = p, appUrl = appUrl, appLink = appLink) - Console.log("RevylMcpServerFactory: $p device ready (session ${session.index})") - Console.log(" Viewer: ${session.viewerUrl}") - } - - val primaryPlatform = platforms.first() - val devicePlatform = if (primaryPlatform == "ios") TrailblazeDevicePlatform.IOS else TrailblazeDevicePlatform.ANDROID - val deviceInfo = TrailblazeDeviceInfo( - trailblazeDeviceId = TrailblazeDeviceId(instanceId = "revyl-mcp", trailblazeDevicePlatform = devicePlatform), - trailblazeDriverType = if (primaryPlatform == "ios") TrailblazeDriverType.REVYL_IOS else TrailblazeDriverType.REVYL_ANDROID, - widthPixels = 0, - heightPixels = 0, - ) - val revylDeviceService = RevylDeviceService(cliClient) - val agent = RevylTrailblazeAgent( - cliClient = cliClient, - platform = primaryPlatform, - trailblazeLogger = TrailblazeLogger.createNoOp(), - trailblazeDeviceInfoProvider = { deviceInfo }, - sessionProvider = { - TrailblazeSession(sessionId = SessionId("revyl-mcp"), startTime = kotlinx.datetime.Clock.System.now()) - }, - ) - val bridge = RevylMcpBridge(cliClient, revylDeviceService, agent, primaryPlatform) - - val logsDir = File(System.getProperty("user.dir"), ".trailblaze/logs") - logsDir.mkdirs() - val logsRepo = LogsRepo(logsDir) - - return TrailblazeMcpServer( - logsRepo = logsRepo, - mcpBridge = bridge, - trailsDirProvider = { trailsDir }, - targetTestAppProvider = { TrailblazeHostAppTarget.DefaultTrailblazeHostAppTarget }, - llmModelListsProvider = { emptySet() }, - ) - } -} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt index 39f2f24f..0ad81c95 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylTrailblazeAgent.kt @@ -30,10 +30,15 @@ import xyz.block.trailblaze.toolcalls.commands.TapOnPointTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.WaitForIdleSyncTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.NetworkConnectionTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.LongPressOnElementWithTextTrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.commands.memory.MemoryTrailblazeTool import xyz.block.trailblaze.toolcalls.getToolNameFromAnnotation import xyz.block.trailblaze.utils.ElementComparator import xyz.block.trailblaze.util.Console +import xyz.block.trailblaze.revyl.RevylCliClient +import xyz.block.trailblaze.revyl.RevylDefaults +import xyz.block.trailblaze.revyl.RevylScreenState +import xyz.block.trailblaze.revyl.tools.RevylExecutableTool /** * [TrailblazeAgent] implementation that routes all device actions through @@ -119,7 +124,7 @@ class RevylTrailblazeAgent( } } is InputTextTrailblazeTool -> { - val r = cliClient.typeText(tool.text, target = "focused input field") + val r = cliClient.typeText(tool.text, target = "text input field") TrailblazeToolResult.Success(message = "Typed '${tool.text}' at (${r.x}, ${r.y})") } is SwipeTrailblazeTool -> { @@ -189,6 +194,11 @@ class RevylTrailblazeAgent( val r = cliClient.longPress(tool.text) TrailblazeToolResult.Success(message = "Long-pressed '${tool.text}' at (${r.x}, ${r.y})") } + is RevylExecutableTool -> { + kotlinx.coroutines.runBlocking { + tool.executeWithRevyl(cliClient, buildRevylToolContext(screenStateProvider)) + } + } is ObjectiveStatusTrailblazeTool -> TrailblazeToolResult.Success() is MemoryTrailblazeTool -> TrailblazeToolResult.Success() else -> { @@ -205,4 +215,28 @@ class RevylTrailblazeAgent( ) } } + + /** + * Builds a [TrailblazeToolExecutionContext] for executing [RevylExecutableTool] + * instances, using this agent's logger, session provider, and device info. + * + * @param screenStateProvider Optional lazy provider for fresh screenshots. + * @return A context suitable for Revyl native tool execution. + */ + private fun buildRevylToolContext( + screenStateProvider: (() -> ScreenState)?, + ): TrailblazeToolExecutionContext { + val deviceInfo = trailblazeDeviceInfoProvider() + val effectiveScreenStateProvider = screenStateProvider + ?: { RevylScreenState(cliClient, platform) } + return TrailblazeToolExecutionContext( + screenState = null, + traceId = null, + trailblazeDeviceInfo = deviceInfo, + sessionProvider = sessionProvider, + screenStateProvider = effectiveScreenStateProvider, + trailblazeLogger = trailblazeLogger, + memory = memory, + ) + } } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/mcp/TrailblazeMcpBridgeImpl.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/mcp/TrailblazeMcpBridgeImpl.kt index bf921bd0..5b61aa68 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/mcp/TrailblazeMcpBridgeImpl.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/mcp/TrailblazeMcpBridgeImpl.kt @@ -835,10 +835,11 @@ class TrailblazeMcpBridgeImpl( } override fun getInnerAgentBuiltInToolClasses(): Set> { - return if (getDriverType() == TrailblazeDriverType.PLAYWRIGHT_NATIVE) { - PlaywrightNativeToolSet.LlmToolSet.toolClasses - } else { - emptySet() + return when (getDriverType()) { + TrailblazeDriverType.PLAYWRIGHT_NATIVE -> PlaywrightNativeToolSet.LlmToolSet.toolClasses + TrailblazeDriverType.REVYL_ANDROID, + TrailblazeDriverType.REVYL_IOS -> xyz.block.trailblaze.revyl.tools.RevylNativeToolSet.RevylLlmToolSet.toolClasses + else -> emptySet() } } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeDeviceManager.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeDeviceManager.kt index 545af507..271a47bf 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeDeviceManager.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/TrailblazeDeviceManager.kt @@ -66,6 +66,8 @@ import xyz.block.trailblaze.host.rules.BasePlaywrightElectronTest import xyz.block.trailblaze.host.rules.BasePlaywrightNativeTest import xyz.block.trailblaze.util.Console import xyz.block.trailblaze.util.isMacOs +import xyz.block.trailblaze.revyl.RevylCliClient +import xyz.block.trailblaze.revyl.RevylScreenState /** * Manages device discovery, selection, and state across the application. @@ -382,8 +384,8 @@ class TrailblazeDeviceManager( TrailblazeDriverType.REVYL_ANDROID, TrailblazeDriverType.REVYL_IOS -> { val platform = if (driverType == TrailblazeDriverType.REVYL_ANDROID) "android" else "ios" - xyz.block.trailblaze.host.revyl.RevylScreenState( - xyz.block.trailblaze.host.revyl.RevylCliClient(), + RevylScreenState( + revylCliClient, platform, ) } @@ -612,7 +614,7 @@ class TrailblazeDeviceManager( // Uses `revyl device targets --json` so the list stays in sync // with the backend without code changes. try { - val targets = xyz.block.trailblaze.host.revyl.RevylCliClient().getDeviceTargets() + val targets = revylCliClient.getDeviceTargets() for (target in targets) { val driverType = if (target.platform == "android") TrailblazeDriverType.REVYL_ANDROID else TrailblazeDriverType.REVYL_IOS @@ -741,6 +743,8 @@ class TrailblazeDeviceManager( fun getCurrentSelectedTargetApp(): TrailblazeHostAppTarget? = settingsRepo.getCurrentSelectedTargetApp() + private val revylCliClient: RevylCliClient by lazy { RevylCliClient() } + // Store running test instances per device - allows forceful driver shutdown private val maestroDriverByDeviceMap: MutableMap = java.util.concurrent.ConcurrentHashMap() diff --git a/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt b/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt index 49b46c99..c01db957 100644 --- a/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt +++ b/trailblaze-host/src/test/java/xyz/block/trailblaze/host/revyl/RevylDemo.kt @@ -1,5 +1,7 @@ package xyz.block.trailblaze.host.revyl +import xyz.block.trailblaze.revyl.RevylCliClient + /** * Standalone demo that drives a Revyl cloud device through a Bug Bazaar * e-commerce flow using [RevylCliClient] — the same integration layer diff --git a/trailblaze-revyl/build.gradle.kts b/trailblaze-revyl/build.gradle.kts index 4e1ba4cf..1b793c3e 100644 --- a/trailblaze-revyl/build.gradle.kts +++ b/trailblaze-revyl/build.gradle.kts @@ -6,7 +6,6 @@ plugins { dependencies { api(project(":trailblaze-common")) api(project(":trailblaze-agent")) - implementation(project(":trailblaze-host")) implementation(project(":trailblaze-tracing")) implementation(libs.kotlinx.serialization.json) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylActionResult.kt similarity index 96% rename from trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt rename to trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylActionResult.kt index 25a8b647..93688a2d 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylActionResult.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylActionResult.kt @@ -1,9 +1,8 @@ -package xyz.block.trailblaze.host.revyl +package xyz.block.trailblaze.revyl import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.jsonPrimitive import xyz.block.trailblaze.logs.client.TrailblazeJsonInstance import xyz.block.trailblaze.util.Console diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylCliClient.kt similarity index 91% rename from trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt rename to trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylCliClient.kt index a6d0f8ef..b4eb5064 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylCliClient.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylCliClient.kt @@ -1,4 +1,4 @@ -package xyz.block.trailblaze.host.revyl +package xyz.block.trailblaze.revyl import kotlinx.serialization.json.int import kotlinx.serialization.json.intOrNull @@ -32,7 +32,7 @@ class RevylCliClient( private val json = TrailblazeJsonInstance private val sessions = mutableMapOf() - private var activeSessionIndex: Int = 0 + private var activeSessionIndex: Int = ACTIVE_SESSION private val resolvedBinary: String = revylBinaryOverride ?: "revyl" private var cliVerified = false @@ -81,6 +81,12 @@ class RevylCliClient( /** Environment variable name for the Revyl API key. */ const val REVYL_API_KEY_ENV = "REVYL_API_KEY" + + /** Platform identifier for iOS devices. */ + const val PLATFORM_IOS = "ios" + + /** Platform identifier for Android devices. */ + const val PLATFORM_ANDROID = "android" } // --------------------------------------------------------------------------- @@ -177,9 +183,12 @@ class RevylCliClient( * @param appLink Optional deep-link to open after launch. * @param deviceModel Optional explicit device model (e.g. "Pixel 7"). * @param osVersion Optional explicit OS version (e.g. "Android 14"). + * @param appId Optional Revyl `apps` table UUID. The backend resolves the + * latest build artifact and package name from this id. + * @param buildVersionId Optional `builds` table UUID. When provided the + * backend uses this exact build instead of resolving the latest. * @return The newly created [RevylSession] parsed from CLI JSON output. * @throws RevylCliException If the CLI exits with a non-zero code. - * @throws IllegalArgumentException If only one of deviceModel/osVersion is provided. */ fun startSession( platform: String, @@ -187,12 +196,10 @@ class RevylCliClient( appLink: String? = null, deviceModel: String? = null, osVersion: String? = null, + appId: String? = null, + buildVersionId: String? = null, ): RevylSession { verifyCliAvailable() - require(deviceModel.isNullOrBlank() == osVersion.isNullOrBlank()) { - "deviceModel and osVersion must both be provided or both be null/blank " + - "(got deviceModel=$deviceModel, osVersion=$osVersion)" - } val args = mutableListOf("device", "start", "--platform", platform.lowercase()) if (!appUrl.isNullOrBlank()) { args += listOf("--app-url", appUrl) @@ -206,6 +213,12 @@ class RevylCliClient( if (!osVersion.isNullOrBlank()) { args += listOf("--os-version", osVersion) } + if (!appId.isNullOrBlank()) { + args += listOf("--app-id", appId) + } + if (!buildVersionId.isNullOrBlank()) { + args += listOf("--build-version-id", buildVersionId) + } val result = runCli(args) val obj = json.parseToJsonElement(result).jsonObject @@ -242,8 +255,8 @@ class RevylCliClient( if (targetIndex >= 0) args += listOf("-s", targetIndex.toString()) runCli(args) sessions.remove(targetIndex) - if (activeSessionIndex == targetIndex && sessions.isNotEmpty()) { - activeSessionIndex = sessions.keys.first() + if (activeSessionIndex == targetIndex) { + activeSessionIndex = if (sessions.isNotEmpty()) sessions.keys.first() else ACTIVE_SESSION } } @@ -283,7 +296,13 @@ class RevylCliClient( if (!file.exists()) { throw RevylCliException("Screenshot file not found at $outPath") } - return file.readBytes() + return try { + file.readBytes() + } finally { + if (file.absolutePath.contains("revyl-screenshot-")) { + file.delete() + } + } } /** @@ -311,6 +330,18 @@ class RevylCliClient( return RevylActionResult.fromJson(stdout) } + /** + * Double-taps a UI element identified by natural language description. + * + * @param target Natural language description of the element to double-tap. + * @return Action result with the resolved coordinates. + * @throws RevylCliException If the CLI exits with a non-zero code. + */ + fun doubleTap(target: String): RevylActionResult { + val stdout = runCli(deviceArgs("double-tap", "--target", target)) + return RevylActionResult.fromJson(stdout) + } + /** * Types text into an input field, optionally targeting a specific element. * @@ -323,7 +354,7 @@ class RevylCliClient( fun typeText(text: String, target: String? = null, clearFirst: Boolean = false): RevylActionResult { val args = deviceArgs("type", "--text", text).toMutableList() if (!target.isNullOrBlank()) args += listOf("--target", target) - if (clearFirst) args += "--clear-first" + args += "--clear-first=$clearFirst" val stdout = runCli(args) return RevylActionResult.fromJson(stdout) } @@ -461,7 +492,7 @@ class RevylCliClient( val results = mutableListOf() val seenModels = mutableSetOf() - for (platform in listOf("android", "ios")) { + for (platform in listOf(PLATFORM_ANDROID, PLATFORM_IOS)) { val entries = root[platform]?.jsonArray ?: continue for (entry in entries) { val model = entry.jsonObject["Model"]?.jsonPrimitive?.content ?: continue @@ -552,7 +583,9 @@ class RevylCliClient( } private fun createTempScreenshotPath(): String { - return File.createTempFile("revyl-screenshot-", ".png").absolutePath + val tempFile = File.createTempFile("revyl-screenshot-", ".png") + tempFile.deleteOnExit() + return tempFile.absolutePath } } diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylDefaults.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylDefaults.kt new file mode 100644 index 00000000..14dd99f7 --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylDefaults.kt @@ -0,0 +1,30 @@ +package xyz.block.trailblaze.revyl + +/** + * Default screen dimensions for Revyl cloud devices. + * + * These are fallback values used when the session does not report + * screen dimensions. They represent the native resolution of the + * default device provisioned by the Revyl backend for each platform. + */ +object RevylDefaults { + + /** iPhone 16 Pro native resolution (logical points x 3). */ + const val IOS_DEFAULT_WIDTH = 1170 + const val IOS_DEFAULT_HEIGHT = 2532 + + /** Pixel 7 native resolution. */ + const val ANDROID_DEFAULT_WIDTH = 1080 + const val ANDROID_DEFAULT_HEIGHT = 2400 + + /** + * Returns the default (width, height) pair for the given platform string. + * + * @param platform "ios" or "android". + * @return Default dimensions as a Pair. + */ + fun dimensionsForPlatform(platform: String): Pair = when (platform.lowercase()) { + RevylCliClient.PLATFORM_IOS -> Pair(IOS_DEFAULT_WIDTH, IOS_DEFAULT_HEIGHT) + else -> Pair(ANDROID_DEFAULT_WIDTH, ANDROID_DEFAULT_HEIGHT) + } +} diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylLiveStepResult.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylLiveStepResult.kt similarity index 98% rename from trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylLiveStepResult.kt rename to trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylLiveStepResult.kt index ff93c978..39de555a 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylLiveStepResult.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylLiveStepResult.kt @@ -1,4 +1,4 @@ -package xyz.block.trailblaze.host.revyl +package xyz.block.trailblaze.revyl import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylScreenState.kt similarity index 92% rename from trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt rename to trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylScreenState.kt index 3e4d17ce..c844b232 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylScreenState.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylScreenState.kt @@ -1,4 +1,4 @@ -package xyz.block.trailblaze.host.revyl +package xyz.block.trailblaze.revyl import xyz.block.trailblaze.api.ScreenState import xyz.block.trailblaze.api.ViewHierarchyTreeNode @@ -17,7 +17,7 @@ import java.nio.ByteOrder * Screen dimensions are resolved in priority order: * 1. Session-reported dimensions from the worker health endpoint. * 2. PNG IHDR header extraction from the captured screenshot. - * 3. Default fallback (1080x2340). + * 3. Default fallback constants from [RevylDefaults]. * * @param cliClient CLI client used to capture screenshots. * @param platform The device platform ("ios" or "android"). @@ -37,12 +37,14 @@ class RevylScreenState( null } + private val defaultDimensions = RevylDefaults.dimensionsForPlatform(platform) + private val dimensions: Pair = when { sessionScreenWidth > 0 && sessionScreenHeight > 0 -> Pair(sessionScreenWidth, sessionScreenHeight) else -> capturedScreenshot?.let { extractPngDimensions(it) } - ?: Pair(DEFAULT_WIDTH, DEFAULT_HEIGHT) + ?: defaultDimensions } override val screenshotBytes: ByteArray? = capturedScreenshot @@ -64,7 +66,7 @@ class RevylScreenState( override val viewHierarchy: ViewHierarchyTreeNode = viewHierarchyOriginal override val trailblazeDevicePlatform: TrailblazeDevicePlatform = when (platform.lowercase()) { - "ios" -> TrailblazeDevicePlatform.IOS + RevylCliClient.PLATFORM_IOS -> TrailblazeDevicePlatform.IOS else -> TrailblazeDevicePlatform.ANDROID } @@ -74,9 +76,6 @@ class RevylScreenState( ) companion object { - private const val DEFAULT_WIDTH = 1080 - private const val DEFAULT_HEIGHT = 2340 - /** * Extracts width and height from a PNG file's IHDR chunk header. * diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylSession.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylSession.kt similarity index 53% rename from trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylSession.kt rename to trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylSession.kt index 45725a4a..4d6b20a4 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/revyl/RevylSession.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylSession.kt @@ -1,4 +1,7 @@ -package xyz.block.trailblaze.host.revyl +package xyz.block.trailblaze.revyl + +import xyz.block.trailblaze.devices.TrailblazeDevicePlatform +import xyz.block.trailblaze.devices.TrailblazeDriverType /** * Represents an active Revyl cloud device session. @@ -21,4 +24,21 @@ data class RevylSession( val platform: String, val screenWidth: Int = 0, val screenHeight: Int = 0, -) +) { + + /** + * Maps this session's platform string to the corresponding [TrailblazeDriverType]. + */ + fun toDriverType(): TrailblazeDriverType = when (platform) { + RevylCliClient.PLATFORM_IOS -> TrailblazeDriverType.REVYL_IOS + else -> TrailblazeDriverType.REVYL_ANDROID + } + + /** + * Maps this session's platform string to the corresponding [TrailblazeDevicePlatform]. + */ + fun toDevicePlatform(): TrailblazeDevicePlatform = when (platform) { + RevylCliClient.PLATFORM_IOS -> TrailblazeDevicePlatform.IOS + else -> TrailblazeDevicePlatform.ANDROID + } +} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylToolAgent.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylToolAgent.kt index 533f6e03..0b62a39e 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylToolAgent.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylToolAgent.kt @@ -9,8 +9,6 @@ import xyz.block.trailblaze.devices.TrailblazeDeviceId import xyz.block.trailblaze.devices.TrailblazeDeviceInfo import xyz.block.trailblaze.devices.TrailblazeDevicePlatform import xyz.block.trailblaze.devices.TrailblazeDriverType -import xyz.block.trailblaze.host.revyl.RevylCliClient -import xyz.block.trailblaze.host.revyl.RevylScreenState import xyz.block.trailblaze.logs.client.TrailblazeLogger import xyz.block.trailblaze.logs.client.TrailblazeSession import xyz.block.trailblaze.logs.client.TrailblazeSessionProvider @@ -105,15 +103,16 @@ class RevylToolAgent( private fun buildMinimalContext( screenStateProvider: () -> ScreenState, ): TrailblazeToolExecutionContext { - val devicePlatform = if (platform == "ios") TrailblazeDevicePlatform.IOS else TrailblazeDevicePlatform.ANDROID + val devicePlatform = if (platform == RevylCliClient.PLATFORM_IOS) TrailblazeDevicePlatform.IOS else TrailblazeDevicePlatform.ANDROID + val defaultDimensions = RevylDefaults.dimensionsForPlatform(platform) return TrailblazeToolExecutionContext( screenState = null, traceId = null, trailblazeDeviceInfo = TrailblazeDeviceInfo( trailblazeDeviceId = TrailblazeDeviceId(instanceId = "revyl", trailblazeDevicePlatform = devicePlatform), - trailblazeDriverType = if (platform == "ios") TrailblazeDriverType.IOS_HOST else TrailblazeDriverType.ANDROID_HOST, - widthPixels = 0, - heightPixels = 0, + trailblazeDriverType = if (platform == RevylCliClient.PLATFORM_IOS) TrailblazeDriverType.REVYL_IOS else TrailblazeDriverType.REVYL_ANDROID, + widthPixels = defaultDimensions.first, + heightPixels = defaultDimensions.second, ), sessionProvider = TrailblazeSessionProvider { TrailblazeSession(sessionId = SessionId("revyl-tool-agent"), startTime = Clock.System.now()) diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylExecutableTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylExecutableTool.kt index df5e3ad7..83ce563f 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylExecutableTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylExecutableTool.kt @@ -1,6 +1,6 @@ package xyz.block.trailblaze.revyl.tools -import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.revyl.RevylCliClient import xyz.block.trailblaze.toolcalls.ExecutableTrailblazeTool import xyz.block.trailblaze.toolcalls.ReasoningTrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt index 1e08c3e5..9be5ff7e 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt @@ -2,7 +2,7 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable -import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.revyl.RevylCliClient import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult @@ -32,13 +32,13 @@ class RevylNativeAssertTool( context: TrailblazeToolExecutionContext, ): TrailblazeToolResult { Console.log("### Asserting: $assertion") - val screenshot = revylClient.screenshot() - val passed = screenshot.isNotEmpty() - return if (passed) { - TrailblazeToolResult.Success(message = "Assertion check: '$assertion' — screenshot captured for verification.") + val result = revylClient.validation(assertion) + return if (result.success) { + TrailblazeToolResult.Success(message = "Assertion passed: '$assertion'") } else { TrailblazeToolResult.Error.ExceptionThrown( - errorMessage = "Assertion failed: could not capture screen for '$assertion'", + errorMessage = "Assertion failed: '$assertion'" + + (result.statusReason?.let { " -- $it" } ?: ""), ) } } diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeBackTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeBackTool.kt index 5cf8d089..2515dec1 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeBackTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeBackTool.kt @@ -2,7 +2,7 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable -import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.revyl.RevylCliClient import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeDoubleTapTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeDoubleTapTool.kt new file mode 100644 index 00000000..858fbbc5 --- /dev/null +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeDoubleTapTool.kt @@ -0,0 +1,40 @@ +package xyz.block.trailblaze.revyl.tools + +import ai.koog.agents.core.tools.annotations.LLMDescription +import kotlinx.serialization.Serializable +import xyz.block.trailblaze.revyl.RevylCliClient +import xyz.block.trailblaze.toolcalls.TrailblazeToolClass +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.util.Console + +/** + * Double-taps a UI element on the Revyl cloud device using natural language targeting. + * + * Useful for zoom-in gestures on maps, image viewers, and double-tap UI patterns. + * The Revyl CLI resolves the target description to screen coordinates via + * AI-powered visual grounding, then performs the double-tap. + */ +@Serializable +@TrailblazeToolClass("revyl_doubleTap") +@LLMDescription( + "Double-tap a UI element on the device screen. Describe the element in natural language. " + + "Use for zoom-in gestures on maps, image viewers, or any double-tap UI pattern.", +) +class RevylNativeDoubleTapTool( + @param:LLMDescription("Element to double-tap, described in natural language.") + val target: String, + override val reasoning: String? = null, +) : RevylExecutableTool() { + + override suspend fun executeWithRevyl( + revylClient: RevylCliClient, + context: TrailblazeToolExecutionContext, + ): TrailblazeToolResult { + Console.log("### Double-tapping: $target") + val result = revylClient.doubleTap(target) + val feedback = "Double-tapped '$target' at (${result.x}, ${result.y})" + Console.log("### $feedback") + return TrailblazeToolResult.Success(message = feedback) + } +} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeNavigateTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeNavigateTool.kt index 64a4c3e6..5affd3c0 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeNavigateTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeNavigateTool.kt @@ -2,7 +2,7 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable -import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.revyl.RevylCliClient import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativePressKeyTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativePressKeyTool.kt index 446196e5..762d3de0 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativePressKeyTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativePressKeyTool.kt @@ -2,7 +2,7 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable -import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.revyl.RevylCliClient import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeScreenshotTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeScreenshotTool.kt deleted file mode 100644 index c21a32fa..00000000 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeScreenshotTool.kt +++ /dev/null @@ -1,32 +0,0 @@ -package xyz.block.trailblaze.revyl.tools - -import ai.koog.agents.core.tools.annotations.LLMDescription -import kotlinx.serialization.Serializable -import xyz.block.trailblaze.host.revyl.RevylCliClient -import xyz.block.trailblaze.toolcalls.TrailblazeToolClass -import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext -import xyz.block.trailblaze.toolcalls.TrailblazeToolResult -import xyz.block.trailblaze.util.Console - -/** - * Captures a screenshot of the Revyl cloud device screen. - * - * This tool is intended for internal use by the agent framework - * and is not included in the LLM-facing tool set. - */ -@Serializable -@TrailblazeToolClass("revyl_screenshot") -@LLMDescription("Take a screenshot of the current device screen to see what's on it.") -class RevylNativeScreenshotTool( - override val reasoning: String? = null, -) : RevylExecutableTool() { - - override suspend fun executeWithRevyl( - revylClient: RevylCliClient, - context: TrailblazeToolExecutionContext, - ): TrailblazeToolResult { - Console.log("### Taking screenshot") - revylClient.screenshot() - return TrailblazeToolResult.Success(message = "Screenshot captured.") - } -} diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeSwipeTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeSwipeTool.kt index bbfe110b..4682bf09 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeSwipeTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeSwipeTool.kt @@ -2,7 +2,7 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable -import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.revyl.RevylCliClient import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTapTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTapTool.kt index 157ca52f..3fd09a94 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTapTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTapTool.kt @@ -2,7 +2,7 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable -import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.revyl.RevylCliClient import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeToolSet.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeToolSet.kt index 74bf61d4..b376d8b4 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeToolSet.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeToolSet.kt @@ -18,6 +18,7 @@ object RevylNativeToolSet { toolClasses = setOf( RevylNativeTapTool::class, + RevylNativeDoubleTapTool::class, RevylNativeTypeTool::class, RevylNativeSwipeTool::class, RevylNativeNavigateTool::class, diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTypeTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTypeTool.kt index 7626d1f4..fd22f552 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTypeTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeTypeTool.kt @@ -2,7 +2,7 @@ package xyz.block.trailblaze.revyl.tools import ai.koog.agents.core.tools.annotations.LLMDescription import kotlinx.serialization.Serializable -import xyz.block.trailblaze.host.revyl.RevylCliClient +import xyz.block.trailblaze.revyl.RevylCliClient import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext import xyz.block.trailblaze.toolcalls.TrailblazeToolResult diff --git a/trailblaze-revyl/src/test/java/xyz/block/trailblaze/revyl/RevylToolAgentTest.kt b/trailblaze-revyl/src/test/java/xyz/block/trailblaze/revyl/RevylToolAgentTest.kt index f46ae712..8bdfd62f 100644 --- a/trailblaze-revyl/src/test/java/xyz/block/trailblaze/revyl/RevylToolAgentTest.kt +++ b/trailblaze-revyl/src/test/java/xyz/block/trailblaze/revyl/RevylToolAgentTest.kt @@ -10,7 +10,6 @@ import kotlinx.serialization.Transient import org.junit.Test import xyz.block.trailblaze.devices.TrailblazeDevicePlatform import xyz.block.trailblaze.devices.TrailblazeDriverType -import xyz.block.trailblaze.host.revyl.RevylCliClient import xyz.block.trailblaze.revyl.tools.RevylExecutableTool import xyz.block.trailblaze.toolcalls.TrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeToolClass @@ -177,7 +176,7 @@ class RevylToolAgentTest { runTools(agent, tool) val info = capturedContext!!.trailblazeDeviceInfo - assertThat(info.trailblazeDriverType).isEqualTo(TrailblazeDriverType.IOS_HOST) + assertThat(info.trailblazeDriverType).isEqualTo(TrailblazeDriverType.REVYL_IOS) assertThat(info.trailblazeDeviceId.trailblazeDevicePlatform) .isEqualTo(TrailblazeDevicePlatform.IOS) } @@ -191,7 +190,7 @@ class RevylToolAgentTest { runTools(agent, tool) val info = capturedContext!!.trailblazeDeviceInfo - assertThat(info.trailblazeDriverType).isEqualTo(TrailblazeDriverType.ANDROID_HOST) + assertThat(info.trailblazeDriverType).isEqualTo(TrailblazeDriverType.REVYL_ANDROID) assertThat(info.trailblazeDeviceId.trailblazeDevicePlatform) .isEqualTo(TrailblazeDevicePlatform.ANDROID) } diff --git a/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/ExternalLinkUtils.kt b/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/ExternalLinkUtils.kt new file mode 100644 index 00000000..372353f0 --- /dev/null +++ b/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/ExternalLinkUtils.kt @@ -0,0 +1,34 @@ +package xyz.block.trailblaze.ui.tabs.session + +import xyz.block.trailblaze.devices.TrailblazeDeviceInfo + +/** + * A clickable link derived from device metadata. + * + * @property label Human-readable display text for the link. + * @property url Fully-qualified URL to open. + */ +data class ExternalLink(val label: String, val url: String) + +/** + * Extracts external links from device metadata using the `*_url` key convention. + * + * For each metadata entry whose key ends with `_url` and whose value is non-blank, + * a link is produced. The label is resolved from a sibling `*_label` key if present, + * otherwise auto-generated from the key prefix (e.g. `revyl_viewer_url` becomes + * "Open Revyl viewer"). + * + * @param deviceInfo Device info containing the metadata map. Returns empty list if null. + * @return Ordered list of external links. + */ +fun extractExternalLinks(deviceInfo: TrailblazeDeviceInfo?): List { + val metadata = deviceInfo?.metadata.orEmpty() + return metadata.entries + .filter { it.key.endsWith("_url") && it.value.isNotBlank() } + .map { (key, url) -> + val prefix = key.removeSuffix("_url") + val label = metadata["${prefix}_label"] + ?: "Open ${prefix.replace('_', ' ').replaceFirstChar { c -> c.uppercase() }}" + ExternalLink(label, url) + } +} diff --git a/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/SessionDetailHeader.kt b/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/SessionDetailHeader.kt index a39d229e..f5186be4 100644 --- a/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/SessionDetailHeader.kt +++ b/trailblaze-ui/src/commonMain/kotlin/xyz/block/trailblaze/ui/tabs/session/SessionDetailHeader.kt @@ -148,25 +148,16 @@ internal fun SessionDetailHeader( color = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.7f), ) } - // Render external links from device metadata (convention: *_url keys become clickable links) - val metadata = sessionDetail.session.trailblazeDeviceInfo?.metadata.orEmpty() - val externalLinks = metadata.entries - .filter { it.key.endsWith("_url") && it.value.isNotBlank() } - .map { (key, url) -> - val prefix = key.removeSuffix("_url") - val label = metadata["${prefix}_label"] - ?: "Open ${prefix.replace('_', ' ').replaceFirstChar { c -> c.uppercase() }}" - label to url - } + val externalLinks = extractExternalLinks(sessionDetail.session.trailblazeDeviceInfo) if (externalLinks.isNotEmpty()) { val uriHandler = LocalUriHandler.current - for ((label, url) in externalLinks) { + for (link in externalLinks) { TextButton( - onClick = { uriHandler.openUri(url) }, + onClick = { uriHandler.openUri(link.url) }, contentPadding = ButtonDefaults.TextButtonContentPadding, ) { Text( - text = label, + text = link.label, style = MaterialTheme.typography.labelSmall, color = MaterialTheme.colorScheme.primary, ) From c500c536be3617eb7dac291a18ba7bff2c2b0e5d Mon Sep 17 00:00:00 2001 From: Anam Hira Date: Tue, 7 Apr 2026 22:27:05 -0700 Subject: [PATCH 16/16] docs: polish Revyl review nits Signed-off-by: Anam Hira --- .../java/xyz/block/trailblaze/revyl/RevylCliClient.kt | 10 +++++----- .../trailblaze/revyl/tools/RevylNativeAssertTool.kt | 8 ++++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylCliClient.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylCliClient.kt index b4eb5064..02a21374 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylCliClient.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/RevylCliClient.kt @@ -16,12 +16,12 @@ import java.io.File * executes it, and parses the structured JSON output. The CLI handles * auth, backend proxy routing, and AI-powered target grounding. * - * The `revyl` binary must be pre-installed on PATH (or pointed to via - * `REVYL_BINARY` env var). If not found, [startSession] throws a + * The `revyl` binary is resolved from PATH by default, or can be pointed to via + * `REVYL_BINARY` for local/dev overrides. If not found, [startSession] throws a * [RevylCliException] with install instructions. * - * @property revylBinaryOverride Explicit path to the revyl binary. - * Defaults to `REVYL_BINARY` env var, then PATH lookup. + * @property revylBinaryOverride Optional explicit path to the revyl binary. + * Defaults to `REVYL_BINARY`; if null, PATH lookup uses `revyl`. * @property workingDirectory Optional working directory for CLI * invocations. Defaults to the JVM's current directory. */ @@ -599,7 +599,7 @@ class RevylCliException(message: String) : RuntimeException(message) /** * A device model available in the Revyl cloud catalog. * - * @property platform "ios" or "android". + * @property platform One of [RevylCliClient.PLATFORM_IOS] or [RevylCliClient.PLATFORM_ANDROID]. * @property model Human-readable model name (e.g. "iPhone 16", "Pixel 7"). * @property osVersion Runtime / OS version string (e.g. "Android 14", "iOS 18.2"). */ diff --git a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt index 9be5ff7e..e071e547 100644 --- a/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt +++ b/trailblaze-revyl/src/main/java/xyz/block/trailblaze/revyl/tools/RevylNativeAssertTool.kt @@ -19,10 +19,14 @@ import xyz.block.trailblaze.util.Console @LLMDescription( "Assert a visual condition on the device screen. Describe what should be true. " + "Examples: 'the cart total shows \$42.99', 'a success message is visible', " + - "'the Sign In button is disabled', 'there are at least 3 search results'.", + "'the Sign In button is disabled', 'there are at least 3 search results', " + + "'the settings screen is open', 'the order confirmation page is shown'.", ) class RevylNativeAssertTool( - @param:LLMDescription("The condition to verify, described in natural language.") + @param:LLMDescription( + "The condition to verify, described in natural language. " + + "Examples: 'the cart badge shows 2 items', 'the profile tab is selected'.", + ) val assertion: String, override val reasoning: String? = null, ) : RevylExecutableTool() {