diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ed646a7..06c855a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,7 @@ concurrency: cancel-in-progress: true permissions: + id-token: write contents: write pull-requests: write @@ -19,6 +20,7 @@ jobs: outputs: releases_created: ${{ steps.release.outputs.releases_created }} paths_released: ${{ steps.release.outputs.paths_released }} + cli_release_created: ${{ steps.release.outputs['packages/cli--release_created'] }} mcp_release_created: ${{ steps.release.outputs['packages/mcp--release_created'] }} steps: - uses: googleapis/release-please-action@v4 @@ -46,5 +48,22 @@ jobs: - run: npm run build --workspace @testplane/mcp - run: npm publish --workspace @testplane/mcp --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + publish-cli: + needs: release-please + if: ${{ needs.release-please.outputs.cli_release_created == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "22" + registry-url: "https://registry.npmjs.org" + cache: "npm" + + - run: npm ci + + - run: npm run build --workspace @testplane/cli + + - run: npm publish --workspace @testplane/cli --access public diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 5d22bf8..2d8bbb0 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,4 @@ { + "packages/cli": "0.1.0", "packages/mcp": "0.6.0" } diff --git a/README.md b/README.md index adfa044..3989490 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,14 @@ A collection of AI integrations for Testplane, which enables agents to interact - AI Agents no longer have to take guesses as to how your app works — they can truly see what's happening inside a browser and write quality tests for you - Let LLMs use text-based or visual-based snapshots, depending on what works better for your app +### Testplane CLI + +Testplane CLI is a command line interface for inspecting and controlling a browser from your terminal. It makes possible to perform any browser interactions, inspect HTML Reports and Time Travel snapshots, use REPL mode to debug failing tests and more. + +Read [full documentation](./packages/cli/README.md) to learn more. + ### Testplane MCP -Testplane MCP is a [Model Context Protocol server](https://modelcontextprotocol.io/quickstart/user) for Testplane, which enables LLMs to "see" and interact with any web app. +Testplane MCP is a [Model Context Protocol server](https://modelcontextprotocol.io/quickstart/user) for Testplane, which has the same capabilities as Testplane CLI and allows LLMs to "see" and interact with any web app. Read [full documentation](./packages/mcp/README.md) to learn more. diff --git a/packages/cli/README.md b/packages/cli/README.md new file mode 100644 index 0000000..24d0719 --- /dev/null +++ b/packages/cli/README.md @@ -0,0 +1,66 @@ +

+ + + + Testplane CLI logo + +

+ +

+ Total Downloads + Documentation + Latest Release + License + Community Chat +

+ +Testplane CLI is a command line interface for inspecting and controlling a browser from your terminal. It makes possible to perform any browser interactions, inspect HTML Reports and Time Travel snapshots, use REPL mode to debug failing tests and more. + +Read the [full Testplane CLI documentation](https://testplane.io/docs/v8/ai/toolkit/testplane-cli/). + +### Installation + +Install globally: + +```bash +npm install -g @testplane/cli +``` + +Or run it without installing: + +```bash +npx @testplane/cli@latest --help +``` + +### Examples + +Open a page and capture a DOM snapshot: + +```bash +testplane-cli navigate https://example.com +testplane-cli snapshot +``` + +Interact with the current page: + +```bash +testplane-cli click --role button --name "Submit" +testplane-cli type "#email" --value user@example.com +``` + +Run custom Testplane code against the active browser session: + +```bash +testplane-cli run-code "await browser.getUrl()" +testplane-cli run-code --file ./scripts/check-page.js +``` + +### Capabilities + +Testplane CLI runs a reusable local daemon for each project, so commands can share the same browser session across terminal invocations. + +- Various browser interactions: navigate pages, click and type into elements, manage tabs, read console output, take screenshots, and inspect DOM or visual state. +- HTML reporter integration: open reports and inspect captured time travel snapshots. +- REPL mode integration for interactive browser exploration from the terminal. +- Save and restore browser state to handle authenticated flows. +- Run arbitrary Testplane code when built-in commands are not enough. diff --git a/packages/cli/docs/images/logo-dark.svg b/packages/cli/docs/images/logo-dark.svg new file mode 100644 index 0000000..5eb375a --- /dev/null +++ b/packages/cli/docs/images/logo-dark.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/cli/docs/images/logo-light.svg b/packages/cli/docs/images/logo-light.svg new file mode 100644 index 0000000..a0fd467 --- /dev/null +++ b/packages/cli/docs/images/logo-light.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/cli/src/daemon/request-handler.ts b/packages/cli/src/daemon/request-handler.ts index 65a4fc0..659276f 100644 --- a/packages/cli/src/daemon/request-handler.ts +++ b/packages/cli/src/daemon/request-handler.ts @@ -2,11 +2,20 @@ import makeDebug from "debug"; import { z, type ZodRawShape } from "zod"; import { launchBrowserWithOptions, ToolKind, tools } from "@testplane/tools"; import { SessionRegistry } from "./session-registry.js"; -import type { Request, Response } from "../ipc/protocol.js"; +import type { Request, Response, ResponseResult } from "../ipc/protocol.js"; import { formatError } from "../utils/error.js"; const debug = makeDebug("testplane-cli:daemon:request-handler"); +function createNoActiveBrowserResponse(toolName: string): ResponseResult["content"] { + return [ + { + type: "text", + text: `❌ No active browser session. Run "navigate " to auto-start one, or run "launch" before "${toolName}".`, + }, + ]; +} + /** * Handles requests from clients, executing tools in sessions as needed. */ @@ -66,8 +75,19 @@ export class RequestHandler { try { if (tool.kind === ToolKind.Action) { if (!state.browser) { - debug("Auto-launching browser: session=%s", req.sessionName); - state.browser = await launchBrowserWithOptions(state.options); + if (!tool.autoLaunchBrowser) { + debug("Action requires an active browser: session=%s tool=%s", req.sessionName, req.tool); + + return { + id: req.id, + kind: "result", + content: createNoActiveBrowserResponse(tool.name), + isError: true, + }; + } + + debug("Auto-launching browser: session=%s tool=%s", req.sessionName, req.tool); + state.browser = await launchBrowserWithOptions(state.defaultOptions); } // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/packages/cli/src/daemon/session-registry.ts b/packages/cli/src/daemon/session-registry.ts index ed13725..fc99b32 100644 --- a/packages/cli/src/daemon/session-registry.ts +++ b/packages/cli/src/daemon/session-registry.ts @@ -7,6 +7,7 @@ const debug = makeDebug("testplane-cli:daemon:session-registry"); export interface SessionState { browser: WdioBrowser | null; + defaultOptions: BrowserOptions; options: BrowserOptions; activeInteractions: number; expirationTimer: NodeJS.Timeout | null; @@ -39,6 +40,7 @@ export class SessionRegistry { if (!state) { state = { browser: null, + defaultOptions: { ...this._defaultOptions }, options: { ...this._defaultOptions }, activeInteractions: 0, expirationTimer: null, diff --git a/packages/cli/test/daemon.e2e.test.ts b/packages/cli/test/daemon.e2e.test.ts index 15fb9cc..e0f7e42 100644 --- a/packages/cli/test/daemon.e2e.test.ts +++ b/packages/cli/test/daemon.e2e.test.ts @@ -137,6 +137,9 @@ describe("daemon e2e", () => { const spawnResp = await runCli(["close-browser"], runCodeEnv, daemonCwd); expect(spawnResp.code).toBe(0); + const launchResp = await runCli(["launch"], runCodeEnv, callerCwd); + expect(launchResp.code).toBe(0); + const runResp = await runCli(["run-code", "--file", "./script.js"], runCodeEnv, callerCwd); expect(runResp.code).toBe(0); expect(runResp.stdout).toContain('"caller-cwd"'); @@ -147,6 +150,12 @@ describe("daemon e2e", () => { } }); + it("does not auto-launch browser for non-navigate actions", async () => { + const r = await runCli(["--session-name", "no-auto-launch", "click", "#btn"], extraEnv); + expect(r.code).toBe(1); + expect(r.stdout).toContain("No active browser session"); + }); + it("navigates to the playground page", async () => { const r = await runCli(["navigate", playgroundUrl], extraEnv); expect(r.code).toBe(0); diff --git a/packages/cli/test/daemon/request-handler.test.ts b/packages/cli/test/daemon/request-handler.test.ts new file mode 100644 index 0000000..d7004af --- /dev/null +++ b/packages/cli/test/daemon/request-handler.test.ts @@ -0,0 +1,109 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { WdioBrowser } from "testplane"; +import type { Request } from "../../src/ipc/protocol.js"; +import { SessionRegistry } from "../../src/daemon/session-registry.js"; +import { RequestHandler } from "../../src/daemon/request-handler.js"; + +const mockedTools = vi.hoisted(() => { + const launchBrowserWithOptions = vi.fn(); + const navigateCb = vi.fn(); + const clickCb = vi.fn(); + + return { + launchBrowserWithOptions, + navigateCb, + clickCb, + }; +}); + +vi.mock("@testplane/tools", () => { + const ToolKind = { + Standalone: "standalone", + Action: "action", + SessionOpen: "session-open", + SessionClose: "session-close", + }; + + return { + ToolKind, + launchBrowserWithOptions: mockedTools.launchBrowserWithOptions, + tools: [ + { + kind: ToolKind.Action, + autoLaunchBrowser: true, + name: "navigate", + description: "navigate", + schema: {}, + cb: mockedTools.navigateCb, + }, + { + kind: ToolKind.Action, + name: "click", + description: "click", + schema: {}, + cb: mockedTools.clickCb, + }, + ], + }; +}); + +function createRequest(tool: string): Request { + return { + id: 1, + kind: "call", + tool, + sessionName: "default", + args: {}, + }; +} + +function createBrowser(): WdioBrowser { + return { + deleteSession: vi.fn().mockResolvedValue(undefined), + getUrl: vi.fn().mockResolvedValue("about:blank"), + } as unknown as WdioBrowser; +} + +describe("daemon/RequestHandler", () => { + let handler: RequestHandler; + let sessions: SessionRegistry; + let browser: WdioBrowser; + + beforeEach(() => { + vi.clearAllMocks(); + + handler = new RequestHandler(); + sessions = new SessionRegistry({ headless: true }, { sessionTtlMs: 1000 }); + browser = createBrowser(); + + mockedTools.launchBrowserWithOptions.mockResolvedValue(browser); + mockedTools.navigateCb.mockResolvedValue({ content: [{ type: "text", text: "ok" }], isError: false }); + mockedTools.clickCb.mockResolvedValue({ content: [{ type: "text", text: "ok" }], isError: false }); + }); + + it("does not auto-launch browser for action tools that require an existing session", async () => { + const response = await handler.handleRequest(createRequest("click"), sessions); + + expect(mockedTools.launchBrowserWithOptions).not.toHaveBeenCalled(); + expect(response.kind).toBe("result"); + if (response.kind === "result") { + expect(response.isError).toBe(true); + expect(response.content[0].text).toContain("No active browser session"); + } + }); + + it("auto-launches navigate with default options even when custom launch options were used before", async () => { + const state = sessions.getOrCreate("default"); + state.options = { headless: false }; + + const response = await handler.handleRequest(createRequest("navigate"), sessions); + + expect(mockedTools.launchBrowserWithOptions).toHaveBeenCalledTimes(1); + expect(mockedTools.launchBrowserWithOptions).toHaveBeenCalledWith({ headless: true }); + expect(mockedTools.navigateCb).toHaveBeenCalledWith({}, browser); + expect(response.kind).toBe("result"); + if (response.kind === "result") { + expect(response.isError).toBe(false); + } + }); +}); diff --git a/packages/mcp/src/server.ts b/packages/mcp/src/server.ts index 2cccca2..88c1772 100644 --- a/packages/mcp/src/server.ts +++ b/packages/mcp/src/server.ts @@ -14,6 +14,18 @@ interface PackageJson { version: string; } +function createNoActiveBrowserResponse(toolName: string) { + return { + content: [ + { + type: "text", + text: `❌ No active browser session. Run "navigate" to auto-start one, or run "launch" before "${toolName}".`, + }, + ], + isError: true, + }; +} + const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const packageJson: PackageJson = JSON.parse(readFileSync(join(__dirname, "../package.json"), "utf-8")); @@ -24,7 +36,8 @@ export interface ServerOptions { export async function startServer(serverOptions: ServerOptions = {}): Promise { let browser: WdioBrowser | null = null; - let options: BrowserOptions = { headless: serverOptions.headless ?? false }; + const defaultOptions: BrowserOptions = { headless: serverOptions.headless ?? false }; + let sessionOptions: BrowserOptions = { ...defaultOptions }; const server = new McpServer({ name: packageJson.name, @@ -44,14 +57,18 @@ export async function startServer(serverOptions: ServerOptions = {}): Promise { + const result = await client.callTool({ + name: "click", + arguments: { + selector: "#btn", + }, + }); + expect(result.isError).toBe(true); + + const content = result.content as Array<{ type: string; text: string }>; + const text = content.map(c => c.text).join("\n"); + expect(text).toContain("No active browser session"); + }); + it("auto-launches the browser on first action tool call", async () => { const result = await client.callTool({ name: "navigate", arguments: { url: playgroundUrl } }); expect(result.isError).toBe(false); diff --git a/packages/tools/src/tools/navigate.ts b/packages/tools/src/tools/navigate.ts index 8afcd61..19ce92d 100644 --- a/packages/tools/src/tools/navigate.ts +++ b/packages/tools/src/tools/navigate.ts @@ -11,18 +11,24 @@ const navigateCb: ActionTool["cb"] = async (args, browser try { const { url, timeout } = args; - const openOptions: { timeout?: number } = {}; + const openOptions: { timeout?: number; ignoreNetworkErrorsPatterns: RegExp[] } = { + ignoreNetworkErrorsPatterns: [/.*/], + }; + const testplaneOpenOptions: { timeout?: number } = {}; + if (timeout !== undefined) { const browserConfig = await browser.getConfig(); browserConfig.urlHttpTimeout = timeout; openOptions.timeout = timeout; + testplaneOpenOptions.timeout = timeout; } console.error(`Navigating to: ${url}`); await browser.openAndWait(url, openOptions); - const optionsCode = Object.keys(openOptions).length > 0 ? `, ${JSON.stringify(openOptions)}` : ""; + const optionsCode = + Object.keys(testplaneOpenOptions).length > 0 ? `, ${JSON.stringify(testplaneOpenOptions)}` : ""; return await createBrowserStateResponse(browser, { action: `Successfully navigated to ${url}`, @@ -45,6 +51,7 @@ const navigateCb: ActionTool["cb"] = async (args, browser export const navigate: ActionTool = { kind: ToolKind.Action, + autoLaunchBrowser: true, name: "navigate", description: "Open a URL in the browser", schema: navigateSchema, diff --git a/packages/tools/src/types.ts b/packages/tools/src/types.ts index 98b75a9..169195f 100644 --- a/packages/tools/src/types.ts +++ b/packages/tools/src/types.ts @@ -45,6 +45,7 @@ interface ToolBase { export interface ActionTool extends ToolBase { kind: ToolKind.Action; + autoLaunchBrowser?: boolean; cb: (args: ToolArgs, browser: WdioBrowser) => Promise; } diff --git a/packages/tools/test/tools/navigate.test.ts b/packages/tools/test/tools/navigate.test.ts index 45954f6..9cdc3a1 100644 --- a/packages/tools/test/tools/navigate.test.ts +++ b/packages/tools/test/tools/navigate.test.ts @@ -47,6 +47,29 @@ describe( expect(text).toMatch(/The snapshot was saved to: .+\.testplane\/snapshots\/.+\.(yml|html)/); }); + it("should ignore failed resource loads while navigating", async () => { + const flakyResourceServer = http.createServer((_req, res) => { + res.writeHead(200, { "Content-Type": "text/html" }); + res.end( + 'ok', + ); + }); + + await new Promise(resolve => flakyResourceServer.listen(0, resolve)); + const port = (flakyResourceServer.address() as AddressInfo).port; + const flakyUrl = `http://localhost:${port}/`; + + try { + const result = await navigate.cb({ url: flakyUrl, timeout: 5000 }, browser); + + expect(result.isError).toBe(false); + const text = getTextContent(result); + expect(text).toContain(`Successfully navigated to ${flakyUrl}`); + } finally { + await new Promise(resolve => flakyResourceServer.close(() => resolve())); + } + }); + describe("timeout behavior", () => { it("should omit timeout from generated testplane code when not provided", async () => { const result = await navigate.cb({ url: playgroundUrl }, browser); diff --git a/release-please-config.json b/release-please-config.json index eba449b..86a22fc 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -1,6 +1,10 @@ { "bump-minor-pre-major": true, "packages": { + "packages/cli": { + "release-type": "node", + "package-name": "@testplane/cli" + }, "packages/mcp": { "release-type": "node", "package-name": "@testplane/mcp"