From c80850978b159213757c887d6fcbeee038a7f348 Mon Sep 17 00:00:00 2001 From: Aleksey Bykhun Date: Tue, 5 May 2026 17:56:48 -0700 Subject: [PATCH 01/14] fix(registry): render action params as JSON Schema instead of zod _def dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RegisteredAction.promptDescription previously serialized each property of the zod object schema by stringifying its private `_def` AST. For schemas with `.default()` wrappers (e.g. ScrollActionSchema's `down` and `num_pages`), the LLM would see something like: "num_pages": {"type":"default","innerType":{"def":{"type":"number"},...},"defaultValue":1}, "down": {"type":"default","innerType":{"def":{"type":"boolean"},...},"defaultValue":true} The model would plausibly copy the nearby `defaultValue: true` and emit a boolean for `num_pages`. The schema correctly rejected, the same prompt was fed back, and the same mistake recurred until `max_failures=3` tripped. Replace the `_def` walk with `z.toJSONSchema(schema, {unrepresentable:'any'})` (zod v4 native), strip the `$schema` dialect URL, and apply the existing skipKeys filter to both `properties` and `required`. The LLM now sees: {"type":"object","properties":{"down":{"default":true,"type":"boolean"}, "num_pages":{"default":1,"type":"number"},...}, "required":[...], "additionalProperties":false} — a familiar, well-known JSON Schema shape with no zod-internal leakage. Surrounding `${description}: \n{${name}: ...}` envelope is unchanged so the LLM sees the same outer layout. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/controller/registry/views.ts | 58 +++++++++++++++++++++----------- test/promptDescription.test.ts | 54 +++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 19 deletions(-) create mode 100644 test/promptDescription.test.ts diff --git a/src/controller/registry/views.ts b/src/controller/registry/views.ts index 5d880f8e..4612ded8 100644 --- a/src/controller/registry/views.ts +++ b/src/controller/registry/views.ts @@ -23,6 +23,43 @@ type BrowserSession = unknown; type BaseChatModel = unknown; type FileSystem = unknown; +// Render an action's param schema as compact JSON Schema for the LLM prompt. +// Replaces a prior raw dump of zod's private `_def` AST, which leaked +// internal keys like `innerType`/`defaultValue` and confused the LLM into +// copying default booleans into numeric fields (see scroll.num_pages bug). +function renderParamsJsonSchema( + schema: ZodTypeAny, + skipKeys: Set +): Record { + const raw = z.toJSONSchema(schema, { unrepresentable: 'any' }) as Record< + string, + unknown + >; + // Strip dialect noise the LLM doesn't need. + delete raw.$schema; + + const properties = (raw.properties as Record) ?? {}; + const filteredProps: Record = {}; + for (const [key, value] of Object.entries(properties)) { + if (skipKeys.has(key)) { + continue; + } + filteredProps[key] = value; + } + raw.properties = filteredProps; + + if (Array.isArray(raw.required)) { + raw.required = raw.required.filter( + (key: unknown) => typeof key === 'string' && !skipKeys.has(key) + ); + if ((raw.required as string[]).length === 0) { + delete raw.required; + } + } + + return raw; +} + export class RegisteredAction { constructor( public readonly name: string, @@ -64,25 +101,8 @@ export class RegisteredAction { skipKeys.add('output_schema'); } - if (schemaShape) { - const props = Object.fromEntries( - Object.entries(schemaShape) - .filter(([key]) => !skipKeys.has(key)) - .map(([key, value]) => { - const entries = value instanceof z.ZodType ? value._def : value; - const cleanEntries = Object.fromEntries( - Object.entries(entries as Record).filter( - ([propKey]) => !skipKeys.has(propKey) - ) - ); - return [key, cleanEntries]; - }) - ); - description += JSON.stringify(props); - } else { - description += '{}'; - } - + const jsonSchema = renderParamsJsonSchema(this.paramSchema, skipKeys); + description += JSON.stringify(jsonSchema); description += '}'; return description; } diff --git a/test/promptDescription.test.ts b/test/promptDescription.test.ts new file mode 100644 index 00000000..46205b49 --- /dev/null +++ b/test/promptDescription.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { RegisteredAction } from '../src/controller/registry/views.js'; + +const ScrollActionSchema = z.object({ + down: z.boolean().default(true), + num_pages: z.number().default(1), + pages: z.number().optional(), + index: z.number().int().optional(), +}); + +describe('RegisteredAction.promptDescription', () => { + it('renders zod schema as clean JSON Schema, not raw _def dump', () => { + const action = new RegisteredAction( + 'scroll', + 'Scroll the page', + async () => ({}), + ScrollActionSchema + ); + + const prompt = action.promptDescription(); + const jsonStart = prompt.indexOf('{', prompt.indexOf('{scroll:') + '{scroll:'.length); + const inner = prompt.slice(jsonStart, prompt.lastIndexOf('}')); + const parsed = JSON.parse(inner) as Record; + + expect(parsed.type).toBe('object'); + const props = parsed.properties as Record>; + expect(props.num_pages).toMatchObject({ type: 'number', default: 1 }); + expect(props.down).toMatchObject({ type: 'boolean', default: true }); + expect(props.pages).toMatchObject({ type: 'number' }); + // _def-leak guard: these zod-internal keys must not appear anywhere. + expect(prompt).not.toContain('innerType'); + expect(prompt).not.toContain('defaultValue'); + expect(prompt).not.toContain('"def":'); + // $schema noise must be stripped. + expect(prompt).not.toContain('$schema'); + }); + + it('still respects skipKeys for done.success and extract.output_schema', () => { + const doneSchema = z.object({ + success: z.boolean(), + data: z.object({ value: z.string() }), + }); + const doneAction = new RegisteredAction( + 'done', + 'Finish', + async () => ({}), + doneSchema + ); + const donePrompt = doneAction.promptDescription(); + expect(donePrompt).toContain('data'); + expect(donePrompt).not.toContain('"success"'); + }); +}); From e260962280461b63f5bebd387a33261f8190947d Mon Sep 17 00:00:00 2001 From: Aleksey Bykhun Date: Tue, 5 May 2026 17:59:03 -0700 Subject: [PATCH 02/14] fix(agent): feed prettified zod issues + sent params back to LLM on retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When `_validateAndNormalizeActions` rejected an action's params via `actionInfo.paramSchema.safeParse(rawParams)`, the thrown error message was the raw `paramsResult.error.message` — i.e. zod v4's default JSON dump of the `issues` array (`[{"expected":"number","code":"invalid_type", "path":["num_pages"],"message":"Invalid input: expected number, received boolean"}]`). This noisy blob did flow into `state.last_result` and into the next `create_state_messages` turn, but it was hard for the model to parse and gave no corrective hint, so the model retried with the same mistake until `max_failures=3` tripped. Use `z.prettifyError(paramsResult.error)` (zod v4 native) to render issues as readable lines (e.g. `✖ Invalid input: expected number, received boolean → at num_pages`), include the offending params verbatim so the model can diff against the schema, and tag the message with an explicit `Schema validation failed` prefix plus a `Please retry with parameters matching the action's schema exactly` instruction. The existing pipeline does the rest: thrown Error → `_handle_step_error` → `state.last_result = [ActionResult({error: ...})]` → next step's `create_state_messages` injects it into the LLM context. No new injection mechanism, just a better-shaped payload going through the existing one. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/agent/service.ts | 11 +++- test/zod-error-feedback.test.ts | 109 ++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 test/zod-error-feedback.test.ts diff --git a/src/agent/service.ts b/src/agent/service.ts index ea101f45..bf23cefd 100644 --- a/src/agent/service.ts +++ b/src/agent/service.ts @@ -5623,8 +5623,17 @@ export class Agent< {}) as unknown; const paramsResult = actionInfo.paramSchema.safeParse(rawParams); if (!paramsResult.success) { + // Surface a human-readable issue list (zod v4 `prettifyError`) plus + // a corrective hint, rather than the default JSON dump of `.issues`. + // This Error propagates → `_handle_step_error` writes it into + // `state.last_result` → `create_state_messages` injects it into the + // next LLM turn, so the model knows exactly what shape it got wrong. + const pretty = z.prettifyError(paramsResult.error); + const sentParams = JSON.stringify(rawParams); throw new Error( - `Invalid parameters for action '${requestedActionName}': ${paramsResult.error.message}` + `Schema validation failed for action '${requestedActionName}'. ` + + `You sent: ${sentParams}. Issues:\n${pretty}\n` + + `Please retry with parameters matching the action's schema exactly.` ); } diff --git a/test/zod-error-feedback.test.ts b/test/zod-error-feedback.test.ts new file mode 100644 index 00000000..cdd0ee38 --- /dev/null +++ b/test/zod-error-feedback.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; +import { Agent } from '../src/agent/service.js'; +import { ActionResult } from '../src/agent/views.js'; +import { Controller } from '../src/controller/service.js'; +import type { BaseChatModel } from '../src/llm/base.js'; + +const createLlm = (): BaseChatModel => + ({ + model: 'gpt-test', + get provider() { + return 'test'; + }, + get name() { + return 'test'; + }, + get model_name() { + return 'gpt-test'; + }, + ainvoke: vi.fn(async () => ({ completion: 'ok', usage: null })), + }) as unknown as BaseChatModel; + +describe('zod validation error feedback to LLM', () => { + it('throws an error containing prettified zod issues + the bad params on schema mismatch', async () => { + const controller = new Controller(); + controller.registry.action('Scroll the page', { + param_model: z.object({ + down: z.boolean().default(true), + num_pages: z.number().default(1), + }), + })(async function scroll() { + return new ActionResult({}); + }); + + const agent = new Agent({ + task: 'reproduce zod schema mismatch', + llm: createLlm(), + controller, + }); + + try { + const badAction = { scroll: { num_pages: true } }; + expect(() => + (agent as any)._validateAndNormalizeActions([badAction]) + ).toThrow(/Schema validation failed for action 'scroll'/); + + let captured: Error | null = null; + try { + (agent as any)._validateAndNormalizeActions([badAction]); + } catch (e) { + captured = e as Error; + } + expect(captured).not.toBeNull(); + const msg = captured!.message; + // Pretty-printed zod path (not raw JSON dump of issues array). + expect(msg).toContain('num_pages'); + expect(msg).toContain('expected number'); + expect(msg).toContain('received boolean'); + // Echo of what the LLM sent — so it can self-correct. + expect(msg).toContain('"num_pages":true'); + // Corrective hint. + expect(msg).toMatch(/retry with parameters/i); + // Guard: should NOT be the raw zod-issues JSON dump. + expect(msg).not.toContain('"code":"invalid_type"'); + } finally { + await agent.close(); + } + }); + + it('error feeds into next prompt via state.last_result (existing pipeline)', async () => { + const controller = new Controller(); + controller.registry.action('Scroll the page', { + param_model: z.object({ + num_pages: z.number().default(1), + }), + })(async function scroll() { + return new ActionResult({}); + }); + + const agent = new Agent({ + task: 'verify error reaches state.last_result', + llm: createLlm(), + controller, + }); + + try { + const badAction = { scroll: { num_pages: 'three' } }; + let thrown: Error | null = null; + try { + (agent as any)._validateAndNormalizeActions([badAction]); + } catch (e) { + thrown = e as Error; + } + expect(thrown).not.toBeNull(); + // Simulate the agent loop's catch: this is exactly what + // `_handle_step_error` does with the thrown error. + (agent as any).state.last_result = [ + new ActionResult({ error: thrown!.message }), + ]; + + const errorOnState = (agent as any).state.last_result[0] + .error as string; + expect(errorOnState).toContain('Schema validation failed'); + expect(errorOnState).toContain('num_pages'); + } finally { + await agent.close(); + } + }); +}); From f8e8af97ef8d55631435857a091efd963adf1802 Mon Sep 17 00:00:00 2001 From: Aleksey Bykhun Date: Tue, 5 May 2026 18:51:05 -0700 Subject: [PATCH 03/14] chore(dist): commit prebuilt dist for direct fork install So consumers can `npm install github:caffeinum/browser-use` (no #ref) and get working code without needing a `prepare` script + devDeps + pnpm in their build environment. Source of truth is still src/; dist/ should be rebuilt and re-committed on top of any new src/ change. Co-Authored-By: Claude Opus 4.7 (1M context) --- dist/actor/element.d.ts | 19 + dist/actor/element.js | 46 + dist/actor/index.d.ts | 4 + dist/actor/index.js | 4 + dist/actor/mouse.d.ts | 19 + dist/actor/mouse.js | 39 + dist/actor/page.d.ts | 29 + dist/actor/page.js | 88 + dist/actor/utils.d.ts | 4 + dist/actor/utils.js | 35 + dist/agent/cloud-events.d.ts | 282 + dist/agent/cloud-events.js | 381 ++ dist/agent/gif.d.ts | 16 + dist/agent/gif.js | 237 + dist/agent/index.d.ts | 8 + dist/agent/index.js | 8 + dist/agent/judge.d.ts | 17 + dist/agent/judge.js | 197 + dist/agent/message-manager/service.d.ts | 38 + dist/agent/message-manager/service.js | 374 ++ dist/agent/message-manager/utils.d.ts | 2 + dist/agent/message-manager/utils.js | 40 + dist/agent/message-manager/views.d.ts | 30 + dist/agent/message-manager/views.js | 77 + dist/agent/prompts.d.ts | 73 + dist/agent/prompts.js | 474 ++ dist/agent/service.d.ts | 353 ++ dist/agent/service.js | 4089 +++++++++++++ dist/agent/system_prompt.md | 269 + dist/agent/system_prompt_anthropic_flash.md | 240 + dist/agent/system_prompt_browser_use.md | 18 + dist/agent/system_prompt_browser_use_flash.md | 15 + .../system_prompt_browser_use_no_thinking.md | 17 + dist/agent/system_prompt_flash.md | 16 + dist/agent/system_prompt_flash_anthropic.md | 30 + dist/agent/system_prompt_no_thinking.md | 245 + dist/agent/variable-detector.d.ts | 12 + dist/agent/variable-detector.js | 211 + dist/agent/views.d.ts | 961 +++ dist/agent/views.js | 951 +++ dist/browser/browser.d.ts | 7 + dist/browser/browser.js | 5 + dist/browser/cloud/cloud.d.ts | 20 + dist/browser/cloud/cloud.js | 129 + dist/browser/cloud/index.d.ts | 3 + dist/browser/cloud/index.js | 3 + dist/browser/cloud/management.d.ts | 130 + dist/browser/cloud/management.js | 140 + dist/browser/cloud/views.d.ts | 41 + dist/browser/cloud/views.js | 35 + dist/browser/context.d.ts | 8 + dist/browser/context.js | 4 + dist/browser/dvd-screensaver.d.ts | 101 + dist/browser/dvd-screensaver.js | 270 + dist/browser/events.d.ts | 403 ++ dist/browser/events.js | 632 ++ dist/browser/extensions.d.ts | 63 + dist/browser/extensions.js | 359 ++ dist/browser/index.d.ts | 14 + dist/browser/index.js | 13 + dist/browser/playwright-manager.d.ts | 47 + dist/browser/playwright-manager.js | 146 + dist/browser/profile.d.ts | 203 + dist/browser/profile.js | 899 +++ dist/browser/session-manager.d.ts | 85 + dist/browser/session-manager.js | 208 + dist/browser/session.d.ts | 658 ++ dist/browser/session.js | 5292 +++++++++++++++++ dist/browser/types.d.ts | 1181 ++++ dist/browser/types.js | 1 + dist/browser/utils.d.ts | 1 + dist/browser/utils.js | 19 + dist/browser/views.d.ts | 117 + dist/browser/views.js | 104 + .../watchdogs/aboutblank-watchdog.d.ts | 12 + dist/browser/watchdogs/aboutblank-watchdog.js | 131 + dist/browser/watchdogs/base.d.ts | 21 + dist/browser/watchdogs/base.js | 114 + dist/browser/watchdogs/captcha-watchdog.d.ts | 26 + dist/browser/watchdogs/captcha-watchdog.js | 151 + .../watchdogs/cdp-session-watchdog.d.ts | 14 + .../browser/watchdogs/cdp-session-watchdog.js | 177 + dist/browser/watchdogs/crash-watchdog.d.ts | 38 + dist/browser/watchdogs/crash-watchdog.js | 296 + .../watchdogs/default-action-watchdog.d.ts | 49 + .../watchdogs/default-action-watchdog.js | 212 + dist/browser/watchdogs/dom-watchdog.d.ts | 8 + dist/browser/watchdogs/dom-watchdog.js | 31 + .../browser/watchdogs/downloads-watchdog.d.ts | 77 + dist/browser/watchdogs/downloads-watchdog.js | 409 ++ .../watchdogs/har-recording-watchdog.d.ts | 19 + .../watchdogs/har-recording-watchdog.js | 317 + dist/browser/watchdogs/index.d.ts | 16 + dist/browser/watchdogs/index.js | 16 + .../watchdogs/local-browser-watchdog.d.ts | 10 + .../watchdogs/local-browser-watchdog.js | 32 + .../watchdogs/permissions-watchdog.d.ts | 8 + .../browser/watchdogs/permissions-watchdog.js | 73 + dist/browser/watchdogs/popups-watchdog.d.ts | 13 + dist/browser/watchdogs/popups-watchdog.js | 77 + .../browser/watchdogs/recording-watchdog.d.ts | 27 + dist/browser/watchdogs/recording-watchdog.js | 249 + .../watchdogs/screenshot-watchdog.d.ts | 6 + dist/browser/watchdogs/screenshot-watchdog.js | 14 + dist/browser/watchdogs/security-watchdog.d.ts | 10 + dist/browser/watchdogs/security-watchdog.js | 84 + .../watchdogs/storage-state-watchdog.d.ts | 24 + .../watchdogs/storage-state-watchdog.js | 288 + dist/cli.d.ts | 163 + dist/cli.js | 2666 +++++++++ dist/code-use/formatting.d.ts | 3 + dist/code-use/formatting.js | 18 + dist/code-use/index.d.ts | 6 + dist/code-use/index.js | 6 + dist/code-use/namespace.d.ts | 5 + dist/code-use/namespace.js | 81 + dist/code-use/notebook-export.d.ts | 3 + dist/code-use/notebook-export.js | 56 + dist/code-use/service.d.ts | 24 + dist/code-use/service.js | 104 + dist/code-use/utils.d.ts | 4 + dist/code-use/utils.js | 98 + dist/code-use/views.d.ts | 108 + dist/code-use/views.js | 165 + dist/config.d.ts | 123 + dist/config.js | 532 ++ dist/controller/index.d.ts | 3 + dist/controller/index.js | 3 + dist/controller/registry/index.d.ts | 2 + dist/controller/registry/index.js | 2 + dist/controller/registry/service.d.ts | 54 + dist/controller/registry/service.js | 440 ++ dist/controller/registry/views.d.ts | 58 + dist/controller/registry/views.js | 211 + dist/controller/service.d.ts | 58 + dist/controller/service.js | 2488 ++++++++ dist/controller/views.d.ts | 167 + dist/controller/views.js | 140 + .../clickable-element-processor/service.d.ts | 11 + .../clickable-element-processor/service.js | 60 + dist/dom/dom_tree/index.js | 1413 +++++ dist/dom/history-tree-processor/service.d.ts | 19 + dist/dom/history-tree-processor/service.js | 230 + dist/dom/history-tree-processor/view.d.ts | 60 + dist/dom/history-tree-processor/view.js | 65 + dist/dom/markdown-extractor.d.ts | 37 + dist/dom/markdown-extractor.js | 345 ++ dist/dom/playground/extraction.d.ts | 19 + dist/dom/playground/extraction.js | 187 + dist/dom/service.d.ts | 21 + dist/dom/service.js | 303 + dist/dom/utils.d.ts | 1 + dist/dom/utils.js | 6 + dist/dom/views.d.ts | 62 + dist/dom/views.js | 292 + dist/event-bus.d.ts | 111 + dist/event-bus.js | 322 + dist/exceptions.d.ts | 10 + dist/exceptions.js | 22 + dist/filesystem/file-system.d.ts | 86 + dist/filesystem/file-system.js | 900 +++ dist/filesystem/index.d.ts | 1 + dist/filesystem/index.js | 1 + dist/index.d.ts | 38 + dist/index.js | 39 + dist/integrations/gmail/actions.d.ts | 12 + dist/integrations/gmail/actions.js | 113 + dist/integrations/gmail/index.d.ts | 2 + dist/integrations/gmail/index.js | 2 + dist/integrations/gmail/service.d.ts | 61 + dist/integrations/gmail/service.js | 260 + dist/llm/anthropic/chat.d.ts | 45 + dist/llm/anthropic/chat.js | 194 + dist/llm/anthropic/index.d.ts | 2 + dist/llm/anthropic/index.js | 2 + dist/llm/anthropic/serializer.d.ts | 70 + dist/llm/anthropic/serializer.js | 357 ++ dist/llm/aws/chat-anthropic.d.ts | 78 + dist/llm/aws/chat-anthropic.js | 265 + dist/llm/aws/chat-bedrock.d.ts | 42 + dist/llm/aws/chat-bedrock.js | 207 + dist/llm/aws/index.d.ts | 3 + dist/llm/aws/index.js | 3 + dist/llm/aws/serializer.d.ts | 17 + dist/llm/aws/serializer.js | 107 + dist/llm/azure/chat.d.ts | 66 + dist/llm/azure/chat.js | 396 ++ dist/llm/azure/index.d.ts | 1 + dist/llm/azure/index.js | 1 + dist/llm/base.d.ts | 18 + dist/llm/base.js | 1 + dist/llm/browser-use/chat.d.ts | 40 + dist/llm/browser-use/chat.js | 305 + dist/llm/browser-use/index.d.ts | 1 + dist/llm/browser-use/index.js | 1 + dist/llm/cerebras/chat.d.ts | 39 + dist/llm/cerebras/chat.js | 178 + dist/llm/cerebras/index.d.ts | 2 + dist/llm/cerebras/index.js | 2 + dist/llm/cerebras/serializer.d.ts | 7 + dist/llm/cerebras/serializer.js | 82 + dist/llm/deepseek/chat.d.ts | 32 + dist/llm/deepseek/chat.js | 164 + dist/llm/deepseek/index.d.ts | 2 + dist/llm/deepseek/index.js | 2 + dist/llm/deepseek/serializer.d.ts | 6 + dist/llm/deepseek/serializer.js | 57 + dist/llm/exceptions.d.ts | 10 + dist/llm/exceptions.js | 18 + dist/llm/google/chat.d.ts | 64 + dist/llm/google/chat.js | 349 ++ dist/llm/google/index.d.ts | 2 + dist/llm/google/index.js | 2 + dist/llm/google/serializer.d.ts | 14 + dist/llm/google/serializer.js | 171 + dist/llm/groq/chat.d.ts | 34 + dist/llm/groq/chat.js | 151 + dist/llm/groq/index.d.ts | 3 + dist/llm/groq/index.js | 3 + dist/llm/groq/parser.d.ts | 32 + dist/llm/groq/parser.js | 191 + dist/llm/groq/serializer.d.ts | 6 + dist/llm/groq/serializer.js | 56 + dist/llm/litellm/chat.d.ts | 11 + dist/llm/litellm/chat.js | 16 + dist/llm/litellm/index.d.ts | 1 + dist/llm/litellm/index.js | 1 + dist/llm/messages.d.ts | 77 + dist/llm/messages.js | 157 + dist/llm/mistral/chat.d.ts | 43 + dist/llm/mistral/chat.js | 154 + dist/llm/mistral/index.d.ts | 2 + dist/llm/mistral/index.js | 2 + dist/llm/mistral/schema.d.ts | 8 + dist/llm/mistral/schema.js | 27 + dist/llm/models.d.ts | 2 + dist/llm/models.js | 343 ++ dist/llm/oci-raw/chat.d.ts | 64 + dist/llm/oci-raw/chat.js | 350 ++ dist/llm/oci-raw/index.d.ts | 2 + dist/llm/oci-raw/index.js | 2 + dist/llm/oci-raw/serializer.d.ts | 12 + dist/llm/oci-raw/serializer.js | 128 + dist/llm/ollama/chat.d.ts | 27 + dist/llm/ollama/chat.js | 168 + dist/llm/ollama/index.d.ts | 2 + dist/llm/ollama/index.js | 2 + dist/llm/ollama/serializer.d.ts | 7 + dist/llm/ollama/serializer.js | 75 + dist/llm/openai/chat.d.ts | 54 + dist/llm/openai/chat.js | 224 + dist/llm/openai/index.d.ts | 3 + dist/llm/openai/index.js | 3 + dist/llm/openai/like.d.ts | 19 + dist/llm/openai/like.js | 23 + dist/llm/openai/responses-serializer.d.ts | 18 + dist/llm/openai/responses-serializer.js | 72 + dist/llm/openai/serializer.d.ts | 6 + dist/llm/openai/serializer.js | 57 + dist/llm/openrouter/chat.d.ts | 41 + dist/llm/openrouter/chat.js | 160 + dist/llm/openrouter/index.d.ts | 2 + dist/llm/openrouter/index.js | 2 + dist/llm/openrouter/serializer.d.ts | 3 + dist/llm/openrouter/serializer.js | 3 + dist/llm/schema.d.ts | 16 + dist/llm/schema.js | 182 + dist/llm/vercel/chat.d.ts | 50 + dist/llm/vercel/chat.js | 276 + dist/llm/vercel/index.d.ts | 1 + dist/llm/vercel/index.js | 1 + dist/llm/vercel/serializer.d.ts | 5 + dist/llm/vercel/serializer.js | 7 + dist/llm/views.d.ts | 16 + dist/llm/views.js | 14 + dist/logging-config.d.ts | 27 + dist/logging-config.js | 142 + dist/mcp/client.d.ts | 147 + dist/mcp/client.js | 644 ++ dist/mcp/controller.d.ts | 45 + dist/mcp/controller.js | 63 + dist/mcp/index.d.ts | 3 + dist/mcp/index.js | 3 + dist/mcp/server.d.ts | 150 + dist/mcp/server.js | 1020 ++++ dist/observability-decorators.d.ts | 158 + dist/observability-decorators.js | 286 + dist/observability.d.ts | 23 + dist/observability.js | 64 + dist/sandbox/index.d.ts | 2 + dist/sandbox/index.js | 2 + dist/sandbox/sandbox.d.ts | 19 + dist/sandbox/sandbox.js | 140 + dist/sandbox/views.d.ts | 67 + dist/sandbox/views.js | 121 + dist/screenshots/index.d.ts | 1 + dist/screenshots/index.js | 1 + dist/screenshots/service.d.ts | 6 + dist/screenshots/service.js | 28 + dist/skill-cli/direct.d.ts | 100 + dist/skill-cli/direct.js | 984 +++ dist/skill-cli/index.d.ts | 5 + dist/skill-cli/index.js | 5 + dist/skill-cli/protocol.d.ts | 30 + dist/skill-cli/protocol.js | 48 + dist/skill-cli/server.d.ts | 13 + dist/skill-cli/server.js | 546 ++ dist/skill-cli/sessions.d.ts | 24 + dist/skill-cli/sessions.js | 47 + dist/skill-cli/tunnel.d.ts | 61 + dist/skill-cli/tunnel.js | 257 + dist/skills/index.d.ts | 3 + dist/skills/index.js | 3 + dist/skills/service.d.ts | 27 + dist/skills/service.js | 266 + dist/skills/utils.d.ts | 6 + dist/skills/utils.js | 53 + dist/skills/views.d.ts | 40 + dist/skills/views.js | 10 + dist/sync/auth.d.ts | 35 + dist/sync/auth.js | 222 + dist/sync/index.d.ts | 2 + dist/sync/index.js | 2 + dist/sync/service.d.ts | 21 + dist/sync/service.js | 111 + dist/telemetry/index.d.ts | 2 + dist/telemetry/index.js | 2 + dist/telemetry/service.d.ts | 12 + dist/telemetry/service.js | 85 + dist/telemetry/views.d.ts | 126 + dist/telemetry/views.js | 130 + dist/tokens/custom-pricing.d.ts | 2 + dist/tokens/custom-pricing.js | 22 + dist/tokens/index.d.ts | 4 + dist/tokens/index.js | 4 + dist/tokens/mappings.d.ts | 1 + dist/tokens/mappings.js | 3 + dist/tokens/service.d.ts | 35 + dist/tokens/service.js | 441 ++ dist/tokens/views.d.ts | 58 + dist/tokens/views.js | 1 + dist/tools/extraction/index.d.ts | 2 + dist/tools/extraction/index.js | 2 + dist/tools/extraction/schema-utils.d.ts | 6 + dist/tools/extraction/schema-utils.js | 237 + dist/tools/extraction/views.d.ts | 7 + dist/tools/extraction/views.js | 1 + dist/tools/index.d.ts | 5 + dist/tools/index.js | 5 + dist/tools/registry/index.d.ts | 2 + dist/tools/registry/index.js | 2 + dist/tools/registry/service.d.ts | 1 + dist/tools/registry/service.js | 1 + dist/tools/registry/views.d.ts | 1 + dist/tools/registry/views.js | 1 + dist/tools/service.d.ts | 2 + dist/tools/service.js | 1 + dist/tools/utils.d.ts | 2 + dist/tools/utils.js | 57 + dist/tools/views.d.ts | 1 + dist/tools/views.js | 1 + dist/utils.d.ts | 137 + dist/utils.js | 597 ++ 363 files changed, 52184 insertions(+) create mode 100644 dist/actor/element.d.ts create mode 100644 dist/actor/element.js create mode 100644 dist/actor/index.d.ts create mode 100644 dist/actor/index.js create mode 100644 dist/actor/mouse.d.ts create mode 100644 dist/actor/mouse.js create mode 100644 dist/actor/page.d.ts create mode 100644 dist/actor/page.js create mode 100644 dist/actor/utils.d.ts create mode 100644 dist/actor/utils.js create mode 100644 dist/agent/cloud-events.d.ts create mode 100644 dist/agent/cloud-events.js create mode 100644 dist/agent/gif.d.ts create mode 100644 dist/agent/gif.js create mode 100644 dist/agent/index.d.ts create mode 100644 dist/agent/index.js create mode 100644 dist/agent/judge.d.ts create mode 100644 dist/agent/judge.js create mode 100644 dist/agent/message-manager/service.d.ts create mode 100644 dist/agent/message-manager/service.js create mode 100644 dist/agent/message-manager/utils.d.ts create mode 100644 dist/agent/message-manager/utils.js create mode 100644 dist/agent/message-manager/views.d.ts create mode 100644 dist/agent/message-manager/views.js create mode 100644 dist/agent/prompts.d.ts create mode 100644 dist/agent/prompts.js create mode 100644 dist/agent/service.d.ts create mode 100644 dist/agent/service.js create mode 100644 dist/agent/system_prompt.md create mode 100644 dist/agent/system_prompt_anthropic_flash.md create mode 100644 dist/agent/system_prompt_browser_use.md create mode 100644 dist/agent/system_prompt_browser_use_flash.md create mode 100644 dist/agent/system_prompt_browser_use_no_thinking.md create mode 100644 dist/agent/system_prompt_flash.md create mode 100644 dist/agent/system_prompt_flash_anthropic.md create mode 100644 dist/agent/system_prompt_no_thinking.md create mode 100644 dist/agent/variable-detector.d.ts create mode 100644 dist/agent/variable-detector.js create mode 100644 dist/agent/views.d.ts create mode 100644 dist/agent/views.js create mode 100644 dist/browser/browser.d.ts create mode 100644 dist/browser/browser.js create mode 100644 dist/browser/cloud/cloud.d.ts create mode 100644 dist/browser/cloud/cloud.js create mode 100644 dist/browser/cloud/index.d.ts create mode 100644 dist/browser/cloud/index.js create mode 100644 dist/browser/cloud/management.d.ts create mode 100644 dist/browser/cloud/management.js create mode 100644 dist/browser/cloud/views.d.ts create mode 100644 dist/browser/cloud/views.js create mode 100644 dist/browser/context.d.ts create mode 100644 dist/browser/context.js create mode 100644 dist/browser/dvd-screensaver.d.ts create mode 100644 dist/browser/dvd-screensaver.js create mode 100644 dist/browser/events.d.ts create mode 100644 dist/browser/events.js create mode 100644 dist/browser/extensions.d.ts create mode 100644 dist/browser/extensions.js create mode 100644 dist/browser/index.d.ts create mode 100644 dist/browser/index.js create mode 100644 dist/browser/playwright-manager.d.ts create mode 100644 dist/browser/playwright-manager.js create mode 100644 dist/browser/profile.d.ts create mode 100644 dist/browser/profile.js create mode 100644 dist/browser/session-manager.d.ts create mode 100644 dist/browser/session-manager.js create mode 100644 dist/browser/session.d.ts create mode 100644 dist/browser/session.js create mode 100644 dist/browser/types.d.ts create mode 100644 dist/browser/types.js create mode 100644 dist/browser/utils.d.ts create mode 100644 dist/browser/utils.js create mode 100644 dist/browser/views.d.ts create mode 100644 dist/browser/views.js create mode 100644 dist/browser/watchdogs/aboutblank-watchdog.d.ts create mode 100644 dist/browser/watchdogs/aboutblank-watchdog.js create mode 100644 dist/browser/watchdogs/base.d.ts create mode 100644 dist/browser/watchdogs/base.js create mode 100644 dist/browser/watchdogs/captcha-watchdog.d.ts create mode 100644 dist/browser/watchdogs/captcha-watchdog.js create mode 100644 dist/browser/watchdogs/cdp-session-watchdog.d.ts create mode 100644 dist/browser/watchdogs/cdp-session-watchdog.js create mode 100644 dist/browser/watchdogs/crash-watchdog.d.ts create mode 100644 dist/browser/watchdogs/crash-watchdog.js create mode 100644 dist/browser/watchdogs/default-action-watchdog.d.ts create mode 100644 dist/browser/watchdogs/default-action-watchdog.js create mode 100644 dist/browser/watchdogs/dom-watchdog.d.ts create mode 100644 dist/browser/watchdogs/dom-watchdog.js create mode 100644 dist/browser/watchdogs/downloads-watchdog.d.ts create mode 100644 dist/browser/watchdogs/downloads-watchdog.js create mode 100644 dist/browser/watchdogs/har-recording-watchdog.d.ts create mode 100644 dist/browser/watchdogs/har-recording-watchdog.js create mode 100644 dist/browser/watchdogs/index.d.ts create mode 100644 dist/browser/watchdogs/index.js create mode 100644 dist/browser/watchdogs/local-browser-watchdog.d.ts create mode 100644 dist/browser/watchdogs/local-browser-watchdog.js create mode 100644 dist/browser/watchdogs/permissions-watchdog.d.ts create mode 100644 dist/browser/watchdogs/permissions-watchdog.js create mode 100644 dist/browser/watchdogs/popups-watchdog.d.ts create mode 100644 dist/browser/watchdogs/popups-watchdog.js create mode 100644 dist/browser/watchdogs/recording-watchdog.d.ts create mode 100644 dist/browser/watchdogs/recording-watchdog.js create mode 100644 dist/browser/watchdogs/screenshot-watchdog.d.ts create mode 100644 dist/browser/watchdogs/screenshot-watchdog.js create mode 100644 dist/browser/watchdogs/security-watchdog.d.ts create mode 100644 dist/browser/watchdogs/security-watchdog.js create mode 100644 dist/browser/watchdogs/storage-state-watchdog.d.ts create mode 100644 dist/browser/watchdogs/storage-state-watchdog.js create mode 100644 dist/cli.d.ts create mode 100644 dist/cli.js create mode 100644 dist/code-use/formatting.d.ts create mode 100644 dist/code-use/formatting.js create mode 100644 dist/code-use/index.d.ts create mode 100644 dist/code-use/index.js create mode 100644 dist/code-use/namespace.d.ts create mode 100644 dist/code-use/namespace.js create mode 100644 dist/code-use/notebook-export.d.ts create mode 100644 dist/code-use/notebook-export.js create mode 100644 dist/code-use/service.d.ts create mode 100644 dist/code-use/service.js create mode 100644 dist/code-use/utils.d.ts create mode 100644 dist/code-use/utils.js create mode 100644 dist/code-use/views.d.ts create mode 100644 dist/code-use/views.js create mode 100644 dist/config.d.ts create mode 100644 dist/config.js create mode 100644 dist/controller/index.d.ts create mode 100644 dist/controller/index.js create mode 100644 dist/controller/registry/index.d.ts create mode 100644 dist/controller/registry/index.js create mode 100644 dist/controller/registry/service.d.ts create mode 100644 dist/controller/registry/service.js create mode 100644 dist/controller/registry/views.d.ts create mode 100644 dist/controller/registry/views.js create mode 100644 dist/controller/service.d.ts create mode 100644 dist/controller/service.js create mode 100644 dist/controller/views.d.ts create mode 100644 dist/controller/views.js create mode 100644 dist/dom/clickable-element-processor/service.d.ts create mode 100644 dist/dom/clickable-element-processor/service.js create mode 100644 dist/dom/dom_tree/index.js create mode 100644 dist/dom/history-tree-processor/service.d.ts create mode 100644 dist/dom/history-tree-processor/service.js create mode 100644 dist/dom/history-tree-processor/view.d.ts create mode 100644 dist/dom/history-tree-processor/view.js create mode 100644 dist/dom/markdown-extractor.d.ts create mode 100644 dist/dom/markdown-extractor.js create mode 100644 dist/dom/playground/extraction.d.ts create mode 100644 dist/dom/playground/extraction.js create mode 100644 dist/dom/service.d.ts create mode 100644 dist/dom/service.js create mode 100644 dist/dom/utils.d.ts create mode 100644 dist/dom/utils.js create mode 100644 dist/dom/views.d.ts create mode 100644 dist/dom/views.js create mode 100644 dist/event-bus.d.ts create mode 100644 dist/event-bus.js create mode 100644 dist/exceptions.d.ts create mode 100644 dist/exceptions.js create mode 100644 dist/filesystem/file-system.d.ts create mode 100644 dist/filesystem/file-system.js create mode 100644 dist/filesystem/index.d.ts create mode 100644 dist/filesystem/index.js create mode 100644 dist/index.d.ts create mode 100644 dist/index.js create mode 100644 dist/integrations/gmail/actions.d.ts create mode 100644 dist/integrations/gmail/actions.js create mode 100644 dist/integrations/gmail/index.d.ts create mode 100644 dist/integrations/gmail/index.js create mode 100644 dist/integrations/gmail/service.d.ts create mode 100644 dist/integrations/gmail/service.js create mode 100644 dist/llm/anthropic/chat.d.ts create mode 100644 dist/llm/anthropic/chat.js create mode 100644 dist/llm/anthropic/index.d.ts create mode 100644 dist/llm/anthropic/index.js create mode 100644 dist/llm/anthropic/serializer.d.ts create mode 100644 dist/llm/anthropic/serializer.js create mode 100644 dist/llm/aws/chat-anthropic.d.ts create mode 100644 dist/llm/aws/chat-anthropic.js create mode 100644 dist/llm/aws/chat-bedrock.d.ts create mode 100644 dist/llm/aws/chat-bedrock.js create mode 100644 dist/llm/aws/index.d.ts create mode 100644 dist/llm/aws/index.js create mode 100644 dist/llm/aws/serializer.d.ts create mode 100644 dist/llm/aws/serializer.js create mode 100644 dist/llm/azure/chat.d.ts create mode 100644 dist/llm/azure/chat.js create mode 100644 dist/llm/azure/index.d.ts create mode 100644 dist/llm/azure/index.js create mode 100644 dist/llm/base.d.ts create mode 100644 dist/llm/base.js create mode 100644 dist/llm/browser-use/chat.d.ts create mode 100644 dist/llm/browser-use/chat.js create mode 100644 dist/llm/browser-use/index.d.ts create mode 100644 dist/llm/browser-use/index.js create mode 100644 dist/llm/cerebras/chat.d.ts create mode 100644 dist/llm/cerebras/chat.js create mode 100644 dist/llm/cerebras/index.d.ts create mode 100644 dist/llm/cerebras/index.js create mode 100644 dist/llm/cerebras/serializer.d.ts create mode 100644 dist/llm/cerebras/serializer.js create mode 100644 dist/llm/deepseek/chat.d.ts create mode 100644 dist/llm/deepseek/chat.js create mode 100644 dist/llm/deepseek/index.d.ts create mode 100644 dist/llm/deepseek/index.js create mode 100644 dist/llm/deepseek/serializer.d.ts create mode 100644 dist/llm/deepseek/serializer.js create mode 100644 dist/llm/exceptions.d.ts create mode 100644 dist/llm/exceptions.js create mode 100644 dist/llm/google/chat.d.ts create mode 100644 dist/llm/google/chat.js create mode 100644 dist/llm/google/index.d.ts create mode 100644 dist/llm/google/index.js create mode 100644 dist/llm/google/serializer.d.ts create mode 100644 dist/llm/google/serializer.js create mode 100644 dist/llm/groq/chat.d.ts create mode 100644 dist/llm/groq/chat.js create mode 100644 dist/llm/groq/index.d.ts create mode 100644 dist/llm/groq/index.js create mode 100644 dist/llm/groq/parser.d.ts create mode 100644 dist/llm/groq/parser.js create mode 100644 dist/llm/groq/serializer.d.ts create mode 100644 dist/llm/groq/serializer.js create mode 100644 dist/llm/litellm/chat.d.ts create mode 100644 dist/llm/litellm/chat.js create mode 100644 dist/llm/litellm/index.d.ts create mode 100644 dist/llm/litellm/index.js create mode 100644 dist/llm/messages.d.ts create mode 100644 dist/llm/messages.js create mode 100644 dist/llm/mistral/chat.d.ts create mode 100644 dist/llm/mistral/chat.js create mode 100644 dist/llm/mistral/index.d.ts create mode 100644 dist/llm/mistral/index.js create mode 100644 dist/llm/mistral/schema.d.ts create mode 100644 dist/llm/mistral/schema.js create mode 100644 dist/llm/models.d.ts create mode 100644 dist/llm/models.js create mode 100644 dist/llm/oci-raw/chat.d.ts create mode 100644 dist/llm/oci-raw/chat.js create mode 100644 dist/llm/oci-raw/index.d.ts create mode 100644 dist/llm/oci-raw/index.js create mode 100644 dist/llm/oci-raw/serializer.d.ts create mode 100644 dist/llm/oci-raw/serializer.js create mode 100644 dist/llm/ollama/chat.d.ts create mode 100644 dist/llm/ollama/chat.js create mode 100644 dist/llm/ollama/index.d.ts create mode 100644 dist/llm/ollama/index.js create mode 100644 dist/llm/ollama/serializer.d.ts create mode 100644 dist/llm/ollama/serializer.js create mode 100644 dist/llm/openai/chat.d.ts create mode 100644 dist/llm/openai/chat.js create mode 100644 dist/llm/openai/index.d.ts create mode 100644 dist/llm/openai/index.js create mode 100644 dist/llm/openai/like.d.ts create mode 100644 dist/llm/openai/like.js create mode 100644 dist/llm/openai/responses-serializer.d.ts create mode 100644 dist/llm/openai/responses-serializer.js create mode 100644 dist/llm/openai/serializer.d.ts create mode 100644 dist/llm/openai/serializer.js create mode 100644 dist/llm/openrouter/chat.d.ts create mode 100644 dist/llm/openrouter/chat.js create mode 100644 dist/llm/openrouter/index.d.ts create mode 100644 dist/llm/openrouter/index.js create mode 100644 dist/llm/openrouter/serializer.d.ts create mode 100644 dist/llm/openrouter/serializer.js create mode 100644 dist/llm/schema.d.ts create mode 100644 dist/llm/schema.js create mode 100644 dist/llm/vercel/chat.d.ts create mode 100644 dist/llm/vercel/chat.js create mode 100644 dist/llm/vercel/index.d.ts create mode 100644 dist/llm/vercel/index.js create mode 100644 dist/llm/vercel/serializer.d.ts create mode 100644 dist/llm/vercel/serializer.js create mode 100644 dist/llm/views.d.ts create mode 100644 dist/llm/views.js create mode 100644 dist/logging-config.d.ts create mode 100644 dist/logging-config.js create mode 100644 dist/mcp/client.d.ts create mode 100644 dist/mcp/client.js create mode 100644 dist/mcp/controller.d.ts create mode 100644 dist/mcp/controller.js create mode 100644 dist/mcp/index.d.ts create mode 100644 dist/mcp/index.js create mode 100644 dist/mcp/server.d.ts create mode 100644 dist/mcp/server.js create mode 100644 dist/observability-decorators.d.ts create mode 100644 dist/observability-decorators.js create mode 100644 dist/observability.d.ts create mode 100644 dist/observability.js create mode 100644 dist/sandbox/index.d.ts create mode 100644 dist/sandbox/index.js create mode 100644 dist/sandbox/sandbox.d.ts create mode 100644 dist/sandbox/sandbox.js create mode 100644 dist/sandbox/views.d.ts create mode 100644 dist/sandbox/views.js create mode 100644 dist/screenshots/index.d.ts create mode 100644 dist/screenshots/index.js create mode 100644 dist/screenshots/service.d.ts create mode 100644 dist/screenshots/service.js create mode 100644 dist/skill-cli/direct.d.ts create mode 100644 dist/skill-cli/direct.js create mode 100644 dist/skill-cli/index.d.ts create mode 100644 dist/skill-cli/index.js create mode 100644 dist/skill-cli/protocol.d.ts create mode 100644 dist/skill-cli/protocol.js create mode 100644 dist/skill-cli/server.d.ts create mode 100644 dist/skill-cli/server.js create mode 100644 dist/skill-cli/sessions.d.ts create mode 100644 dist/skill-cli/sessions.js create mode 100644 dist/skill-cli/tunnel.d.ts create mode 100644 dist/skill-cli/tunnel.js create mode 100644 dist/skills/index.d.ts create mode 100644 dist/skills/index.js create mode 100644 dist/skills/service.d.ts create mode 100644 dist/skills/service.js create mode 100644 dist/skills/utils.d.ts create mode 100644 dist/skills/utils.js create mode 100644 dist/skills/views.d.ts create mode 100644 dist/skills/views.js create mode 100644 dist/sync/auth.d.ts create mode 100644 dist/sync/auth.js create mode 100644 dist/sync/index.d.ts create mode 100644 dist/sync/index.js create mode 100644 dist/sync/service.d.ts create mode 100644 dist/sync/service.js create mode 100644 dist/telemetry/index.d.ts create mode 100644 dist/telemetry/index.js create mode 100644 dist/telemetry/service.d.ts create mode 100644 dist/telemetry/service.js create mode 100644 dist/telemetry/views.d.ts create mode 100644 dist/telemetry/views.js create mode 100644 dist/tokens/custom-pricing.d.ts create mode 100644 dist/tokens/custom-pricing.js create mode 100644 dist/tokens/index.d.ts create mode 100644 dist/tokens/index.js create mode 100644 dist/tokens/mappings.d.ts create mode 100644 dist/tokens/mappings.js create mode 100644 dist/tokens/service.d.ts create mode 100644 dist/tokens/service.js create mode 100644 dist/tokens/views.d.ts create mode 100644 dist/tokens/views.js create mode 100644 dist/tools/extraction/index.d.ts create mode 100644 dist/tools/extraction/index.js create mode 100644 dist/tools/extraction/schema-utils.d.ts create mode 100644 dist/tools/extraction/schema-utils.js create mode 100644 dist/tools/extraction/views.d.ts create mode 100644 dist/tools/extraction/views.js create mode 100644 dist/tools/index.d.ts create mode 100644 dist/tools/index.js create mode 100644 dist/tools/registry/index.d.ts create mode 100644 dist/tools/registry/index.js create mode 100644 dist/tools/registry/service.d.ts create mode 100644 dist/tools/registry/service.js create mode 100644 dist/tools/registry/views.d.ts create mode 100644 dist/tools/registry/views.js create mode 100644 dist/tools/service.d.ts create mode 100644 dist/tools/service.js create mode 100644 dist/tools/utils.d.ts create mode 100644 dist/tools/utils.js create mode 100644 dist/tools/views.d.ts create mode 100644 dist/tools/views.js create mode 100644 dist/utils.d.ts create mode 100644 dist/utils.js diff --git a/dist/actor/element.d.ts b/dist/actor/element.d.ts new file mode 100644 index 00000000..ae5daf1d --- /dev/null +++ b/dist/actor/element.d.ts @@ -0,0 +1,19 @@ +import type { BrowserSession } from '../browser/session.js'; +import type { DOMElementNode } from '../dom/views.js'; +export declare class Element { + private readonly browser_session; + readonly node: DOMElementNode; + constructor(browser_session: BrowserSession, node: DOMElementNode); + click(): Promise; + fill(value: string, clear?: boolean): Promise; + hover(): Promise; + get_attribute(name: string): Promise; + get_bounding_box(): Promise<{ + x: number; + y: number; + width: number; + height: number; + } | null>; + select_option(values: string | string[]): Promise; + evaluate(page_function: string, ...args: unknown[]): Promise; +} diff --git a/dist/actor/element.js b/dist/actor/element.js new file mode 100644 index 00000000..41b38fdc --- /dev/null +++ b/dist/actor/element.js @@ -0,0 +1,46 @@ +export class Element { + browser_session; + node; + constructor(browser_session, node) { + this.browser_session = browser_session; + this.node = node; + } + async click() { + return this.browser_session._click_element_node(this.node); + } + async fill(value, clear = true) { + return this.browser_session._input_text_element_node(this.node, value, { + clear, + }); + } + async hover() { + const locator = await this.browser_session.get_locate_element(this.node); + if (!locator?.hover) { + return; + } + await locator.hover({ timeout: 5000 }); + } + async get_attribute(name) { + return this.node.attributes?.[name] ?? null; + } + async get_bounding_box() { + const locator = await this.browser_session.get_locate_element(this.node); + if (!locator?.boundingBox) { + return null; + } + return locator.boundingBox(); + } + async select_option(values) { + const list = Array.isArray(values) ? values : [values]; + for (const value of list) { + await this.browser_session.select_dropdown_option(this.node, value); + } + } + async evaluate(page_function, ...args) { + const locator = await this.browser_session.get_locate_element(this.node); + if (!locator?.evaluate) { + throw new Error('Element evaluate is unavailable for this node'); + } + return locator.evaluate(page_function, ...args); + } +} diff --git a/dist/actor/index.d.ts b/dist/actor/index.d.ts new file mode 100644 index 00000000..81054292 --- /dev/null +++ b/dist/actor/index.d.ts @@ -0,0 +1,4 @@ +export * from './element.js'; +export * from './mouse.js'; +export * from './page.js'; +export * from './utils.js'; diff --git a/dist/actor/index.js b/dist/actor/index.js new file mode 100644 index 00000000..81054292 --- /dev/null +++ b/dist/actor/index.js @@ -0,0 +1,4 @@ +export * from './element.js'; +export * from './mouse.js'; +export * from './page.js'; +export * from './utils.js'; diff --git a/dist/actor/mouse.d.ts b/dist/actor/mouse.d.ts new file mode 100644 index 00000000..3e667a90 --- /dev/null +++ b/dist/actor/mouse.d.ts @@ -0,0 +1,19 @@ +import type { BrowserSession } from '../browser/session.js'; +import type { MouseButton } from '../browser/events.js'; +export declare class Mouse { + private readonly browser_session; + private readonly pageRef; + constructor(browser_session: BrowserSession, pageRef?: any | null); + private _page; + click(x: number, y: number, options?: { + button?: MouseButton; + click_count?: number; + }): Promise; + move(x: number, y: number): Promise; + down(options?: { + button?: MouseButton; + }): Promise; + up(options?: { + button?: MouseButton; + }): Promise; +} diff --git a/dist/actor/mouse.js b/dist/actor/mouse.js new file mode 100644 index 00000000..0a02e4ba --- /dev/null +++ b/dist/actor/mouse.js @@ -0,0 +1,39 @@ +export class Mouse { + browser_session; + pageRef; + constructor(browser_session, pageRef = null) { + this.browser_session = browser_session; + this.pageRef = pageRef; + } + async _page() { + if (this.pageRef) { + return this.pageRef; + } + return this.browser_session.get_current_page(); + } + async click(x, y, options = {}) { + const button = options.button ?? 'left'; + await this.browser_session.click_coordinates(x, y, { button }); + } + async move(x, y) { + const page = await this._page(); + if (!page?.mouse?.move) { + return; + } + await page.mouse.move(x, y); + } + async down(options = {}) { + const page = await this._page(); + if (!page?.mouse?.down) { + return; + } + await page.mouse.down({ button: options.button ?? 'left' }); + } + async up(options = {}) { + const page = await this._page(); + if (!page?.mouse?.up) { + return; + } + await page.mouse.up({ button: options.button ?? 'left' }); + } +} diff --git a/dist/actor/page.d.ts b/dist/actor/page.d.ts new file mode 100644 index 00000000..4f46bb15 --- /dev/null +++ b/dist/actor/page.d.ts @@ -0,0 +1,29 @@ +import type { WaitUntilState } from '../browser/events.js'; +import type { BrowserSession } from '../browser/session.js'; +import { Element } from './element.js'; +import { Mouse } from './mouse.js'; +export declare class Page { + private readonly browser_session; + private _mouse; + constructor(browser_session: BrowserSession); + get mouse(): Mouse; + _currentPage(): Promise; + get_url(): Promise; + get_title(): Promise; + goto(url: string, options?: { + wait_until?: WaitUntilState; + timeout_ms?: number | null; + }): Promise; + navigate(url: string, options?: Parameters[1]): Promise; + reload(): Promise; + go_back(): Promise; + go_forward(): Promise; + evaluate(page_function: string | ((...args: unknown[]) => unknown), ...args: unknown[]): Promise; + screenshot(options?: { + full_page?: boolean; + }): Promise; + press(key: string): Promise; + set_viewport_size(width: number, height: number): Promise; + get_element_by_index(index: number): Promise; + must_get_element_by_index(index: number): Promise; +} diff --git a/dist/actor/page.js b/dist/actor/page.js new file mode 100644 index 00000000..2dae6d3b --- /dev/null +++ b/dist/actor/page.js @@ -0,0 +1,88 @@ +import { Element } from './element.js'; +import { Mouse } from './mouse.js'; +export class Page { + browser_session; + _mouse = null; + constructor(browser_session) { + this.browser_session = browser_session; + } + get mouse() { + if (!this._mouse) { + this._mouse = new Mouse(this.browser_session); + } + return this._mouse; + } + async _currentPage() { + const page = await this.browser_session.get_current_page(); + if (!page) { + throw new Error('No active page available'); + } + return page; + } + async get_url() { + const page = await this._currentPage(); + return typeof page.url === 'function' ? page.url() : ''; + } + async get_title() { + const page = await this._currentPage(); + return typeof page.title === 'function' ? page.title() : ''; + } + async goto(url, options = {}) { + await this.browser_session.navigate_to(url, { + wait_until: options.wait_until, + timeout_ms: options.timeout_ms, + }); + } + async navigate(url, options = {}) { + await this.goto(url, options); + } + async reload() { + await this.browser_session.refresh(); + } + async go_back() { + await this.browser_session.go_back(); + } + async go_forward() { + await this.browser_session.go_forward(); + } + async evaluate(page_function, ...args) { + const page = await this._currentPage(); + if (typeof page_function === 'function') { + return page.evaluate(page_function, ...args); + } + if (args.length === 0) { + return page.evaluate(page_function); + } + const expression = `(${page_function})(${args + .map((arg) => JSON.stringify(arg)) + .join(',')})`; + return page.evaluate(expression); + } + async screenshot(options = {}) { + return this.browser_session.take_screenshot(options.full_page ?? false); + } + async press(key) { + await this.browser_session.send_keys(key); + } + async set_viewport_size(width, height) { + const page = await this._currentPage(); + if (!page.setViewportSize) { + return; + } + await page.setViewportSize({ width, height }); + } + async get_element_by_index(index) { + const node = await this.browser_session.get_dom_element_by_index(index); + if (!node) { + return null; + } + return new Element(this.browser_session, node); + } + async must_get_element_by_index(index) { + const element = await this.get_element_by_index(index); + if (!element) { + throw new Error(`Element not found for index ${index}`); + } + return element; + } +} diff --git a/dist/actor/utils.d.ts b/dist/actor/utils.d.ts new file mode 100644 index 00000000..115fdde2 --- /dev/null +++ b/dist/actor/utils.d.ts @@ -0,0 +1,4 @@ +export declare class Utils { + static get_key_info(key: string): [string, number | null]; +} +export declare const get_key_info: (key: string) => [string, number | null]; diff --git a/dist/actor/utils.js b/dist/actor/utils.js new file mode 100644 index 00000000..caf230b6 --- /dev/null +++ b/dist/actor/utils.js @@ -0,0 +1,35 @@ +export class Utils { + static get_key_info(key) { + const key_map = { + Backspace: ['Backspace', 8], + Tab: ['Tab', 9], + Enter: ['Enter', 13], + Escape: ['Escape', 27], + Space: ['Space', 32], + ArrowLeft: ['ArrowLeft', 37], + ArrowUp: ['ArrowUp', 38], + ArrowRight: ['ArrowRight', 39], + ArrowDown: ['ArrowDown', 40], + Delete: ['Delete', 46], + Shift: ['ShiftLeft', 16], + Control: ['ControlLeft', 17], + Alt: ['AltLeft', 18], + Meta: ['MetaLeft', 91], + }; + const direct = key_map[key]; + if (direct) { + return direct; + } + if (key.length === 1) { + if (/[a-z]/i.test(key)) { + const upper = key.toUpperCase(); + return [`Key${upper}`, upper.charCodeAt(0)]; + } + if (/[0-9]/.test(key)) { + return [`Digit${key}`, key.charCodeAt(0)]; + } + } + return [key, null]; + } +} +export const get_key_info = (key) => Utils.get_key_info(key); diff --git a/dist/agent/cloud-events.d.ts b/dist/agent/cloud-events.d.ts new file mode 100644 index 00000000..33ca7c94 --- /dev/null +++ b/dist/agent/cloud-events.d.ts @@ -0,0 +1,282 @@ +interface AgentReference { + task_id: string; + session_id: string; + task: string; + llm: { + model?: string; + model_name?: string; + }; + state: { + stopped: boolean; + paused: boolean; + n_steps: number; + model_dump?: () => Record; + }; + history: { + final_result(): string | null; + is_done(): boolean; + }; + browser_session: { + id: string; + browser_profile?: { + viewport?: { + width: number; + height: number; + }; + user_agent?: string | null; + headless?: boolean; + allowed_domains?: string[]; + }; + }; + browser_profile?: { + viewport?: { + width: number; + height: number; + }; + user_agent?: string | null; + headless?: boolean; + allowed_domains?: string[]; + }; + cloud_sync?: { + auth_client?: { + device_id?: string | null; + }; + }; + _task_start_time?: number; +} +interface AgentWithState extends AgentReference { + state: AgentReference['state'] & { + model_dump?: () => Record; + }; +} +export declare abstract class BaseEvent { + readonly event_type: string; + id: string; + user_id: string; + device_id: string | null; + protected constructor(event_type: string, init?: Partial); + toJSON(): { + event_type: string; + id: string; + user_id: string; + device_id: string | null; + }; +} +export declare class UpdateAgentTaskEvent extends BaseEvent { + stopped: boolean | null; + paused: boolean | null; + done_output: string | null; + finished_at: Date | null; + agent_state: Record | null; + user_feedback_type: string | null; + user_comment: string | null; + gif_url: string | null; + constructor(init: Partial & { + id: string; + }); + static fromAgent(agent: AgentWithState): UpdateAgentTaskEvent; + toJSON(): { + stopped: boolean | null; + paused: boolean | null; + done_output: string | null; + finished_at: string | null; + agent_state: Record | null; + user_feedback_type: string | null; + user_comment: string | null; + gif_url: string | null; + event_type: string; + id: string; + user_id: string; + device_id: string | null; + }; +} +export declare class CreateAgentOutputFileEvent extends BaseEvent { + task_id: string; + file_name: string; + file_content: string | null; + content_type: string | null; + created_at: Date; + constructor(init: { + user_id?: string; + device_id?: string | null; + task_id: string; + id?: string; + file_name: string; + file_content?: string | null; + content_type?: string | null; + created_at?: Date; + }); + static fromAgentAndFile(agent: AgentReference, outputPath: string): Promise; + toJSON(): { + task_id: string; + file_name: string; + file_content: string | null; + content_type: string | null; + created_at: string; + event_type: string; + id: string; + user_id: string; + device_id: string | null; + }; +} +export declare class CreateAgentStepEvent extends BaseEvent { + created_at: Date; + agent_task_id: string; + step: number; + evaluation_previous_goal: string; + memory: string; + next_goal: string; + actions: Array>; + screenshot_url: string | null; + url: string; + constructor(init: { + user_id?: string; + device_id?: string | null; + agent_task_id: string; + id?: string; + step: number; + evaluation_previous_goal: string; + memory: string; + next_goal: string; + actions: Array>; + screenshot_url?: string | null; + url: string; + created_at?: Date; + }); + static fromAgentStep(agent: AgentWithState, model_output: { + current_state: { + evaluation_previous_goal: string; + memory: string; + next_goal: string; + }; + action: any[]; + }, result: Array, actions_data: Array>, browser_state_summary: { + screenshot?: string | null; + url: string; + }): CreateAgentStepEvent; + toJSON(): { + created_at: string; + agent_task_id: string; + step: number; + evaluation_previous_goal: string; + memory: string; + next_goal: string; + actions: Record[]; + screenshot_url: string | null; + url: string; + event_type: string; + id: string; + user_id: string; + device_id: string | null; + }; +} +export declare class CreateAgentTaskEvent extends BaseEvent { + agent_session_id: string; + llm_model: string; + stopped: boolean; + paused: boolean; + task: string; + done_output: string | null; + scheduled_task_id: string | null; + started_at: Date; + finished_at: Date | null; + agent_state: Record; + user_feedback_type: string | null; + user_comment: string | null; + gif_url: string | null; + constructor(init: { + user_id?: string; + device_id?: string | null; + agent_session_id: string; + id?: string; + llm_model: string; + task: string; + stopped?: boolean; + paused?: boolean; + done_output?: string | null; + scheduled_task_id?: string | null; + started_at?: Date; + finished_at?: Date | null; + agent_state?: Record; + user_feedback_type?: string | null; + user_comment?: string | null; + gif_url?: string | null; + }); + static fromAgent(agent: AgentWithState): CreateAgentTaskEvent; + toJSON(): { + agent_session_id: string; + llm_model: string; + task: string; + stopped: boolean; + paused: boolean; + done_output: string | null; + scheduled_task_id: string | null; + started_at: string; + finished_at: string | null; + agent_state: Record; + user_feedback_type: string | null; + user_comment: string | null; + gif_url: string | null; + event_type: string; + id: string; + user_id: string; + device_id: string | null; + }; +} +export declare class CreateAgentSessionEvent extends BaseEvent { + browser_session_id: string; + browser_session_live_url: string; + browser_session_cdp_url: string; + browser_session_stopped: boolean; + browser_session_stopped_at: Date | null; + is_source_api: boolean | null; + browser_state: Record; + browser_session_data: Record | null; + constructor(init: { + user_id?: string; + device_id?: string | null; + browser_session_id: string; + id?: string; + browser_state?: Record; + browser_session_live_url?: string; + browser_session_cdp_url?: string; + browser_session_stopped?: boolean; + browser_session_stopped_at?: Date | null; + is_source_api?: boolean | null; + browser_session_data?: Record | null; + }); + static fromAgent(agent: AgentReference): CreateAgentSessionEvent; + toJSON(): { + browser_session_id: string; + browser_session_live_url: string; + browser_session_cdp_url: string; + browser_session_stopped: boolean; + browser_session_stopped_at: string | null; + is_source_api: boolean | null; + browser_state: Record; + browser_session_data: Record | null; + event_type: string; + id: string; + user_id: string; + device_id: string | null; + }; +} +export declare class UpdateAgentSessionEvent extends BaseEvent { + browser_session_stopped: boolean | null; + browser_session_stopped_at: Date | null; + end_reason: string | null; + constructor(init: Partial & { + id: string; + user_id?: string; + }); + toJSON(): { + browser_session_stopped: boolean | null; + browser_session_stopped_at: string | null; + end_reason: string | null; + event_type: string; + id: string; + user_id: string; + device_id: string | null; + }; +} +export {}; diff --git a/dist/agent/cloud-events.js b/dist/agent/cloud-events.js new file mode 100644 index 00000000..2fbbea72 --- /dev/null +++ b/dist/agent/cloud-events.js @@ -0,0 +1,381 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { uuid7str } from '../utils.js'; +import { createLogger } from '../logging-config.js'; +const MAX_STRING_LENGTH = 100_000; +const MAX_URL_LENGTH = 100_000; +const MAX_TASK_LENGTH = 100_000; +const MAX_COMMENT_LENGTH = 2_000; +const MAX_FILE_CONTENT_SIZE = 50 * 1024 * 1024; +const MAX_LLM_MODEL_LENGTH = 200; +const MAX_END_REASON_LENGTH = 100; +const logger = createLogger('browser_use.agent.cloud_events'); +const estimateBase64DecodedBytes = (value) => Math.floor((value.length * 3) / 4); +const extractBase64Payload = (value) => value.includes(',') ? value.split(',').slice(1).join(',') : value; +const getDeviceId = (agent) => agent.cloud_sync?.auth_client?.device_id ?? null; +const getBrowserProfile = (agent) => agent.browser_profile ?? agent.browser_session?.browser_profile ?? null; +const serializeAgentState = (agent) => { + if (typeof agent.state.model_dump === 'function') { + return agent.state.model_dump(); + } + return { + stopped: agent.state.stopped, + paused: agent.state.paused, + n_steps: agent.state.n_steps, + }; +}; +const toDate = (timestamp) => { + if (!timestamp) { + return null; + } + return new Date(timestamp * 1000); +}; +export class BaseEvent { + event_type; + id; + user_id; + device_id; + constructor(event_type, init = {}) { + this.event_type = event_type; + this.id = init.id ?? uuid7str(); + this.user_id = init.user_id ?? ''; + this.device_id = init.device_id ?? null; + } + toJSON() { + return { + event_type: this.event_type, + id: this.id, + user_id: this.user_id, + device_id: this.device_id, + }; + } +} +export class UpdateAgentTaskEvent extends BaseEvent { + stopped; + paused; + done_output; + finished_at; + agent_state; + user_feedback_type; + user_comment; + gif_url; + constructor(init) { + super('UpdateAgentTaskEvent', init); + this.stopped = init.stopped ?? null; + this.paused = init.paused ?? null; + this.done_output = init.done_output ?? null; + this.finished_at = init.finished_at ?? null; + this.agent_state = init.agent_state ?? null; + this.user_feedback_type = init.user_feedback_type ?? null; + this.user_comment = init.user_comment ?? null; + this.gif_url = init.gif_url ?? null; + } + static fromAgent(agent) { + if (agent._task_start_time == null) { + throw new Error('Agent must have _task_start_time attribute'); + } + return new UpdateAgentTaskEvent({ + id: String(agent.task_id), + device_id: getDeviceId(agent), + stopped: agent.state.stopped, + paused: agent.state.paused, + done_output: agent.history.final_result(), + finished_at: agent.history.is_done() ? new Date() : null, + agent_state: serializeAgentState(agent), + user_feedback_type: null, + user_comment: null, + gif_url: null, + }); + } + toJSON() { + return { + ...super.toJSON(), + stopped: this.stopped, + paused: this.paused, + done_output: this.done_output, + finished_at: this.finished_at?.toISOString() ?? null, + agent_state: this.agent_state, + user_feedback_type: this.user_feedback_type, + user_comment: this.user_comment, + gif_url: this.gif_url, + }; + } +} +export class CreateAgentOutputFileEvent extends BaseEvent { + task_id; + file_name; + file_content; + content_type; + created_at; + constructor(init) { + super('CreateAgentOutputFileEvent', init); + this.task_id = init.task_id; + this.file_name = init.file_name; + if (init.file_content != null) { + const payload = extractBase64Payload(init.file_content); + const estimatedSize = estimateBase64DecodedBytes(payload); + if (estimatedSize > MAX_FILE_CONTENT_SIZE) { + throw new Error(`file_content exceeds maximum size of ${MAX_FILE_CONTENT_SIZE} bytes`); + } + this.file_content = init.file_content; + } + else { + this.file_content = null; + } + this.content_type = init.content_type ?? null; + this.created_at = init.created_at ?? new Date(); + } + static async fromAgentAndFile(agent, outputPath) { + const resolved = path.resolve(outputPath); + await fs.promises.access(resolved, fs.constants.F_OK); + const stats = await fs.promises.stat(resolved); + let fileContent = null; + if (stats.size < MAX_FILE_CONTENT_SIZE) { + const data = await fs.promises.readFile(resolved); + fileContent = data.toString('base64'); + } + return new CreateAgentOutputFileEvent({ + task_id: String(agent.task_id), + device_id: getDeviceId(agent), + file_name: path.basename(resolved), + file_content: fileContent, + content_type: 'image/gif', + }); + } + toJSON() { + return { + ...super.toJSON(), + task_id: this.task_id, + file_name: this.file_name, + file_content: this.file_content, + content_type: this.content_type, + created_at: this.created_at.toISOString(), + }; + } +} +export class CreateAgentStepEvent extends BaseEvent { + created_at; + agent_task_id; + step; + evaluation_previous_goal; + memory; + next_goal; + actions; + screenshot_url; + url; + constructor(init) { + super('CreateAgentStepEvent', init); + this.created_at = init.created_at ?? new Date(); + this.agent_task_id = init.agent_task_id; + this.step = init.step; + this.evaluation_previous_goal = init.evaluation_previous_goal; + this.memory = init.memory; + this.next_goal = init.next_goal; + this.actions = init.actions; + if (init.screenshot_url?.startsWith('data:')) { + const payload = extractBase64Payload(init.screenshot_url); + const estimatedSize = estimateBase64DecodedBytes(payload); + if (estimatedSize > MAX_FILE_CONTENT_SIZE) { + throw new Error(`screenshot_url exceeds maximum size of ${MAX_FILE_CONTENT_SIZE} bytes`); + } + } + this.screenshot_url = init.screenshot_url ?? null; + this.url = init.url; + } + static fromAgentStep(agent, model_output, result, actions_data, browser_state_summary) { + const currentState = model_output.current_state; + const screenshot = browser_state_summary.screenshot + ? `data:image/png;base64,${browser_state_summary.screenshot}` + : null; + if (browser_state_summary.screenshot) { + logger.debug(`Including screenshot in CreateAgentStepEvent, length: ${browser_state_summary.screenshot.length}`); + } + else { + logger.debug('No screenshot in browser_state_summary for CreateAgentStepEvent'); + } + return new CreateAgentStepEvent({ + device_id: getDeviceId(agent), + agent_task_id: String(agent.task_id), + step: agent.state.n_steps, + evaluation_previous_goal: currentState?.evaluation_previous_goal ?? '', + memory: currentState?.memory ?? '', + next_goal: currentState?.next_goal ?? '', + actions: actions_data ?? [], + url: browser_state_summary.url ?? '', + screenshot_url: screenshot, + }); + } + toJSON() { + return { + ...super.toJSON(), + created_at: this.created_at.toISOString(), + agent_task_id: this.agent_task_id, + step: this.step, + evaluation_previous_goal: this.evaluation_previous_goal, + memory: this.memory, + next_goal: this.next_goal, + actions: this.actions, + screenshot_url: this.screenshot_url, + url: this.url, + }; + } +} +export class CreateAgentTaskEvent extends BaseEvent { + agent_session_id; + llm_model; + stopped; + paused; + task; + done_output; + scheduled_task_id; + started_at; + finished_at; + agent_state; + user_feedback_type; + user_comment; + gif_url; + constructor(init) { + super('CreateAgentTaskEvent', init); + this.agent_session_id = init.agent_session_id; + if (init.llm_model.length > MAX_LLM_MODEL_LENGTH) { + throw new Error(`llm_model exceeds maximum length of ${MAX_LLM_MODEL_LENGTH}`); + } + if (init.task.length > MAX_TASK_LENGTH) { + throw new Error(`task exceeds maximum length of ${MAX_TASK_LENGTH}`); + } + this.llm_model = init.llm_model; + this.task = init.task; + this.stopped = init.stopped ?? false; + this.paused = init.paused ?? false; + this.done_output = init.done_output ?? null; + this.scheduled_task_id = init.scheduled_task_id ?? null; + this.started_at = init.started_at ?? new Date(); + this.finished_at = init.finished_at ?? null; + this.agent_state = init.agent_state ?? {}; + this.user_feedback_type = init.user_feedback_type ?? null; + this.user_comment = init.user_comment ?? null; + this.gif_url = init.gif_url ?? null; + } + static fromAgent(agent) { + const startedAt = toDate(agent._task_start_time) ?? new Date(); + return new CreateAgentTaskEvent({ + id: String(agent.task_id), + device_id: getDeviceId(agent), + agent_session_id: String(agent.session_id), + task: agent.task, + llm_model: agent.llm.model_name || agent.llm.model || 'unknown', + agent_state: serializeAgentState(agent), + stopped: false, + paused: false, + started_at: startedAt, + finished_at: null, + done_output: null, + scheduled_task_id: null, + user_feedback_type: null, + user_comment: null, + gif_url: null, + }); + } + toJSON() { + return { + ...super.toJSON(), + agent_session_id: this.agent_session_id, + llm_model: this.llm_model, + task: this.task, + stopped: this.stopped, + paused: this.paused, + done_output: this.done_output, + scheduled_task_id: this.scheduled_task_id, + started_at: this.started_at.toISOString(), + finished_at: this.finished_at?.toISOString() ?? null, + agent_state: this.agent_state, + user_feedback_type: this.user_feedback_type, + user_comment: this.user_comment, + gif_url: this.gif_url, + }; + } +} +export class CreateAgentSessionEvent extends BaseEvent { + browser_session_id; + browser_session_live_url; + browser_session_cdp_url; + browser_session_stopped; + browser_session_stopped_at; + is_source_api; + browser_state; + browser_session_data; + constructor(init) { + super('CreateAgentSessionEvent', init); + this.browser_session_id = init.browser_session_id; + this.browser_session_live_url = init.browser_session_live_url ?? ''; + this.browser_session_cdp_url = init.browser_session_cdp_url ?? ''; + this.browser_session_stopped = init.browser_session_stopped ?? false; + this.browser_session_stopped_at = init.browser_session_stopped_at ?? null; + this.is_source_api = init.is_source_api ?? null; + this.browser_state = init.browser_state ?? {}; + this.browser_session_data = init.browser_session_data ?? null; + } + static fromAgent(agent) { + const profile = getBrowserProfile(agent); + return new CreateAgentSessionEvent({ + id: String(agent.session_id), + device_id: getDeviceId(agent), + browser_session_id: agent.browser_session.id, + browser_state: { + viewport: profile?.viewport ?? { width: 1280, height: 720 }, + user_agent: profile?.user_agent ?? null, + headless: profile?.headless ?? true, + initial_url: null, + final_url: null, + total_pages_visited: 0, + session_duration_seconds: 0, + }, + browser_session_data: { + cookies: [], + secrets: {}, + allowed_domains: profile?.allowed_domains ?? [], + }, + }); + } + toJSON() { + return { + ...super.toJSON(), + browser_session_id: this.browser_session_id, + browser_session_live_url: this.browser_session_live_url, + browser_session_cdp_url: this.browser_session_cdp_url, + browser_session_stopped: this.browser_session_stopped, + browser_session_stopped_at: this.browser_session_stopped_at?.toISOString() ?? null, + is_source_api: this.is_source_api, + browser_state: this.browser_state, + browser_session_data: this.browser_session_data, + }; + } +} +export class UpdateAgentSessionEvent extends BaseEvent { + browser_session_stopped; + browser_session_stopped_at; + end_reason; + constructor(init) { + super('UpdateAgentSessionEvent', init); + this.browser_session_stopped = init.browser_session_stopped ?? null; + this.browser_session_stopped_at = init.browser_session_stopped_at ?? null; + if (init.end_reason != null) { + const endReason = String(init.end_reason); + if (endReason.length > MAX_END_REASON_LENGTH) { + throw new Error(`end_reason exceeds maximum length of ${MAX_END_REASON_LENGTH}`); + } + this.end_reason = endReason; + } + else { + this.end_reason = null; + } + } + toJSON() { + return { + ...super.toJSON(), + browser_session_stopped: this.browser_session_stopped, + browser_session_stopped_at: this.browser_session_stopped_at?.toISOString() ?? null, + end_reason: this.end_reason, + }; + } +} diff --git a/dist/agent/gif.d.ts b/dist/agent/gif.d.ts new file mode 100644 index 00000000..ba14c9cf --- /dev/null +++ b/dist/agent/gif.d.ts @@ -0,0 +1,16 @@ +import type { AgentHistoryList } from './views.js'; +export declare const decode_unicode_escapes_to_utf8: (text: string) => string; +export declare const is_valid_gif_screenshot_candidate: (screenshot: string | null | undefined, pageUrl: string | null | undefined) => boolean; +export interface HistoryGifOptions { + output_path?: string; + duration?: number; + show_goals?: boolean; + show_task?: boolean; + show_logo?: boolean; + font_size?: number; + title_font_size?: number; + goal_font_size?: number; + margin?: number; + line_spacing?: number; +} +export declare const create_history_gif: (task: string, history: AgentHistoryList, { output_path, duration, show_goals, show_task, show_logo, font_size, title_font_size, goal_font_size, line_spacing, }?: HistoryGifOptions) => Promise; diff --git a/dist/agent/gif.js b/dist/agent/gif.js new file mode 100644 index 00000000..c670614e --- /dev/null +++ b/dist/agent/gif.js @@ -0,0 +1,237 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { createCanvas, loadImage } from 'canvas'; +import GIFEncoder from 'gif-encoder-2'; +import { createLogger } from '../logging-config.js'; +import { PLACEHOLDER_4PX_SCREENSHOT } from '../browser/views.js'; +import { is_new_tab_page } from '../utils.js'; +const logger = createLogger('browser_use.agent.gif'); +export const decode_unicode_escapes_to_utf8 = (text) => { + if (!text.includes('\\u')) { + return text; + } + try { + return Buffer.from(text, 'latin1').toString('utf8'); + } + catch { + return text; + } +}; +const asDataUrl = (screenshot) => { + if (!screenshot) { + return ''; + } + return screenshot.startsWith('data:') + ? screenshot + : `data:image/png;base64,${screenshot}`; +}; +const loadScreenshot = async (screenshot) => { + const normalized = asDataUrl(screenshot); + return loadImage(normalized); +}; +const FONT_CANDIDATES = [ + '"PingFang"', + '"STHeiti Medium"', + '"Microsoft YaHei"', + '"SimHei"', + '"SimSun"', + '"Noto Sans CJK SC"', + '"Arial"', + '"Helvetica"', + '"sans-serif"', +]; +const pickFont = () => FONT_CANDIDATES.join(', '); +export const is_valid_gif_screenshot_candidate = (screenshot, pageUrl) => { + if (!screenshot || screenshot === PLACEHOLDER_4PX_SCREENSHOT) { + return false; + } + return !is_new_tab_page(pageUrl ?? ''); +}; +const wrapText = (ctx, text, maxWidth) => { + const words = decode_unicode_escapes_to_utf8(text).split(/\s+/); + const lines = []; + let currentLine = ''; + for (const word of words) { + const testLine = currentLine ? `${currentLine} ${word}` : word; + const metrics = ctx.measureText(testLine); + if (metrics.width > maxWidth && currentLine) { + lines.push(currentLine); + currentLine = word; + } + else { + currentLine = testLine; + } + } + if (currentLine) { + lines.push(currentLine); + } + return lines; +}; +const drawRoundedRect = (ctx, x, y, width, height, radius, fillStyle) => { + ctx.save(); + ctx.fillStyle = fillStyle; + ctx.beginPath(); + ctx.moveTo(x + radius, y); + ctx.lineTo(x + width - radius, y); + ctx.quadraticCurveTo(x + width, y, x + width, y + radius); + ctx.lineTo(x + width, y + height - radius); + ctx.quadraticCurveTo(x + width, y + height, x + width - radius, y + height); + ctx.lineTo(x + radius, y + height); + ctx.quadraticCurveTo(x, y + height, x, y + height - radius); + ctx.lineTo(x, y + radius); + ctx.quadraticCurveTo(x, y, x + radius, y); + ctx.closePath(); + ctx.fill(); + ctx.restore(); +}; +const addOverlayToContext = (ctx, width, height, stepNumber, goalText, fontFamily, titleFontSize) => { + const margin = 40; + const textColor = 'rgba(255,255,255,1)'; + const boxColor = 'rgba(0,0,0,0.7)'; + ctx.save(); + ctx.fillStyle = textColor; + ctx.font = `${titleFontSize}px ${fontFamily}`; + ctx.textBaseline = 'top'; + const stepText = String(stepNumber); + const stepMetrics = ctx.measureText(stepText); + const stepWidth = stepMetrics.width; + const stepHeight = titleFontSize; + const stepX = margin; + const stepY = height - stepHeight - margin - 10; + drawRoundedRect(ctx, stepX - 20, stepY - 20, stepWidth + 40, stepHeight + 40, 15, boxColor); + ctx.fillText(stepText, stepX, stepY); + const maxWidth = width - margin * 4; + const lines = wrapText(ctx, goalText, maxWidth); + const totalHeight = lines.length * (titleFontSize + 10); + const goalX = (width - maxWidth) / 2; + const goalY = stepY - totalHeight - 80; + drawRoundedRect(ctx, goalX - 20, goalY - 20, maxWidth + 40, totalHeight + 40, 15, boxColor); + lines.forEach((line, idx) => { + ctx.fillText(line, goalX, goalY + idx * (titleFontSize + 10)); + }); + ctx.restore(); +}; +const addLogo = (ctx, width, image) => { + if (!image) + return; + ctx.save(); + const margin = 20; + const targetHeight = 150; + const aspect = image.width / image.height || 1; + const targetWidth = targetHeight * aspect; + ctx.globalAlpha = 0.9; + // @ts-ignore - drawImage signature compatibility + ctx.drawImage(image, width - targetWidth - margin, margin, targetWidth, targetHeight); + ctx.restore(); +}; +const loadLogo = async () => { + try { + const logoPath = path.resolve('static/browser-use.png'); + await fs.promises.access(logoPath, fs.constants.F_OK); + return await loadImage(logoPath); + } + catch { + return null; + } +}; +const renderTaskFrame = (ctx, width, height, task, fontFamily, fontSize, lineSpacing, logo) => { + ctx.save(); + ctx.fillStyle = '#000'; + ctx.fillRect(0, 0, width, height); + ctx.fillStyle = '#fff'; + ctx.font = `${fontSize}px ${fontFamily}`; + ctx.textAlign = 'center'; + ctx.textBaseline = 'top'; + const maxWidth = width - 280; + const lines = wrapText(ctx, task, maxWidth); + const totalHeight = lines.length * fontSize * lineSpacing; + let y = height / 2 - totalHeight / 2; + lines.forEach((line) => { + ctx.fillText(line, width / 2, y); + y += fontSize * lineSpacing; + }); + ctx.restore(); + addLogo(ctx, width, logo); +}; +const drawScreenshotFrame = (ctx, width, height, image, stepNumber, goalText, fontFamily, titleFontSize, logo) => { + // @ts-ignore - drawImage signature compatibility + ctx.drawImage(image, 0, 0, width, height); + if (goalText) { + addOverlayToContext(ctx, width, height, stepNumber, goalText, fontFamily, titleFontSize); + } + addLogo(ctx, width, logo); +}; +export const create_history_gif = async (task, history, { output_path = 'agent_history.gif', duration = 3000, show_goals = true, show_task = true, show_logo = false, font_size = 40, title_font_size = 56, goal_font_size = 44, line_spacing = 1.5, } = {}) => { + if (!history.history.length) { + logger.warn('No history to create GIF from'); + return; + } + const screenshots = history.screenshots(); + let firstRealScreenshot = null; + for (let index = 0; index < screenshots.length; index += 1) { + const screenshot = screenshots[index]; + const pageUrl = history.history[index]?.state?.url ?? null; + if (is_valid_gif_screenshot_candidate(screenshot, pageUrl)) { + firstRealScreenshot = screenshot; + break; + } + } + if (!firstRealScreenshot) { + logger.warn('No valid screenshots found (all are placeholders or from new tab pages)'); + return; + } + const firstImage = await loadScreenshot(firstRealScreenshot); + const width = firstImage.width; + const height = firstImage.height; + const canvas = createCanvas(width, height); + const ctx = canvas.getContext('2d'); + const encoder = new GIFEncoder(width, height); + await fs.promises.mkdir(path.dirname(path.resolve(output_path)), { + recursive: true, + }); + const writeStream = fs.createWriteStream(path.resolve(output_path)); + encoder.createReadStream().pipe(writeStream); + encoder.start(); + encoder.setRepeat(0); + encoder.setDelay(duration); + encoder.setQuality(10); + const fontFamily = pickFont(); + const logo = show_logo ? await loadLogo() : null; + if (show_task && task) { + ctx.clearRect(0, 0, width, height); + // @ts-ignore - node-canvas types differ slightly from browser canvas + renderTaskFrame(ctx, width, height, task, fontFamily, font_size + 16, line_spacing, logo); + encoder.addFrame(ctx); + } + for (let index = 0; index < screenshots.length; index += 1) { + const historyItem = history.history[index]; + if (is_new_tab_page(historyItem?.state?.url ?? '')) { + logger.debug(`Skipping screenshot from new tab page (${historyItem?.state?.url ?? ''}) at step ${index}`); + continue; + } + const screenshot = screenshots[index]; + if (!screenshot || screenshot === PLACEHOLDER_4PX_SCREENSHOT) { + continue; + } + try { + const image = await loadScreenshot(screenshot); + ctx.clearRect(0, 0, width, height); + const goalText = show_goals && + history.history[index]?.model_output?.current_state.next_goal + ? history.history[index].model_output.current_state.next_goal + : ''; + // @ts-ignore - node-canvas types differ slightly from browser canvas + drawScreenshotFrame(ctx, width, height, image, index + 1, goalText, fontFamily, goal_font_size, logo); + encoder.addFrame(ctx); + } + catch (error) { + logger.warn(`Failed to process screenshot at step ${index + 1}: ${error.message}`); + } + } + encoder.finish(); + await new Promise((resolve, reject) => { + writeStream.on('finish', resolve); + writeStream.on('error', reject); + }); + logger.info(`Created GIF at ${output_path}`); +}; diff --git a/dist/agent/index.d.ts b/dist/agent/index.d.ts new file mode 100644 index 00000000..e3f1009e --- /dev/null +++ b/dist/agent/index.d.ts @@ -0,0 +1,8 @@ +export * from './views.js'; +export * from './message-manager/service.js'; +export * from './message-manager/views.js'; +export * from './message-manager/utils.js'; +export * from './prompts.js'; +export * from './cloud-events.js'; +export * from './gif.js'; +export * from './service.js'; diff --git a/dist/agent/index.js b/dist/agent/index.js new file mode 100644 index 00000000..e3f1009e --- /dev/null +++ b/dist/agent/index.js @@ -0,0 +1,8 @@ +export * from './views.js'; +export * from './message-manager/service.js'; +export * from './message-manager/views.js'; +export * from './message-manager/utils.js'; +export * from './prompts.js'; +export * from './cloud-events.js'; +export * from './gif.js'; +export * from './service.js'; diff --git a/dist/agent/judge.d.ts b/dist/agent/judge.d.ts new file mode 100644 index 00000000..9d5cf890 --- /dev/null +++ b/dist/agent/judge.d.ts @@ -0,0 +1,17 @@ +import { type Message } from '../llm/messages.js'; +export interface ConstructJudgeMessagesOptions { + task: string; + final_result: string; + agent_steps: string[]; + screenshot_paths: string[]; + max_images?: number; + ground_truth?: string | null; + use_vision?: boolean | 'auto'; +} +export interface ConstructSimpleJudgeMessagesOptions { + task: string; + final_result: string; + current_date?: string; +} +export declare const construct_judge_messages: (options: ConstructJudgeMessagesOptions) => Message[]; +export declare const construct_simple_judge_messages: (options: ConstructSimpleJudgeMessagesOptions) => Message[]; diff --git a/dist/agent/judge.js b/dist/agent/judge.js new file mode 100644 index 00000000..1e988127 --- /dev/null +++ b/dist/agent/judge.js @@ -0,0 +1,197 @@ +import fs from 'node:fs'; +import { ContentPartImageParam, ContentPartTextParam, ImageURL, SystemMessage, UserMessage, } from '../llm/messages.js'; +const truncateText = (text, maxLength, fromBeginning = false) => { + if (text.length <= maxLength) { + return text; + } + if (fromBeginning) { + return `...[text truncated]${text.slice(-(maxLength - 20))}`; + } + return `${text.slice(0, maxLength - 23)}...[text truncated]...`; +}; +const encodeImage = (imagePath) => { + try { + if (!fs.existsSync(imagePath)) { + return null; + } + return fs.readFileSync(imagePath).toString('base64'); + } + catch { + return null; + } +}; +export const construct_judge_messages = (options) => { + const { task, final_result, agent_steps, screenshot_paths, max_images = 10, ground_truth = null, use_vision = true, } = options; + const task_truncated = truncateText(task, 40000); + const final_result_truncated = truncateText(final_result, 40000); + const steps_text_truncated = truncateText(agent_steps.join('\n'), 40000); + const encoded_images = []; + if (use_vision !== false) { + const selected = screenshot_paths.length > max_images + ? screenshot_paths.slice(-max_images) + : screenshot_paths; + for (const screenshotPath of selected) { + const encoded = encodeImage(screenshotPath); + if (!encoded) { + continue; + } + encoded_images.push(new ContentPartImageParam(new ImageURL(`data:image/png;base64,${encoded}`, 'auto', 'image/png'))); + } + } + const ground_truth_section = ground_truth + ? ` +**GROUND TRUTH VALIDATION (HIGHEST PRIORITY):** +The section contains verified correct information for this task. This can be: +- Evaluation criteria: Specific conditions that must be met (e.g., "The success popup should show up", "Must extract exactly 5 items") +- Factual answers: The correct answer to a question or information retrieval task (e.g. "10/11/24", "Paris") +- Expected outcomes: What should happen after task completion (e.g., "Google Doc must be created", "File should be downloaded") + +The ground truth takes ABSOLUTE precedence over all other evaluation criteria. If the ground truth is not satisfied by the agent's execution and final response, the verdict MUST be false. +` + : ''; + const system_prompt = `You are an expert judge evaluating browser automation agent performance. + + +${ground_truth_section} +**PRIMARY EVALUATION CRITERIA (in order of importance):** +1. **Task Satisfaction (Most Important)**: Did the agent accomplish what the user asked for? Break down the task into the key criteria and evaluate if the agent all of them. Focus on user intent and final outcome. +2. **Output Quality**: Is the final result in the correct format and complete? Does it match exactly what was requested? +3. **Tool Effectiveness**: Did the browser interactions work as expected? Were tools used appropriately? How many % of the tools failed? +4. **Agent Reasoning**: Quality of decision-making, planning, and problem-solving throughout the trajectory. +5. **Browser Handling**: Navigation stability, error recovery, and technical execution. If the browser crashes, does not load or a captcha blocks the task, the score must be very low. + +**VERDICT GUIDELINES:** +- true: Task completed as requested, human-like execution, all of the users criteria were met and the agent did not make up any information. +- false: Task not completed, or only partially completed. + +**Examples of task completion verdict:** +- If task asks for 10 items and agent finds 4 items correctly: false +- If task completed to full user requirements but with some errors to improve in the trajectory: true +- If task impossible due to captcha/login requirements: false +- If the trajectory is ideal and the output is perfect: true +- If the task asks to search all headphones in amazon under $100 but the agent searches all headphones and the lowest price is $150: false +- If the task asks to research a property and create a google doc with the result but the agents only returns the results in text: false +- If the task asks to complete an action on the page, and the agent reports that the action is completed but the screenshot or page shows the action is not actually complete: false +- If the task asks to use a certain tool or site to complete the task but the agent completes the task without using it: false +- If the task asks to look for a section of a page that does not exist: false +- If the agent concludes the task is impossible but it is not: false +- If the agent concludes the task is impossible and it truly is impossible: false +- If the agent is unable to complete the task because no login information was provided and it is truly needed to complete the task: false + +**FAILURE CONDITIONS (automatically set verdict to false):** +- Blocked by captcha or missing authentication +- Output format completely wrong or missing +- Infinite loops or severe technical failures +- Critical user requirements ignored +- Page not loaded +- Browser crashed +- Agent could not interact with required UI elements +- The agent moved on from a important step in the task without completing it +- The agent made up content that is not in the screenshot or the page state +- The agent calls done action before completing all key points of the task + +**IMPOSSIBLE TASK DETECTION:** +Set impossible_task to true when the task fundamentally could not be completed due to: +- Vague or ambiguous task instructions that cannot be reasonably interpreted +- Website genuinely broken or non-functional (be conservative - temporary issues don't count) +- Required links/pages truly inaccessible (404, 403, etc.) +- Task requires authentication/login but no credentials were provided +- Task asks for functionality that doesn't exist on the target site +- Other insurmountable external obstacles beyond the agent's control + +Do NOT mark as impossible if: +- Agent made poor decisions but task was achievable +- Temporary page loading issues that could be retried +- Agent didn't try the right approach +- Website works but agent struggled with it + +**CAPTCHA DETECTION:** +Set reached_captcha to true if: +- Screenshots show captcha challenges (reCAPTCHA, hCaptcha, etc.) +- Agent reports being blocked by bot detection +- Error messages indicate captcha/verification requirements +- Any evidence the agent encountered anti-bot measures during execution + +**IMPORTANT EVALUATION NOTES:** +- **evaluate for action** - For each key step of the trace, double check whether the action that the agent tried to performed actually happened. If the required action did not actually occur, the verdict should be false. +- **screenshot is not entire content** - The agent has the entire DOM content, but the screenshot is only part of the content. If the agent extracts information from the page, but you do not see it in the screenshot, you can assume this information is there. +- **Penalize poor tool usage** - Wrong tools, inefficient approaches, ignoring available information. +- **ignore unexpected dates and times** - These agent traces are from varying dates, you can assume the dates the agent uses for search or filtering are correct. +- **IMPORTANT**: be very picky about the user's request - Have very high standard for the agent completing the task exactly to the user's request. +- **IMPORTANT**: be initially doubtful of the agent's self reported success, be sure to verify that its methods are valid and fulfill the user's desires to a tee. + + + +Respond with EXACTLY this JSON structure (no additional text before or after): + +{ + "reasoning": "Breakdown of user task into key points. Detailed analysis covering: what went well, what didn't work, trajectory quality assessment, tool usage evaluation, output quality review, and overall user satisfaction prediction.", + "verdict": true or false, + "failure_reason": "Max 5 sentences explanation of why the task was not completed successfully in case of failure. If verdict is true, use an empty string.", + "impossible_task": true or false, + "reached_captcha": true or false +} +`; + const ground_truth_prompt = ground_truth + ? ` + +${ground_truth} + +` + : ''; + const user_prompt = ` +${task_truncated || 'No task provided'} + +${ground_truth_prompt} + +${steps_text_truncated || 'No agent trajectory provided'} + + + +${final_result_truncated || 'No final result provided'} + + +${encoded_images.length} screenshots from execution are attached. + +Evaluate this agent execution given the criteria and respond with the exact JSON structure requested.`; + const content_parts = [ + new ContentPartTextParam(user_prompt), + ]; + content_parts.push(...encoded_images); + return [new SystemMessage(system_prompt), new UserMessage(content_parts)]; +}; +export const construct_simple_judge_messages = (options) => { + const task_truncated = truncateText(options.task, 20000); + const final_result_truncated = truncateText(options.final_result, 20000); + const current_date = options.current_date ?? new Date().toISOString().slice(0, 10); + const system_prompt = `You are a strict verifier checking whether a browser automation agent actually completed its task. + +Today's date is ${current_date}. The agent ran recently - dates near today are expected and NOT fabricated. + +Given the task and the agent's final response, determine if the response genuinely satisfies ALL requirements. + +Check for these common failure patterns: +1. **Incorrect data**: Wrong number of items, missing filters/criteria, wrong format +2. **Unverified actions**: Agent claims to have submitted a form, posted a comment, or saved a file but there's no evidence +3. **Incomplete results**: Some requirements from the task are not addressed in the response +4. **Fabricated content**: Data that looks plausible but wasn't actually extracted from any page. NOTE: dates and times close to today's date (${current_date}) are NOT fabricated - the agent browses live websites and extracts real-time content. +5. **Partial completion reported as success**: Response acknowledges failure or blockers (captcha, access denied, etc.) but still claims success + +Respond with EXACTLY this JSON structure: +{ + "is_correct": true or false, + "reason": "Brief explanation if not correct, empty string if correct" +} + +Be strict: if the response doesn't clearly satisfy every requirement, set is_correct to false.`; + const user_prompt = ` +${task_truncated || 'No task provided'} + + + +${final_result_truncated || 'No response provided'} + + +Does the agent's response fully satisfy all requirements of the task? Respond with the JSON structure.`; + return [new SystemMessage(system_prompt), new UserMessage(user_prompt)]; +}; diff --git a/dist/agent/message-manager/service.d.ts b/dist/agent/message-manager/service.d.ts new file mode 100644 index 00000000..cf8659b3 --- /dev/null +++ b/dist/agent/message-manager/service.d.ts @@ -0,0 +1,38 @@ +import { ContentPartImageParam, ContentPartTextParam, SystemMessage, UserMessage, type Message } from '../../llm/messages.js'; +import type { BaseChatModel } from '../../llm/base.js'; +import { ActionResult, AgentOutput, AgentStepInfo, MessageCompactionSettings } from '../views.js'; +import { BrowserStateSummary } from '../../browser/views.js'; +import { FileSystem } from '../../filesystem/file-system.js'; +import { MessageManagerState } from './views.js'; +export declare class MessageManager { + private readonly fileSystem; + private readonly state; + private readonly useThinking; + private readonly sensitiveData?; + private readonly maxHistoryItems; + private readonly visionDetailLevel; + private readonly includeToolCallExamples; + private readonly includeRecentEvents; + private readonly sampleImages; + private readonly llmScreenshotSize; + private task; + private systemPrompt; + private sensitiveDataDescription; + private lastInputMessages; + private includeAttributes; + last_state_message_text: string | null; + constructor(task: string, systemMessage: SystemMessage, fileSystem: FileSystem, state?: MessageManagerState, useThinking?: boolean, includeAttributes?: string[] | null, sensitiveData?: Record> | undefined, maxHistoryItems?: number | null, visionDetailLevel?: 'auto' | 'low' | 'high', includeToolCallExamples?: boolean, includeRecentEvents?: boolean, sampleImages?: Array | null, llmScreenshotSize?: [number, number] | null); + get agent_history_description(): string; + add_new_task(new_task: string): void; + private updateAgentHistoryDescription; + private getSensitiveDataDescription; + prepare_step_state(browser_state_summary: BrowserStateSummary, model_output?: AgentOutput | null, result?: ActionResult[] | null, step_info?: AgentStepInfo | null, sensitive_data?: Record> | null): void; + maybe_compact_messages(llm: BaseChatModel | null, settings: MessageCompactionSettings | null, step_info?: AgentStepInfo | null): Promise; + create_state_messages(browser_state_summary: BrowserStateSummary, model_output?: AgentOutput | null, result?: ActionResult[] | null, step_info?: AgentStepInfo | null, use_vision?: boolean | 'auto', page_filtered_actions?: string | null, sensitive_data?: Record> | null, available_file_paths?: string[] | null, include_recent_events?: boolean | null, plan_description?: string | null, unavailable_skills_info?: string | null, skip_state_update?: boolean): void; + get_messages(): Message[]; + private setMessageWithType; + private addContextMessage; + _add_context_message(message: SystemMessage | UserMessage): void; + private extractStateMessageText; + private filterSensitiveData; +} diff --git a/dist/agent/message-manager/service.js b/dist/agent/message-manager/service.js new file mode 100644 index 00000000..1f8ef9b9 --- /dev/null +++ b/dist/agent/message-manager/service.js @@ -0,0 +1,374 @@ +import { ContentPartTextParam, SystemMessage, UserMessage, } from '../../llm/messages.js'; +import { AgentMessagePrompt } from '../prompts.js'; +import { MessageManagerState, HistoryItem } from './views.js'; +import { match_url_with_domain_pattern } from '../../utils.js'; +import { createLogger } from '../../logging-config.js'; +const logger = createLogger('browser_use.agent.message_manager'); +export class MessageManager { + fileSystem; + state; + useThinking; + sensitiveData; + maxHistoryItems; + visionDetailLevel; + includeToolCallExamples; + includeRecentEvents; + sampleImages; + llmScreenshotSize; + task; + systemPrompt; + sensitiveDataDescription = ''; + lastInputMessages = []; + includeAttributes; + last_state_message_text = null; + constructor(task, systemMessage, fileSystem, state = new MessageManagerState(), useThinking = true, includeAttributes = null, sensitiveData, maxHistoryItems = null, visionDetailLevel = 'auto', includeToolCallExamples = false, includeRecentEvents = false, sampleImages = null, llmScreenshotSize = null) { + this.fileSystem = fileSystem; + this.state = state; + this.useThinking = useThinking; + this.sensitiveData = sensitiveData; + this.maxHistoryItems = maxHistoryItems; + this.visionDetailLevel = visionDetailLevel; + this.includeToolCallExamples = includeToolCallExamples; + this.includeRecentEvents = includeRecentEvents; + this.sampleImages = sampleImages; + this.llmScreenshotSize = llmScreenshotSize; + if (this.maxHistoryItems != null && this.maxHistoryItems <= 5) { + throw new Error('max_history_items must be null or greater than 5'); + } + this.task = task; + this.systemPrompt = systemMessage; + this.includeAttributes = includeAttributes ?? []; + if (!this.state.history.system_message) { + this.setMessageWithType(this.systemPrompt, 'system'); + } + } + get agent_history_description() { + const compactedPrefix = this.state.compacted_memory + ? `\n${this.state.compacted_memory}\n\n` + : ''; + if (this.maxHistoryItems == null) { + return (compactedPrefix + + this.state.agent_history_items + .map((item) => item.to_string()) + .join('\n')); + } + const totalItems = this.state.agent_history_items.length; + if (totalItems <= this.maxHistoryItems) { + return (compactedPrefix + + this.state.agent_history_items + .map((item) => item.to_string()) + .join('\n')); + } + const omitted = totalItems - this.maxHistoryItems; + const keepRecent = this.maxHistoryItems - 1; + const parts = []; + parts.push(this.state.agent_history_items[0].to_string()); + parts.push(`[... ${omitted} previous steps omitted...]`); + parts.push(...this.state.agent_history_items + .slice(-keepRecent) + .map((item) => item.to_string())); + return compactedPrefix + parts.join('\n'); + } + add_new_task(new_task) { + const normalizedTask = ` ${new_task.trim()} `; + if (!this.task.includes('')) { + this.task = `${this.task}`; + } + this.task += `\n${normalizedTask}`; + this.state.agent_history_items.push(new HistoryItem(null, null, null, null, null, null, normalizedTask)); + } + updateAgentHistoryDescription(model_output, result, step_info) { + const results = result ?? []; + const stepNumber = step_info?.step_number ?? null; + this.state.read_state_description = ''; + this.state.read_state_images = []; + let actionResults = ''; + let readStateIndex = 0; + results.forEach((action) => { + if (action.include_extracted_content_only_once && + action.extracted_content) { + this.state.read_state_description += `\n${action.extracted_content}\n\n`; + readStateIndex += 1; + } + if (Array.isArray(action.images) && action.images.length > 0) { + this.state.read_state_images.push(...action.images); + } + if (action.long_term_memory) { + actionResults += `${action.long_term_memory}\n`; + } + else if (action.extracted_content && + !action.include_extracted_content_only_once) { + actionResults += `${action.extracted_content}\n`; + } + if (action.error) { + const err = action.error.length > 200 + ? `${action.error.slice(0, 100)}......${action.error.slice(-100)}` + : action.error; + actionResults += `${err}\n`; + } + }); + const MAX_CONTENT_SIZE = 60000; + if (this.state.read_state_description.length > MAX_CONTENT_SIZE) { + this.state.read_state_description = `${this.state.read_state_description.slice(0, MAX_CONTENT_SIZE)}\n... [Content truncated at 60k characters]`; + } + this.state.read_state_description = + this.state.read_state_description.trim(); + let normalizedActionResults = actionResults + ? `Result\n${actionResults}`.trim() + : null; + if (normalizedActionResults) { + if (normalizedActionResults.length > MAX_CONTENT_SIZE) { + normalizedActionResults = `${normalizedActionResults.slice(0, MAX_CONTENT_SIZE)}\n... [Content truncated at 60k characters]`; + } + } + if (!model_output) { + if (stepNumber != null) { + if (stepNumber === 0 && normalizedActionResults) { + this.state.agent_history_items.push(new HistoryItem(stepNumber, null, null, null, normalizedActionResults, null, null)); + } + else if (stepNumber > 0) { + this.state.agent_history_items.push(new HistoryItem(stepNumber, null, null, null, null, 'Agent failed to output in the right format.', null)); + } + } + return; + } + const brain = model_output.current_state; + this.state.agent_history_items.push(new HistoryItem(stepNumber, brain.evaluation_previous_goal, brain.memory, brain.next_goal, normalizedActionResults, null, null)); + } + getSensitiveDataDescription(currentUrl, sensitiveData = this.sensitiveData) { + const placeholders = new Set(); + if (!sensitiveData) { + return ''; + } + for (const [key, value] of Object.entries(sensitiveData)) { + if (value && typeof value === 'object' && !Array.isArray(value)) { + if (currentUrl && + match_url_with_domain_pattern(currentUrl, key, true)) { + Object.keys(value).forEach((entry) => placeholders.add(entry)); + } + } + else if (typeof value === 'string') { + placeholders.add(key); + } + } + if (!placeholders.size) { + return ''; + } + const placeholderList = `[${Array.from(placeholders) + .sort() + .map((placeholder) => `'${placeholder.replaceAll("'", "\\'")}'`) + .join(', ')}]`; + return `Here are placeholders for sensitive data:\n${placeholderList}\nTo use them, write the placeholder name`; + } + prepare_step_state(browser_state_summary, model_output = null, result = null, step_info = null, sensitive_data = null) { + this.state.history.context_messages = []; + this.updateAgentHistoryDescription(model_output, result, step_info); + const effectiveSensitiveData = sensitive_data ?? this.sensitiveData; + if (effectiveSensitiveData) { + this.sensitiveDataDescription = this.getSensitiveDataDescription(browser_state_summary.url, effectiveSensitiveData); + } + } + async maybe_compact_messages(llm, settings, step_info = null) { + if (!settings || !settings.enabled || !llm || !step_info) { + return false; + } + const stepsSince = step_info.step_number - (this.state.last_compaction_step ?? 0); + if (stepsSince < settings.compact_every_n_steps) { + return false; + } + const historyItems = this.state.agent_history_items; + const fullHistoryText = historyItems + .map((item) => item.to_string()) + .join('\n') + .trim(); + const triggerCharCount = settings.trigger_char_count ?? 40000; + if (fullHistoryText.length < triggerCharCount) { + return false; + } + logger.debug(`Compacting message history (items=${historyItems.length}, chars=${fullHistoryText.length})`); + const compactionSections = []; + if (this.state.compacted_memory) { + compactionSections.push(`\n${this.state.compacted_memory}\n`); + } + compactionSections.push(`\n${fullHistoryText}\n`); + if (settings.include_read_state && this.state.read_state_description) { + compactionSections.push(`\n${this.state.read_state_description}\n`); + } + let compactionInput = compactionSections.join('\n\n'); + if (this.sensitiveData) { + const filtered = this.filterSensitiveData(new UserMessage(compactionInput)); + compactionInput = filtered.text; + } + let systemPrompt = 'You are summarizing an agent run for prompt compaction.\n' + + 'Capture task requirements, key facts, decisions, partial progress, errors, and next steps.\n' + + 'Preserve important entities, values, URLs, and file paths.\n' + + 'Return plain text only. Do not include tool calls or JSON.'; + if (settings.summary_max_chars) { + systemPrompt += ` Keep under ${settings.summary_max_chars} characters if possible.`; + } + let summary; + try { + const response = await llm.ainvoke([ + new SystemMessage(systemPrompt), + new UserMessage(compactionInput), + ]); + summary = String(response?.completion ?? '').trim(); + } + catch (error) { + logger.warning(`Failed to compact messages: ${error instanceof Error ? error.message : String(error)}`); + return false; + } + if (!summary) { + return false; + } + if (settings.summary_max_chars && + summary.length > settings.summary_max_chars) { + summary = `${summary.slice(0, settings.summary_max_chars).trimEnd()}…`; + } + this.state.compacted_memory = summary; + this.state.compaction_count += 1; + this.state.last_compaction_step = step_info.step_number; + const keepLast = Math.max(0, settings.keep_last_items); + if (historyItems.length > keepLast + 1) { + if (keepLast === 0) { + this.state.agent_history_items = [historyItems[0]]; + } + else { + this.state.agent_history_items = [ + historyItems[0], + ...historyItems.slice(-keepLast), + ]; + } + } + logger.debug(`Compaction complete (summary_chars=${summary.length}, history_items=${this.state.agent_history_items.length})`); + return true; + } + create_state_messages(browser_state_summary, model_output = null, result = null, step_info = null, use_vision = true, page_filtered_actions = null, sensitive_data = null, available_file_paths = null, include_recent_events = null, plan_description = null, unavailable_skills_info = null, skip_state_update = false) { + if (!skip_state_update) { + this.prepare_step_state(browser_state_summary, model_output, result, step_info, sensitive_data); + } + const screenshots = []; + let includeScreenshotRequested = false; + if (result) { + for (const actionResult of result) { + if (actionResult.metadata?.include_screenshot) { + includeScreenshotRequested = true; + break; + } + } + } + let includeScreenshot = false; + if (use_vision === true) { + includeScreenshot = true; + } + else if (use_vision === 'auto') { + includeScreenshot = includeScreenshotRequested; + } + if (includeScreenshot && browser_state_summary.screenshot) { + screenshots.push(browser_state_summary.screenshot); + } + const effectiveUseVision = screenshots.length > 0; + const includeRecentEvents = include_recent_events ?? this.includeRecentEvents; + const prompt = new AgentMessagePrompt({ + browser_state_summary, + file_system: this.fileSystem, + agent_history_description: this.agent_history_description, + read_state_description: this.state.read_state_description, + task: this.task, + include_attributes: this.includeAttributes, + step_info, + page_filtered_actions, + sensitive_data: this.sensitiveDataDescription, + available_file_paths, + screenshots, + vision_detail_level: this.visionDetailLevel, + include_recent_events: includeRecentEvents, + sample_images: this.sampleImages, + read_state_images: this.state.read_state_images, + llm_screenshot_size: this.llmScreenshotSize, + plan_description, + unavailable_skills_info, + }); + const message = prompt.get_user_message(effectiveUseVision); + this.last_state_message_text = this.extractStateMessageText(message); + this.setMessageWithType(message, 'state'); + } + get_messages() { + logger.debug(''); + this.lastInputMessages = this.state.history.get_messages(); + return this.lastInputMessages; + } + setMessageWithType(message, messageType) { + if (messageType === 'system') { + this.state.history.system_message = message; + } + else { + const filtered = this.sensitiveData + ? this.filterSensitiveData(message) + : message; + this.state.history.state_message = filtered; + } + } + addContextMessage(message) { + this.state.history.context_messages.push(message); + } + _add_context_message(message) { + this.addContextMessage(message); + } + extractStateMessageText(message) { + if (typeof message.content === 'string') { + return message.content; + } + if (!Array.isArray(message.content)) { + return null; + } + return message.content + .map((part) => { + if (part instanceof ContentPartTextParam) { + return part.text; + } + return null; + }) + .filter((part) => typeof part === 'string') + .join('\n'); + } + filterSensitiveData(message) { + if (!this.sensitiveData) { + return message; + } + const replaceSensitive = (value) => { + const placeholders = {}; + for (const [keyOrDomain, content] of Object.entries(this.sensitiveData)) { + if (content && typeof content === 'object' && !Array.isArray(content)) { + for (const [k, v] of Object.entries(content)) { + if (v) + placeholders[k] = v; + } + } + else if (typeof content === 'string') { + placeholders[keyOrDomain] = content; + } + } + if (!Object.keys(placeholders).length) { + return value; + } + let updated = value; + for (const [key, val] of Object.entries(placeholders)) { + updated = updated.replaceAll(val, `${key}`); + } + return updated; + }; + if (typeof message.content === 'string') { + message.content = replaceSensitive(message.content); + } + else if (Array.isArray(message.content)) { + message.content = message.content.map((part) => { + if (part instanceof ContentPartTextParam) { + part.text = replaceSensitive(part.text); + } + return part; + }); + } + return message; + } +} diff --git a/dist/agent/message-manager/utils.d.ts b/dist/agent/message-manager/utils.d.ts new file mode 100644 index 00000000..d09d42a5 --- /dev/null +++ b/dist/agent/message-manager/utils.d.ts @@ -0,0 +1,2 @@ +import { Message } from '../../llm/messages.js'; +export declare const saveConversation: (inputMessages: Message[], response: unknown, target: string, encoding?: BufferEncoding) => Promise; diff --git a/dist/agent/message-manager/utils.js b/dist/agent/message-manager/utils.js new file mode 100644 index 00000000..246202f6 --- /dev/null +++ b/dist/agent/message-manager/utils.js @@ -0,0 +1,40 @@ +import fs from 'node:fs'; +import path from 'node:path'; +const serializeResponse = (response) => { + if (!response) { + return ''; + } + if (typeof response.model_dump_json === 'function') { + try { + const raw = response.model_dump_json({ exclude_unset: true }); + return JSON.stringify(JSON.parse(raw), null, 2); + } + catch { + /* fall through */ + } + } + try { + return JSON.stringify(response, null, 2); + } + catch { + return String(response); + } +}; +const formatConversation = (messages, response) => { + const lines = []; + messages.forEach((message) => { + lines.push(` ${message.role} `); + lines.push(typeof message.text === 'function' + ? message.text() + : (message.text ?? '')); + lines.push(''); + }); + lines.push(serializeResponse(response)); + return lines.join('\n'); +}; +export const saveConversation = async (inputMessages, response, target, encoding = 'utf-8') => { + const targetPath = path.resolve(target); + await fs.promises.mkdir(path.dirname(targetPath), { recursive: true }); + const payload = formatConversation(inputMessages, response); + await fs.promises.writeFile(targetPath, payload, { encoding }); +}; diff --git a/dist/agent/message-manager/views.d.ts b/dist/agent/message-manager/views.d.ts new file mode 100644 index 00000000..915954d5 --- /dev/null +++ b/dist/agent/message-manager/views.d.ts @@ -0,0 +1,30 @@ +import type { Message } from '../../llm/messages.js'; +export declare class HistoryItem { + step_number: number | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + action_results: string | null; + error: string | null; + system_message: string | null; + constructor(step_number?: number | null, evaluation_previous_goal?: string | null, memory?: string | null, next_goal?: string | null, action_results?: string | null, error?: string | null, system_message?: string | null); + to_string(): string; +} +export declare class MessageHistory { + system_message: Message | null; + state_message: Message | null; + context_messages: Message[]; + get_messages(): Message[]; +} +export declare class MessageManagerState { + history: MessageHistory; + tool_id: number; + agent_history_items: HistoryItem[]; + read_state_description: string; + read_state_images: Array>; + compacted_memory: string | null; + compaction_count: number; + last_compaction_step: number | null; + get historyMessages(): Message[]; + get_messages(): Message[]; +} diff --git a/dist/agent/message-manager/views.js b/dist/agent/message-manager/views.js new file mode 100644 index 00000000..85bb7a55 --- /dev/null +++ b/dist/agent/message-manager/views.js @@ -0,0 +1,77 @@ +export class HistoryItem { + step_number; + evaluation_previous_goal; + memory; + next_goal; + action_results; + error; + system_message; + constructor(step_number = null, evaluation_previous_goal = null, memory = null, next_goal = null, action_results = null, error = null, system_message = null) { + this.step_number = step_number; + this.evaluation_previous_goal = evaluation_previous_goal; + this.memory = memory; + this.next_goal = next_goal; + this.action_results = action_results; + this.error = error; + this.system_message = system_message; + if (this.error && this.system_message) { + throw new Error('Cannot have both error and system_message at the same time'); + } + } + to_string() { + const stepStr = this.step_number != null ? 'step' : 'step_unknown'; + if (this.error) { + return `<${stepStr}>\n${this.error}`; + } + if (this.system_message) { + return this.system_message; + } + const parts = []; + if (this.evaluation_previous_goal) { + parts.push(`${this.evaluation_previous_goal}`); + } + if (this.memory) { + parts.push(`${this.memory}`); + } + if (this.next_goal) { + parts.push(`${this.next_goal}`); + } + if (this.action_results) { + parts.push(this.action_results); + } + const content = parts.join('\n'); + return `<${stepStr}>\n${content}`; + } +} +export class MessageHistory { + system_message = null; + state_message = null; + context_messages = []; + get_messages() { + const messages = []; + if (this.system_message) + messages.push(this.system_message); + if (this.state_message) + messages.push(this.state_message); + messages.push(...this.context_messages); + return messages; + } +} +export class MessageManagerState { + history = new MessageHistory(); + tool_id = 1; + agent_history_items = [ + new HistoryItem(0, null, null, null, null, null, 'Agent initialized'), + ]; + read_state_description = ''; + read_state_images = []; + compacted_memory = null; + compaction_count = 0; + last_compaction_step = null; + get historyMessages() { + return this.history.get_messages(); + } + get_messages() { + return this.history.get_messages(); + } +} diff --git a/dist/agent/prompts.d.ts b/dist/agent/prompts.d.ts new file mode 100644 index 00000000..e3edb18d --- /dev/null +++ b/dist/agent/prompts.d.ts @@ -0,0 +1,73 @@ +import { SystemMessage, UserMessage, ContentPartTextParam, ContentPartImageParam } from '../llm/messages.js'; +import type { AgentStepInfo } from './views.js'; +import type { BrowserStateSummary } from '../browser/views.js'; +import type { FileSystem } from '../filesystem/file-system.js'; +export declare class SystemPrompt { + private readonly maxActionsPerStep; + private readonly overrideSystemMessage; + private readonly extendSystemMessage; + private readonly useThinking; + private readonly flashMode; + private readonly isAnthropic; + private readonly isBrowserUseModel; + private readonly modelName; + private promptTemplate; + private systemMessage; + constructor(maxActionsPerStep?: number, overrideSystemMessage?: string | null, extendSystemMessage?: string | null, useThinking?: boolean, flashMode?: boolean, isAnthropic?: boolean, isBrowserUseModel?: boolean, modelName?: string | null); + private isAnthropic45Model; + private loadPromptTemplate; + get_system_message(): SystemMessage; +} +interface AgentMessagePromptInit { + browser_state_summary: BrowserStateSummary; + file_system: FileSystem; + agent_history_description?: string | null; + read_state_description?: string | null; + task?: string | null; + include_attributes?: string[] | null; + step_info?: AgentStepInfo | null; + page_filtered_actions?: string | null; + max_clickable_elements_length?: number; + sensitive_data?: string | null; + available_file_paths?: string[] | null; + screenshots?: string[] | null; + vision_detail_level?: 'auto' | 'low' | 'high'; + include_recent_events?: boolean; + sample_images?: Array | null; + read_state_images?: Array> | null; + llm_screenshot_size?: [number, number] | null; + unavailable_skills_info?: string | null; + plan_description?: string | null; +} +export declare class AgentMessagePrompt { + private readonly browserState; + private readonly fileSystem; + private readonly agentHistoryDescription; + private readonly readStateDescription; + private readonly task; + private readonly includeAttributes?; + private readonly stepInfo?; + private readonly pageFilteredActions; + private readonly maxClickableElementsLength; + private readonly sensitiveData; + private readonly availableFilePaths?; + private readonly screenshots; + private readonly visionDetailLevel; + private readonly includeRecentEvents; + private readonly sampleImages; + private readonly readStateImages; + private readonly llmScreenshotSize; + private readonly unavailableSkillsInfo; + private readonly planDescription; + constructor(init: AgentMessagePromptInit); + private extractPageStatistics; + private browserStateDescription; + private agentStateDescription; + private resizeScreenshotForLlm; + get_user_message(use_vision?: boolean): UserMessage; +} +export declare const get_rerun_summary_prompt: (originalTask: string, totalSteps: number, successCount: number, errorCount: number) => string; +export declare const get_rerun_summary_message: (prompt: string, screenshotB64?: string | null) => UserMessage; +export declare const get_ai_step_system_prompt: () => string; +export declare const get_ai_step_user_prompt: (query: string, statsSummary: string, content: string) => string; +export {}; diff --git a/dist/agent/prompts.js b/dist/agent/prompts.js new file mode 100644 index 00000000..c1a62e62 --- /dev/null +++ b/dist/agent/prompts.js @@ -0,0 +1,474 @@ +var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) { + var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d; + if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc); + else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r; + return c > 3 && r && Object.defineProperty(target, key, r), r; +}; +import fs from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { Image, createCanvas } from 'canvas'; +import { SystemMessage, UserMessage, ContentPartTextParam, ContentPartImageParam, ImageURL, } from '../llm/messages.js'; +import { observe_debug } from '../observability.js'; +import { is_new_tab_page, sanitize_surrogates } from '../utils.js'; +import { createLogger } from '../logging-config.js'; +import { DOMElementNode } from '../dom/views.js'; +const logger = createLogger('browser_use.agent.prompts'); +const readPromptTemplate = (filename) => { + const filePath = fileURLToPath(new URL(filename, import.meta.url)); + return fs.readFileSync(filePath, 'utf-8'); +}; +export class SystemPrompt { + maxActionsPerStep; + overrideSystemMessage; + extendSystemMessage; + useThinking; + flashMode; + isAnthropic; + isBrowserUseModel; + modelName; + promptTemplate = ''; + systemMessage; + constructor(maxActionsPerStep = 3, overrideSystemMessage = null, extendSystemMessage = null, useThinking = true, flashMode = false, isAnthropic = false, isBrowserUseModel = false, modelName = null) { + this.maxActionsPerStep = maxActionsPerStep; + this.overrideSystemMessage = overrideSystemMessage; + this.extendSystemMessage = extendSystemMessage; + this.useThinking = useThinking; + this.flashMode = flashMode; + this.isAnthropic = isAnthropic; + this.isBrowserUseModel = isBrowserUseModel; + this.modelName = modelName; + if (overrideSystemMessage !== null) { + this.promptTemplate = overrideSystemMessage; + } + else { + this.loadPromptTemplate(); + } + let prompt = this.promptTemplate.replace('{max_actions}', String(this.maxActionsPerStep)); + if (this.extendSystemMessage) { + prompt += `\n${this.extendSystemMessage}`; + } + this.systemMessage = new SystemMessage(prompt); + this.systemMessage.cache = true; + } + isAnthropic45Model() { + if (!this.modelName) { + return false; + } + const modelLower = this.modelName.toLowerCase(); + const isOpus45 = modelLower.includes('opus') && + (modelLower.includes('4.5') || modelLower.includes('4-5')); + const isHaiku45 = modelLower.includes('haiku') && + (modelLower.includes('4.5') || modelLower.includes('4-5')); + return isOpus45 || isHaiku45; + } + loadPromptTemplate() { + let templateName = './system_prompt.md'; + if (this.isBrowserUseModel) { + if (this.flashMode) { + templateName = './system_prompt_browser_use_flash.md'; + } + else if (this.useThinking) { + templateName = './system_prompt_browser_use.md'; + } + else { + templateName = './system_prompt_browser_use_no_thinking.md'; + } + } + else if (this.flashMode && this.isAnthropic45Model()) { + templateName = './system_prompt_anthropic_flash.md'; + } + else if (this.flashMode && this.isAnthropic) { + templateName = './system_prompt_flash_anthropic.md'; + } + else if (this.flashMode) { + templateName = './system_prompt_flash.md'; + } + else if (!this.useThinking) { + templateName = './system_prompt_no_thinking.md'; + } + try { + this.promptTemplate = readPromptTemplate(templateName); + } + catch (error) { + throw new Error(`Failed to load system prompt template: ${error.message}`, { cause: error }); + } + } + get_system_message() { + return this.systemMessage; + } +} +export class AgentMessagePrompt { + browserState; + fileSystem; + agentHistoryDescription; + readStateDescription; + task; + includeAttributes; + stepInfo; + pageFilteredActions; + maxClickableElementsLength; + sensitiveData; + availableFilePaths; + screenshots; + visionDetailLevel; + includeRecentEvents; + sampleImages; + readStateImages; + llmScreenshotSize; + unavailableSkillsInfo; + planDescription; + constructor(init) { + this.browserState = init.browser_state_summary; + this.fileSystem = init.file_system; + this.agentHistoryDescription = init.agent_history_description ?? null; + this.readStateDescription = init.read_state_description ?? null; + this.task = init.task ?? null; + this.includeAttributes = init.include_attributes ?? null; + this.stepInfo = init.step_info ?? null; + this.pageFilteredActions = init.page_filtered_actions ?? null; + this.maxClickableElementsLength = + init.max_clickable_elements_length ?? 40000; + this.sensitiveData = init.sensitive_data ?? null; + this.availableFilePaths = init.available_file_paths ?? null; + this.screenshots = init.screenshots ?? []; + this.visionDetailLevel = init.vision_detail_level ?? 'auto'; + this.includeRecentEvents = init.include_recent_events ?? false; + this.sampleImages = init.sample_images ?? []; + this.readStateImages = init.read_state_images ?? []; + this.llmScreenshotSize = init.llm_screenshot_size ?? null; + this.unavailableSkillsInfo = init.unavailable_skills_info ?? null; + this.planDescription = init.plan_description ?? null; + } + extractPageStatistics() { + const stats = { + links: 0, + iframes: 0, + shadow_open: 0, + shadow_closed: 0, + scroll_containers: 0, + images: 0, + interactive_elements: 0, + total_elements: 0, + }; + const root = this.browserState.element_tree; + if (!root) { + return stats; + } + const traverseNode = (node) => { + stats.total_elements += 1; + const tag = String(node.tag_name ?? '').toLowerCase(); + if (tag === 'a') { + stats.links += 1; + } + else if (tag === 'iframe' || tag === 'frame') { + stats.iframes += 1; + } + else if (tag === 'img') { + stats.images += 1; + } + if (node.is_interactive) { + stats.interactive_elements += 1; + } + if (node.shadow_root) { + // The TS DOM snapshot currently tracks presence of a shadow root, but + // does not expose open-vs-closed mode; count these as open for parity. + stats.shadow_open += 1; + } + for (const child of node.children) { + if (child instanceof DOMElementNode) { + traverseNode(child); + } + } + }; + traverseNode(root); + return stats; + } + browserStateDescription() { + const pageStats = this.extractPageStatistics(); + let statsText = ''; + if (pageStats.total_elements < 10) { + statsText += 'Page appears empty (SPA not loaded?) - '; + } + statsText += `${pageStats.links} links, ${pageStats.interactive_elements} interactive, ${pageStats.iframes} iframes`; + if (pageStats.shadow_open > 0 || pageStats.shadow_closed > 0) { + statsText += `, ${pageStats.shadow_open} shadow(open), ${pageStats.shadow_closed} shadow(closed)`; + } + if (pageStats.images > 0) { + statsText += `, ${pageStats.images} images`; + } + statsText += `, ${pageStats.total_elements} total elements`; + statsText += '\n'; + let elementsText = this.browserState.llm_representation(this.includeAttributes ?? undefined); + let truncatedText = ''; + if (elementsText.length > this.maxClickableElementsLength) { + elementsText = elementsText.slice(0, this.maxClickableElementsLength); + truncatedText = ` (truncated to ${this.maxClickableElementsLength} characters)`; + } + let hasContentAbove = false; + let hasContentBelow = false; + const pi = this.browserState.page_info; + let pageInfoText = ''; + if (pi) { + const pagesAbove = pi.viewport_height > 0 ? pi.pixels_above / pi.viewport_height : 0; + const pagesBelow = pi.viewport_height > 0 ? pi.pixels_below / pi.viewport_height : 0; + hasContentAbove = pagesAbove > 0; + hasContentBelow = pagesBelow > 0; + pageInfoText = ''; + pageInfoText += `${pagesAbove.toFixed(1)} above, `; + pageInfoText += `${pagesBelow.toFixed(1)} below `; + pageInfoText += '\n'; + } + if (elementsText) { + if (!hasContentAbove) { + elementsText = `[Start of page]\n${elementsText}`; + } + if (!hasContentBelow) { + elementsText = `${elementsText}\n[End of page]`; + } + } + else { + elementsText = 'empty page'; + } + let tabsText = ''; + const resolveTabIdentifier = (tab) => typeof tab.tab_id === 'string' && tab.tab_id.trim() + ? tab.tab_id.trim().slice(-4) + : String(tab.page_id); + const currentTabCandidates = []; + for (const tab of this.browserState.tabs) { + if (tab.url === this.browserState.url && + tab.title === this.browserState.title) { + currentTabCandidates.push(resolveTabIdentifier(tab)); + } + } + const currentTabId = currentTabCandidates.length === 1 ? currentTabCandidates[0] : null; + for (const tab of this.browserState.tabs) { + tabsText += `Tab ${resolveTabIdentifier(tab)}: ${tab.url} - ${tab.title.slice(0, 30)}\n`; + } + const currentTabText = currentTabId !== null ? `Current tab: ${currentTabId}` : ''; + const pdfMessage = this.browserState.is_pdf_viewer + ? 'PDF viewer cannot be rendered. In this page, DO NOT use the extract action as PDF content cannot be rendered. Use the read_file action on the downloaded PDF in available_file_paths to read the full text content.\n\n' + : ''; + const recentEventsText = this.includeRecentEvents && this.browserState.recent_events + ? `Recent browser events: ${this.browserState.recent_events}\n` + : ''; + let closedPopupsText = ''; + if (Array.isArray(this.browserState.closed_popup_messages) && + this.browserState.closed_popup_messages.length > 0) { + closedPopupsText = 'Auto-closed JavaScript dialogs:\n'; + for (const popupMessage of this.browserState.closed_popup_messages) { + closedPopupsText += ` - ${popupMessage}\n`; + } + closedPopupsText += '\n'; + } + return `${statsText}${currentTabText} +Available tabs: +${tabsText} +${pageInfoText} +${recentEventsText}${closedPopupsText}${pdfMessage}Interactive elements${truncatedText}: +${elementsText} +`; + } + agentStateDescription() { + const todoContents = this.fileSystem.get_todo_contents(); + const todoText = todoContents || '[empty todo.md, fill it when applicable]'; + const now = new Date(); + const pad = (value) => String(value).padStart(2, '0'); + const dateString = `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`; + let stepInfoDescription = this.stepInfo != null + ? `Step${this.stepInfo.step_number + 1} maximum:${this.stepInfo.max_steps}\n` + : ''; + stepInfoDescription += `Today:${dateString}`; + let agentState = ` +${this.task ?? ''} + + +${this.fileSystem.describe()} + + +${todoText} + +`; + if (this.planDescription) { + agentState += ` +${this.planDescription} + +`; + } + if (this.sensitiveData) { + agentState += `${this.sensitiveData} +`; + } + agentState += `${stepInfoDescription} +`; + if (this.availableFilePaths?.length) { + agentState += `${this.availableFilePaths.join('\n')} +Use with absolute paths +`; + } + return agentState; + } + resizeScreenshotForLlm(screenshotB64) { + if (!this.llmScreenshotSize) { + return screenshotB64; + } + try { + const [targetWidth, targetHeight] = this.llmScreenshotSize; + const image = new Image(); + image.src = Buffer.from(screenshotB64, 'base64'); + if (image.width === targetWidth && image.height === targetHeight) { + return screenshotB64; + } + logger.info(`Resizing screenshot from ${image.width}x${image.height} to ${targetWidth}x${targetHeight} for LLM`); + const canvas = createCanvas(targetWidth, targetHeight); + const context = canvas.getContext('2d'); + context.drawImage(image, 0, 0, targetWidth, targetHeight); + return canvas.toBuffer('image/png').toString('base64'); + } + catch (error) { + logger.warning(`Failed to resize screenshot: ${error.message}, using original`); + return screenshotB64; + } + } + // @ts-ignore - Decorator type mismatch with TypeScript strict mode + get_user_message(use_vision = true) { + if (is_new_tab_page(this.browserState.url) && + this.stepInfo && + this.stepInfo.step_number === 0 && + this.browserState.tabs.length === 1) { + use_vision = false; + } + let stateDescription = ` +${(this.agentHistoryDescription ?? '').trim()} + + +`; + stateDescription += ` +${this.agentStateDescription().trim()} + +`; + stateDescription += ` +${this.browserStateDescription().trim()} + +`; + const readState = (this.readStateDescription ?? '').trim(); + if (readState) { + stateDescription += ` +${readState} + +`; + } + if (this.pageFilteredActions) { + stateDescription += ` +${this.pageFilteredActions} + +`; + } + if (this.unavailableSkillsInfo) { + stateDescription += `\n${this.unavailableSkillsInfo}\n`; + } + stateDescription = sanitize_surrogates(stateDescription); + const hasReadStateImages = this.readStateImages.length > 0; + if ((use_vision === true && this.screenshots.length > 0) || + hasReadStateImages) { + const parts = [ + new ContentPartTextParam(stateDescription), + ]; + parts.push(...this.sampleImages); + this.screenshots.forEach((shot, index) => { + const label = index === this.screenshots.length - 1 + ? 'Current screenshot:' + : 'Previous screenshot:'; + const processedScreenshot = this.resizeScreenshotForLlm(shot); + parts.push(new ContentPartTextParam(label)); + parts.push(new ContentPartImageParam(new ImageURL(`data:image/png;base64,${processedScreenshot}`, this.visionDetailLevel, 'image/png'))); + }); + for (const imageInfo of this.readStateImages) { + const imageName = typeof imageInfo.name === 'string' ? imageInfo.name : 'unknown'; + const imageData = typeof imageInfo.data === 'string' ? imageInfo.data : null; + if (!imageData) { + continue; + } + const mediaType = imageName.toLowerCase().endsWith('.png') + ? 'image/png' + : 'image/jpeg'; + parts.push(new ContentPartTextParam(`Image from file: ${imageName}`)); + parts.push(new ContentPartImageParam(new ImageURL(`data:${mediaType};base64,${imageData}`, this.visionDetailLevel, mediaType))); + } + const message = new UserMessage(parts); + message.cache = true; + return message; + } + const message = new UserMessage(stateDescription); + message.cache = true; + return message; + } +} +__decorate([ + observe_debug({ + name: 'agent_message_prompt:get_user_message', + ignore_input: true, + ignore_output: true, + }) +], AgentMessagePrompt.prototype, "get_user_message", null); +export const get_rerun_summary_prompt = (originalTask, totalSteps, successCount, errorCount) => `You are analyzing the completion of a rerun task. Based on the screenshot and execution info, provide a summary. + +Original task: ${originalTask} + +Execution statistics: +- Total steps: ${totalSteps} +- Successful steps: ${successCount} +- Failed steps: ${errorCount} + +Analyze the screenshot to determine: +1. Whether the task completed successfully +2. What the final state shows +3. Overall completion status (complete/partial/failed) + +Respond with: +- summary: A clear, concise summary of what happened during the rerun +- success: Whether the task completed successfully (true/false) +- completion_status: One of "complete", "partial", or "failed"`; +export const get_rerun_summary_message = (prompt, screenshotB64 = null) => { + if (screenshotB64) { + const parts = [ + new ContentPartTextParam(prompt), + new ContentPartImageParam(new ImageURL(`data:image/png;base64,${screenshotB64}`)), + ]; + return new UserMessage(parts); + } + return new UserMessage(prompt); +}; +export const get_ai_step_system_prompt = () => ` +You are an expert at extracting data from webpages. + + +You will be given: +1. A query describing what to extract +2. The markdown of the webpage (filtered to remove noise) +3. Optionally, a screenshot of the current page state + + + +- Extract information from the webpage that is relevant to the query +- ONLY use the information available in the webpage - do not make up information +- If the information is not available, mention that clearly +- If the query asks for all items, list all of them + + + +- Present ALL relevant information in a concise way +- Do not use conversational format - directly output the relevant information +- If information is unavailable, state that clearly + +`.trim(); +export const get_ai_step_user_prompt = (query, statsSummary, content) => ` +${query} + + + +${statsSummary} + + + +${content} +`; diff --git a/dist/agent/service.d.ts b/dist/agent/service.d.ts new file mode 100644 index 00000000..0f02127b --- /dev/null +++ b/dist/agent/service.d.ts @@ -0,0 +1,353 @@ +import { createLogger } from '../logging-config.js'; +import { EventBus } from '../event-bus.js'; +import type { Controller } from '../controller/service.js'; +import type { FileSystem } from '../filesystem/file-system.js'; +import { SystemPrompt } from './prompts.js'; +import { MessageManager } from './message-manager/service.js'; +import { BrowserStateSummary } from '../browser/views.js'; +import { BrowserSession } from '../browser/session.js'; +import { BrowserProfile } from '../browser/profile.js'; +import type { Browser, BrowserContext, Page } from '../browser/types.js'; +import type { BaseChatModel } from '../llm/base.js'; +import { ContentPartImageParam, ContentPartTextParam } from '../llm/messages.js'; +import { ActionResult, AgentHistoryList, AgentOutput, AgentSettings, AgentState, AgentStepInfo, ActionModel, DetectedVariable, MessageCompactionSettings } from './views.js'; +import type { StructuredOutputParser } from './views.js'; +import { ScreenshotService } from '../screenshots/service.js'; +import { ProductTelemetry } from '../telemetry/service.js'; +import { type SkillService } from '../skills/index.js'; +export declare const log_response: (response: AgentOutput, registry?: Controller, logInstance?: import("../logging-config.js").Logger) => void; +type ControllerContext = unknown; +type AgentHookFunc = (agent: Agent) => Promise | void; +interface RerunHistoryOptions { + max_retries?: number; + skip_failures?: boolean; + delay_between_actions?: number; + max_step_interval?: number; + wait_for_elements?: boolean; + summary_llm?: BaseChatModel | null; + ai_step_llm?: BaseChatModel | null; + signal?: AbortSignal | null; +} +interface LoadAndRerunOptions extends RerunHistoryOptions { + variables?: Record | null; +} +interface AgentConstructorParams { + task: string; + llm?: BaseChatModel | null; + page?: Page | null; + browser?: Browser | BrowserSession | null; + browser_context?: BrowserContext | null; + browser_profile?: BrowserProfile | null; + browser_session?: BrowserSession | null; + tools?: Controller | null; + controller?: Controller | null; + sensitive_data?: Record> | null; + initial_actions?: Array>> | null; + directly_open_url?: boolean; + register_new_step_callback?: ((summary: BrowserStateSummary, output: AgentOutput, step: number) => void | Promise) | null; + register_done_callback?: ((history: AgentHistoryList) => void | Promise) | null; + register_should_stop_callback?: (() => Promise) | null; + register_external_agent_status_raise_error_callback?: (() => Promise) | null; + output_model_schema?: StructuredOutputParser | null; + extraction_schema?: Record | null; + use_vision?: boolean | 'auto'; + include_recent_events?: boolean; + sample_images?: Array | null; + llm_screenshot_size?: [number, number] | null; + save_conversation_path?: string | null; + save_conversation_path_encoding?: BufferEncoding | null; + max_failures?: number; + override_system_message?: string | null; + extend_system_message?: string | null; + generate_gif?: boolean | string; + available_file_paths?: string[] | null; + include_attributes?: string[]; + max_actions_per_step?: number; + use_thinking?: boolean; + flash_mode?: boolean; + use_judge?: boolean; + ground_truth?: string | null; + max_history_items?: number | null; + page_extraction_llm?: BaseChatModel | null; + fallback_llm?: BaseChatModel | null; + judge_llm?: BaseChatModel | null; + skill_ids?: Array | null; + skills?: Array | null; + skill_service?: SkillService | null; + enable_planning?: boolean; + planning_replan_on_stall?: number; + planning_exploration_limit?: number; + injected_agent_state?: AgentState | null; + context?: Context | null; + source?: string | null; + file_system_path?: string | null; + task_id?: string | null; + cloud_sync?: any; + calculate_cost?: boolean; + display_files_in_done_text?: boolean; + include_tool_call_examples?: boolean; + vision_detail_level?: AgentSettings['vision_detail_level']; + session_attachment_mode?: AgentSettings['session_attachment_mode']; + llm_timeout?: number | null; + step_timeout?: number; + final_response_after_failure?: boolean; + message_compaction?: MessageCompactionSettings | boolean | null; + loop_detection_window?: number; + loop_detection_enabled?: boolean; + _url_shortening_limit?: number; +} +export declare class Agent { + private static _sharedSessionStepLocks; + static DEFAULT_AGENT_DATA_DIR: string; + browser_session: BrowserSession | null; + llm: BaseChatModel; + judge_llm: BaseChatModel; + unfiltered_actions: string; + initial_actions: Array>> | null; + initial_url: string | null; + register_new_step_callback: AgentConstructorParams['register_new_step_callback']; + register_done_callback: AgentConstructorParams['register_done_callback']; + register_should_stop_callback: AgentConstructorParams['register_should_stop_callback']; + register_external_agent_status_raise_error_callback: AgentConstructorParams['register_external_agent_status_raise_error_callback']; + context: Context | null; + telemetry: ProductTelemetry; + eventbus: EventBus; + enable_cloud_sync: boolean; + cloud_sync: any; + file_system: FileSystem | null; + screenshot_service: ScreenshotService | null; + agent_directory: string; + private _current_screenshot_path; + has_downloads_path: boolean; + private _last_known_downloads; + version: string; + source: string; + step_start_time: number; + _external_pause_event: { + resolve: (() => void) | null; + promise: Promise; + }; + output_model_schema: StructuredOutputParser | null; + extraction_schema: Record | null; + id: string; + task_id: string; + session_id: string; + task: string; + controller: Controller; + settings: AgentSettings; + token_cost_service: any; + state: AgentState; + history: AgentHistoryList; + _message_manager: MessageManager; + available_file_paths: string[]; + sensitive_data: Record> | null; + _logger: ReturnType | null; + _file_system_path: string | null; + agent_current_page: Page | null; + _session_start_time: number; + _task_start_time: number; + _force_exit_telemetry_logged: boolean; + private _closePromise; + private _hasBrowserSessionClaim; + private _sharedPinnedTabId; + private _enforceDoneOnlyForCurrentStep; + system_prompt_class: SystemPrompt; + ActionModel: typeof ActionModel; + AgentOutput: typeof AgentOutput; + DoneActionModel: typeof ActionModel; + DoneAgentOutput: typeof AgentOutput; + private _fallback_llm; + private _using_fallback_llm; + private _original_llm; + private _url_shortening_limit; + private skill_service; + private _skills_registered; + constructor(params: AgentConstructorParams); + private _normalizeMessageCompactionSetting; + private _createSessionIdWithAgentSuffix; + private _buildEventBusName; + private _copyBrowserProfile; + private _getBrowserContextFromPage; + private _claim_or_isolate_browser_session; + private _release_browser_session_claim; + private _has_any_browser_session_attachments; + private _is_shared_session_mode; + private _capture_shared_pinned_tab; + private _restore_shared_pinned_tab_if_needed; + private _run_with_shared_session_step_lock; + private _cleanup_shared_session_step_lock_if_unused; + private _init_browser_session; + /** + * Convert dictionary-based actions to ActionModel instances + */ + private _convertInitialActions; + /** + * Handle model-specific vision capabilities + * Some models like DeepSeek and Grok don't support vision yet + */ + private _handleModelSpecificVision; + /** + * Verify that the LLM API keys are setup and the LLM API is responding properly. + * Also handles model capability detection. + */ + private _verifyAndSetupLlm; + /** + * Validates security settings when sensitive_data is provided + * Checks if allowed_domains is properly configured to prevent credential leakage + */ + private _validateSecuritySettings; + private _initFileSystem; + private _setScreenshotService; + get logger(): import("../logging-config.js").Logger; + get message_manager(): MessageManager; + /** + * Get the browser instance from the browser session + */ + get browser(): Browser; + /** + * Get the browser context from the browser session + */ + get browserContext(): BrowserContext; + /** + * Get the browser profile from the browser session + */ + get browserProfile(): BrowserProfile; + get is_using_fallback_llm(): boolean; + get current_llm_model(): string; + /** + * Add a new task to the agent, keeping the same task_id as tasks are continuous + */ + addNewTask(newTask: string): void; + private _enhanceTaskWithSchema; + private _getOutputModelSchemaPayload; + private _getToolsOutputModelSchema; + private _getOutputModelSchemaName; + private _resolveStructuredOutputActionSchema; + private _extract_start_url; + /** + * Take a step and return whether the task is done and valid + * @returns Tuple of [is_done, is_valid] + */ + takeStep(stepInfo?: AgentStepInfo): Promise<[boolean, boolean]>; + /** + * Remove think tags from text + */ + private _removeThinkTags; + /** + * Log a comprehensive summary of the next action(s) + */ + private _logNextActionSummary; + private _set_browser_use_version_and_source; + /** + * Setup dynamic action models from controller's registry + * Initially only include actions with no filters + */ + private _setup_action_models; + private _register_skills_as_actions; + private _get_unavailable_skills_info; + /** + * Update action models with page-specific actions + * Called during each step to filter actions based on current page context + */ + private _updateActionModelsForPage; + private _execute_initial_actions; + run(max_steps?: number, on_step_start?: AgentHookFunc | null, on_step_end?: AgentHookFunc | null): Promise>; + private _executeWithTimeout; + _step(step_info?: AgentStepInfo | null, signal?: AbortSignal | null): Promise; + private _prepare_context; + private _maybe_compact_messages; + private _storeScreenshotForStep; + private _get_next_action; + private _execute_actions; + private _post_process; + multi_act(actions: Array>>, options?: { + check_for_new_elements?: boolean; + signal?: AbortSignal | null; + }): Promise; + private _generate_rerun_summary; + private _execute_ai_step; + rerun_history(history: AgentHistoryList, options?: RerunHistoryOptions): Promise; + private _execute_history_step; + private _historyStepNeedsElementMatching; + private _countExpectedElementsFromHistory; + private _waitForMinimumElements; + private _extractActionIndex; + private _extractActionType; + private _sameHistoryElement; + private _is_redundant_retry_step; + private _is_menu_opener_step; + private _is_menu_item_element; + private _reexecute_menu_opener; + private _formatHistoryElementForError; + private _update_action_indices; + load_and_rerun(history_file?: string | null, options?: LoadAndRerunOptions): Promise; + detect_variables(): Record; + save_history(file_path?: string | null): void; + private _coerceHistoryElement; + private _substitute_variables_in_history; + private _clone_history_for_substitution; + private _createAbortError; + private _throwIfAborted; + private _relayAbortSignal; + private _formatDelaySeconds; + private _sleep; + wait_until_resumed(): Promise; + log_completion(): Promise; + pause(): void; + resume(): void; + stop(): void; + close(): Promise; + /** + * Get the trace and trace_details objects for the agent + * Contains comprehensive metadata about the agent run for debugging and analysis + */ + get_trace_object(): { + trace: Record; + trace_details: Record; + }; + private _log_agent_run; + private _createInterruptedError; + private _raise_if_stopped_or_paused; + private _handle_post_llm_processing; + /** Handle all types of errors that can occur during a step (python c011 parity). */ + private _handle_step_error; + private _finalize; + private _handle_final_step; + private _max_total_failures; + private _handle_failure_limit_recovery; + private _update_plan_from_model_output; + private _render_plan_description; + private _inject_replan_nudge; + private _inject_exploration_nudge; + private _inject_loop_detection_nudge; + private _update_loop_detector_actions; + private _update_loop_detector_page_state; + private _inject_budget_warning; + private _run_simple_judge; + private _judge_trace; + private _judge_and_log; + private _replace_urls_in_text; + private _process_messages_and_replace_long_urls_shorter_ones; + private _replace_shortened_urls_in_string; + private _replace_shortened_urls_in_value; + private _parseCompletionPayload; + private _isModelActionMissing; + private _getOutputActionNames; + private _toStrictActionParamSchema; + private _buildActionOutputSchema; + private _buildLlmOutputFormat; + private _get_model_output_with_retry; + private _try_switch_to_fallback_llm; + private _log_fallback_switch; + private _validateAndNormalizeActions; + private _update_action_models_for_page; + private _check_and_update_downloads; + private _update_available_file_paths; + private _log_step_context; + private _log_first_step_startup; + private _log_step_completion_summary; + private _log_agent_event; + private _make_history_item; + save_file_system_state(): void; +} +export {}; diff --git a/dist/agent/service.js b/dist/agent/service.js new file mode 100644 index 00000000..c39073c1 --- /dev/null +++ b/dist/agent/service.js @@ -0,0 +1,4089 @@ +import path from 'node:path'; +import fs from 'node:fs'; +import os from 'node:os'; +import process from 'node:process'; +import { createHash } from 'node:crypto'; +import { config as loadEnv } from 'dotenv'; +import { z } from 'zod'; +import { createLogger } from '../logging-config.js'; +import { CONFIG } from '../config.js'; +import { EventBus } from '../event-bus.js'; +import { uuid7str, SignalHandler, get_browser_use_version, check_latest_browser_use_version, sanitize_surrogates, } from '../utils.js'; +import { Controller as DefaultController } from '../controller/service.js'; +import { FileSystem as AgentFileSystem, DEFAULT_FILE_SYSTEM_PATH, } from '../filesystem/file-system.js'; +import { SystemPrompt, get_ai_step_system_prompt, get_ai_step_user_prompt, get_rerun_summary_message, get_rerun_summary_prompt, } from './prompts.js'; +import { MessageManager } from './message-manager/service.js'; +import { BrowserStateHistory } from '../browser/views.js'; +import { BrowserSession } from '../browser/session.js'; +import { BrowserProfile, DEFAULT_BROWSER_PROFILE } from '../browser/profile.js'; +import { HistoryTreeProcessor } from '../dom/history-tree-processor/service.js'; +import { DOMHistoryElement } from '../dom/history-tree-processor/view.js'; +import { DEFAULT_INCLUDE_ATTRIBUTES, } from '../dom/views.js'; +import { extractCleanMarkdownFromHtml } from '../dom/markdown-extractor.js'; +import { ChatBrowserUse } from '../llm/browser-use/chat.js'; +import { ModelProviderError, ModelRateLimitError } from '../llm/exceptions.js'; +import { AssistantMessage, ContentPartTextParam, SystemMessage, UserMessage, } from '../llm/messages.js'; +import { getLlmByName } from '../llm/models.js'; +import { ActionResult, AgentHistory, AgentHistoryList, AgentOutput, AgentState, AgentStepInfo, AgentError, StepMetadata, ActionModel, PlanItem, defaultMessageCompactionSettings, normalizeMessageCompactionSettings, } from './views.js'; +import { detect_variables_in_history, substitute_in_dict, } from './variable-detector.js'; +import { CreateAgentOutputFileEvent, CreateAgentSessionEvent, CreateAgentTaskEvent, CreateAgentStepEvent, UpdateAgentTaskEvent, } from './cloud-events.js'; +import { create_history_gif } from './gif.js'; +import { ScreenshotService } from '../screenshots/service.js'; +import { productTelemetry } from '../telemetry/service.js'; +import { AgentTelemetryEvent } from '../telemetry/views.js'; +import { TokenCost } from '../tokens/service.js'; +import { construct_judge_messages, construct_simple_judge_messages, } from './judge.js'; +import { CloudSkillService, MissingCookieException, build_skill_parameters_schema, get_skill_slug, } from '../skills/index.js'; +loadEnv(); +const logger = createLogger('browser_use.agent'); +const URL_PATTERN = /https?:\/\/[^\s<>"']+|www\.[^\s<>"']+|[^\s<>"']+\.[a-z]{2,}(?:\/[^\s<>"']*)?/gi; +export const log_response = (response, registry, logInstance = logger) => { + if (response.current_state.thinking) { + logInstance.debug(`💡 Thinking:\n${response.current_state.thinking}`); + } + const evalGoal = response.current_state.evaluation_previous_goal; + if (evalGoal) { + if (evalGoal.toLowerCase().includes('success')) { + logInstance.info(` \x1b[32m👍 Eval: ${evalGoal}\x1b[0m`); + } + else if (evalGoal.toLowerCase().includes('failure')) { + logInstance.info(` \x1b[31m⚠️ Eval: ${evalGoal}\x1b[0m`); + } + else { + logInstance.info(` ❔ Eval: ${evalGoal}`); + } + } + if (response.current_state.memory) { + logInstance.info(` 🧠 Memory: ${response.current_state.memory}`); + } + const nextGoal = response.current_state.next_goal; + if (nextGoal) { + logInstance.info(` \x1b[34m🎯 Next goal: ${nextGoal}\x1b[0m`); + } +}; +class AsyncMutex { + locked = false; + waiters = []; + async acquire() { + if (!this.locked) { + this.locked = true; + let released = false; + return () => { + if (released) { + return; + } + released = true; + this.release(); + }; + } + await new Promise((resolve) => this.waiters.push(resolve)); + this.locked = true; + let released = false; + return () => { + if (released) { + return; + } + released = true; + this.release(); + }; + } + release() { + const next = this.waiters.shift(); + if (next) { + next(); + return; + } + this.locked = false; + } +} +class ExecutionTimeoutError extends Error { + constructor() { + super('Operation timed out'); + this.name = 'ExecutionTimeoutError'; + } +} +const ensureDir = (target) => { + if (!fs.existsSync(target)) { + fs.mkdirSync(target, { recursive: true }); + } +}; +const resolve_agent_llm = (llm) => { + if (llm) { + return llm; + } + const defaultLlmName = CONFIG.DEFAULT_LLM.trim(); + if (defaultLlmName) { + return getLlmByName(defaultLlmName); + } + return new ChatBrowserUse(); +}; +const get_model_timeout = (llm) => { + const modelName = String(llm?.model ?? '').toLowerCase(); + if (modelName.includes('gemini')) { + if (modelName.includes('3-pro')) { + return 90; + } + return 75; + } + if (modelName.includes('groq')) { + return 30; + } + if (modelName.includes('o3') || + modelName.includes('claude') || + modelName.includes('sonnet') || + modelName.includes('deepseek')) { + return 90; + } + return 75; +}; +const defaultAgentOptions = () => ({ + use_vision: true, + include_recent_events: false, + sample_images: null, + llm_screenshot_size: null, + save_conversation_path: null, + save_conversation_path_encoding: 'utf-8', + max_failures: 3, + directly_open_url: true, + override_system_message: null, + extend_system_message: null, + generate_gif: false, + available_file_paths: [], + include_attributes: undefined, + max_actions_per_step: 5, + use_thinking: true, + flash_mode: false, + use_judge: true, + ground_truth: null, + max_history_items: null, + page_extraction_llm: null, + fallback_llm: null, + judge_llm: null, + skill_ids: null, + skills: null, + skill_service: null, + enable_planning: true, + planning_replan_on_stall: 3, + planning_exploration_limit: 5, + context: null, + source: null, + file_system_path: null, + task_id: null, + cloud_sync: null, + calculate_cost: false, + display_files_in_done_text: true, + include_tool_call_examples: false, + session_attachment_mode: 'copy', + vision_detail_level: 'auto', + llm_timeout: null, + step_timeout: 180, + final_response_after_failure: true, + message_compaction: true, + loop_detection_window: 20, + loop_detection_enabled: true, + _url_shortening_limit: 25, +}); +const AgentLLMOutputSchema = z.object({ + thinking: z.string().optional().nullable(), + evaluation_previous_goal: z.string().optional().nullable(), + memory: z.string().optional().nullable(), + next_goal: z.string().optional().nullable(), + current_plan_item: z.number().int().optional().nullable(), + plan_update: z.array(z.string()).optional().nullable(), + action: z + .array(z.record(z.string(), z.any())) + .optional() + .nullable() + .default([]), +}); +const DoneOnlyLLMOutputSchema = AgentLLMOutputSchema.extend({ + action: z + .array(z.object({ + done: z.object({}).passthrough(), + })) + .optional() + .nullable() + .default([]), +}); +const SimpleJudgeSchema = z.object({ + is_correct: z.boolean(), + reason: z.string().optional().default(''), +}); +const JudgeSchema = z.object({ + reasoning: z.string().optional().nullable().default(''), + verdict: z.boolean(), + failure_reason: z.string().optional().nullable().default(''), + impossible_task: z.boolean().optional().default(false), + reached_captcha: z.boolean().optional().default(false), +}); +const AgentLLMOutputFormat = AgentLLMOutputSchema; +AgentLLMOutputFormat.schema = AgentLLMOutputSchema; +const DoneOnlyLLMOutputFormat = DoneOnlyLLMOutputSchema; +DoneOnlyLLMOutputFormat.schema = DoneOnlyLLMOutputSchema; +const SimpleJudgeOutputFormat = SimpleJudgeSchema; +SimpleJudgeOutputFormat.schema = SimpleJudgeSchema; +const JudgeOutputFormat = JudgeSchema; +JudgeOutputFormat.schema = JudgeSchema; +export class Agent { + static _sharedSessionStepLocks = new Map(); + static DEFAULT_AGENT_DATA_DIR = path.join(process.cwd(), DEFAULT_FILE_SYSTEM_PATH); + browser_session = null; + llm; + judge_llm; + unfiltered_actions; + initial_actions; + initial_url = null; + register_new_step_callback; + register_done_callback; + register_should_stop_callback; + register_external_agent_status_raise_error_callback; + context; + telemetry; + eventbus; + enable_cloud_sync; + cloud_sync = null; + file_system = null; + screenshot_service = null; + agent_directory; + _current_screenshot_path = null; + has_downloads_path = false; + _last_known_downloads = []; + version = 'unknown'; + source = 'unknown'; + step_start_time = 0; + _external_pause_event = { + resolve: null, + promise: Promise.resolve(), + }; + output_model_schema; + extraction_schema; + id; + task_id; + session_id; + task; + controller; + settings; + token_cost_service; + state; + history; + _message_manager; + available_file_paths = []; + sensitive_data; + _logger = null; + _file_system_path = null; + agent_current_page = null; + _session_start_time = 0; + _task_start_time = 0; + _force_exit_telemetry_logged = false; + _closePromise = null; + _hasBrowserSessionClaim = false; + _sharedPinnedTabId = null; + _enforceDoneOnlyForCurrentStep = false; + system_prompt_class; + ActionModel = ActionModel; + AgentOutput = AgentOutput; + DoneActionModel = ActionModel; + DoneAgentOutput = AgentOutput; + _fallback_llm = null; + _using_fallback_llm = false; + _original_llm = null; + _url_shortening_limit = 25; + skill_service = null; + _skills_registered = false; + constructor(params) { + const { task, llm, page = null, browser = null, browser_context = null, browser_profile = null, browser_session = null, tools = null, controller = null, sensitive_data = null, initial_actions = null, directly_open_url = true, register_new_step_callback = null, register_done_callback = null, register_should_stop_callback = null, register_external_agent_status_raise_error_callback = null, output_model_schema = null, extraction_schema = null, use_vision = true, include_recent_events = false, sample_images = null, llm_screenshot_size = null, save_conversation_path = null, save_conversation_path_encoding = 'utf-8', max_failures = 3, override_system_message = null, extend_system_message = null, generate_gif = false, available_file_paths = [], include_attributes, max_actions_per_step = 5, use_thinking = true, flash_mode = false, use_judge = true, ground_truth = null, max_history_items = null, page_extraction_llm = null, fallback_llm = null, judge_llm = null, skill_ids = null, skills = null, skill_service = null, enable_planning = true, planning_replan_on_stall = 3, planning_exploration_limit = 5, context = null, source = null, file_system_path = null, task_id = null, cloud_sync = null, calculate_cost = false, display_files_in_done_text = true, include_tool_call_examples = false, vision_detail_level = 'auto', session_attachment_mode = 'copy', llm_timeout = null, step_timeout = 180, final_response_after_failure = true, message_compaction = true, loop_detection_window = 20, loop_detection_enabled = true, _url_shortening_limit = 25, } = { ...defaultAgentOptions(), ...params }; + const resolvedLlm = resolve_agent_llm(llm); + const effectivePageExtractionLlm = page_extraction_llm ?? resolvedLlm; + const effectiveJudgeLlm = judge_llm ?? resolvedLlm; + const effectiveFlashMode = flash_mode || resolvedLlm?.provider === 'browser-use'; + const effectiveEnablePlanning = effectiveFlashMode + ? false + : enable_planning; + const effectiveLlmTimeout = typeof llm_timeout === 'number' + ? llm_timeout + : get_model_timeout(resolvedLlm); + const normalizedMessageCompaction = this._normalizeMessageCompactionSetting(message_compaction); + let resolvedLlmScreenshotSize = llm_screenshot_size ?? null; + if (resolvedLlmScreenshotSize !== null) { + if (!Array.isArray(resolvedLlmScreenshotSize) || + resolvedLlmScreenshotSize.length !== 2) { + throw new Error('llm_screenshot_size must be a tuple of [width, height]'); + } + const [width, height] = resolvedLlmScreenshotSize; + if (!Number.isInteger(width) || !Number.isInteger(height)) { + throw new Error('llm_screenshot_size dimensions must be integers'); + } + if (width < 100 || height < 100) { + throw new Error('llm_screenshot_size dimensions must be at least 100 pixels'); + } + logger.info(`LLM screenshot resizing enabled: ${width}x${height}`); + } + if (resolvedLlmScreenshotSize == null) { + const modelName = String(resolvedLlm?.model ?? ''); + if (modelName.startsWith('claude-sonnet')) { + resolvedLlmScreenshotSize = [1400, 850]; + logger.info('Auto-configured LLM screenshot size for Claude Sonnet: 1400x850'); + } + } + this.llm = resolvedLlm; + this.judge_llm = effectiveJudgeLlm; + this._fallback_llm = fallback_llm; + this._using_fallback_llm = false; + this._original_llm = resolvedLlm; + this._url_shortening_limit = Math.max(0, Math.trunc(_url_shortening_limit)); + this.id = task_id || uuid7str(); + this.task_id = this.id; + this.session_id = uuid7str(); + this.available_file_paths = available_file_paths || []; + if (tools && controller) { + throw new Error('Cannot specify both "tools" and "controller". Use "tools" only.'); + } + const resolvedController = (tools ?? + controller ?? + new DefaultController({ + exclude_actions: use_vision !== 'auto' ? ['screenshot'] : [], + display_files_in_done_text, + })); + const toolsOutputModel = this._getToolsOutputModelSchema(resolvedController); + let resolvedOutputModelSchema = output_model_schema ?? null; + if (resolvedOutputModelSchema && + toolsOutputModel && + resolvedOutputModelSchema !== toolsOutputModel) { + this.logger.warning(`output_model_schema (${this._getOutputModelSchemaName(resolvedOutputModelSchema)}) differs from Tools output_model (${this._getOutputModelSchemaName(toolsOutputModel)}). Using Agent output_model_schema.`); + } + else if (!resolvedOutputModelSchema && toolsOutputModel) { + resolvedOutputModelSchema = toolsOutputModel; + } + this.output_model_schema = resolvedOutputModelSchema; + this.extraction_schema = extraction_schema ?? null; + if (!this.extraction_schema && this.output_model_schema) { + this.extraction_schema = + this._getOutputModelSchemaPayload(this.output_model_schema) ?? null; + } + this.task = this._enhanceTaskWithSchema(task, this.output_model_schema); + this.sensitive_data = sensitive_data; + this.controller = resolvedController; + const setCoordinateClicking = this.controller + ?.set_coordinate_clicking; + if (typeof setCoordinateClicking === 'function') { + const modelName = String(this.llm?.model ?? '').toLowerCase(); + const supportsCoordinateClicking = [ + 'claude-sonnet-4', + 'claude-opus-4', + 'gemini-3-pro', + 'browser-use/', + ].some((pattern) => modelName.includes(pattern)); + setCoordinateClicking.call(this.controller, supportsCoordinateClicking); + } + const structuredOutputActionSchema = this._resolveStructuredOutputActionSchema(this.output_model_schema); + if (structuredOutputActionSchema) { + this.controller.use_structured_output_action(structuredOutputActionSchema); + } + if (skills && skill_ids) { + throw new Error('Cannot specify both "skills" and "skill_ids". Use "skills" only.'); + } + const resolvedSkillIds = skills ?? skill_ids; + if (skill_service) { + this.skill_service = skill_service; + } + else if (resolvedSkillIds && resolvedSkillIds.length > 0) { + this.skill_service = new CloudSkillService({ + skill_ids: resolvedSkillIds, + }); + } + if (use_vision !== 'auto') { + const excludeAction = this.controller?.exclude_action; + if (typeof excludeAction === 'function') { + excludeAction.call(this.controller, 'screenshot'); + } + else { + this.controller.registry.exclude_action?.('screenshot'); + } + } + let resolvedInitialActions = initial_actions; + const hasFollowUpState = Boolean(params.injected_agent_state?.follow_up_task); + if (directly_open_url && + !hasFollowUpState && + !resolvedInitialActions?.length) { + const extractedUrl = this._extract_start_url(task); + if (extractedUrl) { + this.initial_url = extractedUrl; + this.logger.info(`🔗 Found URL in task: ${extractedUrl}, adding as initial action...`); + resolvedInitialActions = [ + { go_to_url: { url: extractedUrl, new_tab: false } }, + ]; + } + } + this.initial_actions = resolvedInitialActions + ? this._convertInitialActions(resolvedInitialActions) + : null; + this.register_new_step_callback = register_new_step_callback; + this.register_done_callback = register_done_callback; + this.register_should_stop_callback = register_should_stop_callback; + this.register_external_agent_status_raise_error_callback = + register_external_agent_status_raise_error_callback; + this.context = context; + this.agent_directory = Agent.DEFAULT_AGENT_DATA_DIR; + this.settings = { + use_vision, + include_recent_events, + vision_detail_level, + save_conversation_path, + save_conversation_path_encoding, + max_failures, + generate_gif, + override_system_message, + extend_system_message, + include_attributes: include_attributes ?? [...DEFAULT_INCLUDE_ATTRIBUTES], + max_actions_per_step, + use_thinking, + flash_mode: effectiveFlashMode, + use_judge, + ground_truth, + max_history_items, + page_extraction_llm: effectivePageExtractionLlm, + enable_planning: effectiveEnablePlanning, + planning_replan_on_stall, + planning_exploration_limit, + calculate_cost, + include_tool_call_examples, + session_attachment_mode, + llm_timeout: effectiveLlmTimeout, + step_timeout, + final_response_after_failure, + message_compaction: normalizedMessageCompaction, + loop_detection_window, + loop_detection_enabled, + }; + this.token_cost_service = new TokenCost(calculate_cost); + if (calculate_cost) { + this.token_cost_service.initialize().catch((error) => { + this.logger.debug(`Failed to initialize token cost service: ${error.message}`); + }); + } + this.token_cost_service.register_llm(resolvedLlm); + this.token_cost_service.register_llm(effectivePageExtractionLlm); + this.token_cost_service.register_llm(effectiveJudgeLlm); + if (normalizedMessageCompaction?.compaction_llm) { + this.token_cost_service.register_llm(normalizedMessageCompaction.compaction_llm); + } + this.state = params.injected_agent_state || new AgentState(); + this.state.loop_detector.window_size = this.settings.loop_detection_window; + this.history = new AgentHistoryList([], null); + this.telemetry = productTelemetry; + this._file_system_path = file_system_path; + this.file_system = this._initFileSystem(file_system_path); + this._setScreenshotService(); + this._setup_action_models(); + this._set_browser_use_version_and_source(source); + this.browser_session = this._init_browser_session({ + page, + browser, + browser_context, + browser_profile, + browser_session, + }); + if (this.browser_session) { + this.browser_session.llm_screenshot_size = resolvedLlmScreenshotSize; + } + this.has_downloads_path = Boolean(this.browser_session?.browser_profile?.downloads_path); + if (this.has_downloads_path) { + this._last_known_downloads = []; + this.logger.debug('📁 Initialized download tracking for agent'); + } + this.system_prompt_class = new SystemPrompt(this.settings.max_actions_per_step, this.settings.override_system_message, this.settings.extend_system_message, this.settings.use_thinking, this.settings.flash_mode, String(this.llm?.provider ?? '').toLowerCase() === 'anthropic', String(this.llm?.model ?? '') + .toLowerCase() + .includes('browser-use/'), String(this.llm?.model ?? '')); + this._message_manager = new MessageManager(this.task, this.system_prompt_class.get_system_message(), this.file_system, this.state.message_manager_state, this.settings.use_thinking, this.settings.include_attributes, sensitive_data ?? undefined, this.settings.max_history_items, this.settings.vision_detail_level, this.settings.include_tool_call_examples, this.settings.include_recent_events, sample_images ?? null, resolvedLlmScreenshotSize); + this.unfiltered_actions = this.controller.registry.get_prompt_description(); + this.eventbus = new EventBus(this._buildEventBusName()); + this.enable_cloud_sync = CONFIG.BROWSER_USE_CLOUD_SYNC; + if (this.enable_cloud_sync || cloud_sync) { + this.cloud_sync = cloud_sync ?? null; + if (this.cloud_sync) { + this.eventbus.on('*', this.cloud_sync.handle_event?.bind(this.cloud_sync) ?? (() => { })); + } + } + this._external_pause_event = { + resolve: null, + promise: Promise.resolve(), + }; + this._session_start_time = 0; + this._task_start_time = 0; + this._force_exit_telemetry_logged = false; + // Security validation for sensitive_data and allowed_domains + this._validateSecuritySettings(); + this._capture_shared_pinned_tab(); + // LLM verification and setup + this._verifyAndSetupLlm(); + // Model-specific vision handling + this._handleModelSpecificVision(); + } + _normalizeMessageCompactionSetting(messageCompaction) { + if (messageCompaction == null) { + return null; + } + if (typeof messageCompaction === 'boolean') { + return normalizeMessageCompactionSettings({ + ...defaultMessageCompactionSettings(), + enabled: messageCompaction, + }); + } + return normalizeMessageCompactionSettings({ + ...defaultMessageCompactionSettings(), + ...messageCompaction, + }); + } + _createSessionIdWithAgentSuffix() { + const suffix = this.id.slice(-4); + const generated = uuid7str(); + return `${generated.slice(0, -4)}${suffix}`; + } + _buildEventBusName() { + let agentIdSuffix = String(this.id).slice(-4).replace(/-/g, '_'); + if (agentIdSuffix && /^\d/.test(agentIdSuffix)) { + agentIdSuffix = `a${agentIdSuffix}`; + } + return `Agent_${agentIdSuffix}`; + } + _copyBrowserProfile(profile) { + const source = profile ?? DEFAULT_BROWSER_PROFILE; + const clonedConfig = typeof structuredClone === 'function' + ? structuredClone(source.config) + : JSON.parse(JSON.stringify(source.config)); + return new BrowserProfile(clonedConfig); + } + _getBrowserContextFromPage(page, browser_context) { + if (!page) { + return browser_context; + } + const contextAttr = page.context; + if (typeof contextAttr === 'function') { + try { + const resolved = contextAttr.call(page); + return resolved ?? browser_context; + } + catch { + return browser_context; + } + } + return contextAttr ?? browser_context; + } + _claim_or_isolate_browser_session(browser_session) { + const claimMode = this.settings.session_attachment_mode === 'shared' + ? 'shared' + : 'exclusive'; + this._hasBrowserSessionClaim = false; + const claimSession = (session) => { + const claimFn = session.claim_agent ?? session.claimAgent; + if (typeof claimFn !== 'function') { + if (this.settings.session_attachment_mode === 'strict' || + this.settings.session_attachment_mode === 'shared') { + throw new Error(`session_attachment_mode='${this.settings.session_attachment_mode}' requires BrowserSession.claim_agent()/release_agent() support.`); + } + return 'noop'; + } + const claimed = Boolean(claimFn.call(session, this.id, claimMode)); + return claimed ? 'claimed' : 'failed'; + }; + const getAttachedAgentIds = (session) => { + const pluralGetter = session.get_attached_agent_ids ?? + session.getAttachedAgentIds; + if (typeof pluralGetter === 'function') { + const value = pluralGetter.call(session); + if (Array.isArray(value)) { + return value.filter((item) => typeof item === 'string'); + } + } + const singleGetter = session.get_attached_agent_id ?? + session.getAttachedAgentId; + if (typeof singleGetter !== 'function') { + return []; + } + const value = singleGetter.call(session); + return typeof value === 'string' ? [value] : []; + }; + const claimResult = claimSession(browser_session); + if (claimResult !== 'failed') { + this._hasBrowserSessionClaim = claimResult === 'claimed'; + return browser_session; + } + const currentOwners = getAttachedAgentIds(browser_session); + const ownerLabel = currentOwners.length > 0 ? currentOwners.join(', ') : 'unknown'; + if (this.settings.session_attachment_mode === 'strict') { + throw new Error(`BrowserSession is already attached to Agent ${ownerLabel}. Set session_attachment_mode='copy' to allow automatic isolation.`); + } + if (this.settings.session_attachment_mode === 'shared') { + throw new Error(`BrowserSession is already attached in exclusive mode by Agent ${ownerLabel}. Configure all participating agents with session_attachment_mode='shared' or use session_attachment_mode='copy'.`); + } + this.logger.warning(`⚠️ BrowserSession is already attached to Agent ${ownerLabel}. Creating an isolated copy for this Agent.`); + const modelCopyFn = browser_session.model_copy ?? browser_session.modelCopy; + if (typeof modelCopyFn !== 'function') { + throw new Error(`BrowserSession is attached to another Agent (${ownerLabel}) and cannot be safely reused. Provide a separate BrowserSession.`); + } + const isolated = modelCopyFn.call(browser_session); + const isolatedClaimResult = claimSession(isolated); + if (isolatedClaimResult === 'failed') { + throw new Error('Failed to claim isolated BrowserSession for current Agent'); + } + this._hasBrowserSessionClaim = isolatedClaimResult === 'claimed'; + return isolated; + } + _release_browser_session_claim(browser_session) { + if (!browser_session || !this._hasBrowserSessionClaim) { + return; + } + const releaseFn = browser_session.release_agent ?? + browser_session.releaseAgent; + if (typeof releaseFn !== 'function') { + return; + } + const released = releaseFn.call(browser_session, this.id); + if (!released) { + this.logger.warning('⚠️ BrowserSession claim was not released because it is currently attached to another Agent.'); + } + this._hasBrowserSessionClaim = false; + } + _has_any_browser_session_attachments(browser_session) { + if (!browser_session) { + return false; + } + const pluralGetter = browser_session.get_attached_agent_ids ?? + browser_session.getAttachedAgentIds; + if (typeof pluralGetter === 'function') { + const value = pluralGetter.call(browser_session); + if (Array.isArray(value)) { + return value.some((item) => typeof item === 'string'); + } + } + const singleGetter = browser_session.get_attached_agent_id ?? + browser_session.getAttachedAgentId; + if (typeof singleGetter !== 'function') { + return false; + } + return typeof singleGetter.call(browser_session) === 'string'; + } + _is_shared_session_mode() { + return this.settings.session_attachment_mode === 'shared'; + } + _capture_shared_pinned_tab() { + if (!this._is_shared_session_mode() || !this.browser_session) { + return; + } + const activeTab = this.browser_session.active_tab; + const pageId = activeTab?.page_id; + if (typeof pageId === 'number') { + this._sharedPinnedTabId = pageId; + } + } + async _restore_shared_pinned_tab_if_needed() { + if (!this._is_shared_session_mode() || !this.browser_session) { + return; + } + const switchFn = this.browser_session.switch_to_tab ?? + this.browser_session.switchToTab; + if (typeof switchFn !== 'function') { + return; + } + if (this._sharedPinnedTabId == null) { + this._capture_shared_pinned_tab(); + return; + } + try { + await switchFn.call(this.browser_session, this._sharedPinnedTabId); + } + catch { + this._capture_shared_pinned_tab(); + } + } + async _run_with_shared_session_step_lock(callback) { + if (!this._is_shared_session_mode() || !this.browser_session) { + return callback(); + } + const sessionId = this.browser_session.id; + let lock = Agent._sharedSessionStepLocks.get(sessionId); + if (!lock) { + lock = new AsyncMutex(); + Agent._sharedSessionStepLocks.set(sessionId, lock); + } + const release = await lock.acquire(); + try { + return await callback(); + } + finally { + release(); + } + } + _cleanup_shared_session_step_lock_if_unused(browser_session) { + if (!browser_session) { + return; + } + if (this._has_any_browser_session_attachments(browser_session)) { + return; + } + Agent._sharedSessionStepLocks.delete(browser_session.id); + } + _init_browser_session(init) { + let { page, browser, browser_context, browser_profile, browser_session } = init; + if (browser instanceof BrowserSession) { + browser_session = browser_session ?? browser; + browser = null; + } + if (browser_session) { + const ownsResources = browser_session._owns_browser_resources; + if (ownsResources === false && + this.settings.session_attachment_mode === 'copy') { + this.logger.warning("⚠️ Non-owning BrowserSession detected. session_attachment_mode='copy' will isolate this Agent with a cloned BrowserSession."); + const modelCopyFn = browser_session.model_copy ?? + browser_session.modelCopy; + if (typeof modelCopyFn === 'function') { + const isolated = modelCopyFn.call(browser_session); + return this._claim_or_isolate_browser_session(isolated); + } + } + return this._claim_or_isolate_browser_session(browser_session); + } + const resolvedContext = this._getBrowserContextFromPage(page, browser_context); + const resolvedProfile = this._copyBrowserProfile(browser_profile); + return this._claim_or_isolate_browser_session(new BrowserSession({ + browser_profile: resolvedProfile, + browser: browser ?? null, + browser_context: resolvedContext, + page, + id: this._createSessionIdWithAgentSuffix(), + })); + } + /** + * Convert dictionary-based actions to ActionModel instances + */ + _convertInitialActions(actions) { + const convertedActions = []; + for (const actionDict of actions) { + // Each actionDict should have a single key-value pair + const actionName = Object.keys(actionDict)[0]; + const params = actionDict[actionName]; + try { + // Get the parameter model for this action from registry + const actionInfo = this.controller.registry.get_all_actions().get(actionName) ?? null; + if (!actionInfo) { + this.logger.warning(`⚠️ Unknown action "${actionName}" in initial_actions, skipping`); + continue; + } + const paramModel = actionInfo.paramSchema; + if (!paramModel) { + this.logger.warning(`⚠️ No parameter model for action "${actionName}", using raw params`); + convertedActions.push(actionDict); + continue; + } + // Validate parameters using Zod schema + const validatedParams = paramModel.parse(params); + if (!validatedParams || + typeof validatedParams !== 'object' || + Array.isArray(validatedParams)) { + this.logger.warning(`⚠️ Parsed params for action "${actionName}" are not an object, skipping`); + continue; + } + // Create action with validated parameters + convertedActions.push({ + [actionName]: validatedParams, + }); + } + catch (error) { + this.logger.error(`❌ Failed to validate initial action "${actionName}": ${error}`); + // Skip invalid actions + continue; + } + } + return convertedActions; + } + /** + * Handle model-specific vision capabilities + * Some models like DeepSeek and Grok don't support vision yet + */ + _handleModelSpecificVision() { + const modelName = this.llm.model?.toLowerCase() || ''; + // Handle DeepSeek models + if (modelName.includes('deepseek') && this.settings.use_vision) { + this.logger.warning('⚠️ DeepSeek models do not support use_vision=True yet. Setting use_vision=False for now...'); + this.settings.use_vision = false; + } + // Handle XAI models that currently do not support vision + if ((modelName.includes('grok-3') || modelName.includes('grok-code')) && + this.settings.use_vision) { + this.logger.warning('⚠️ This XAI model does not support use_vision=True yet. Setting use_vision=False for now...'); + this.settings.use_vision = false; + } + } + /** + * Verify that the LLM API keys are setup and the LLM API is responding properly. + * Also handles model capability detection. + */ + _verifyAndSetupLlm() { + // Skip verification if already done or if configured to skip + if (this.llm._verified_api_keys === true || + CONFIG.SKIP_LLM_API_KEY_VERIFICATION) { + this.llm._verified_api_keys = true; + return true; + } + // Mark as verified + this.llm._verified_api_keys = true; + // Log LLM information + this.logger.debug(`🤖 Using LLM: ${this.llm.model || 'unknown model'}`); + return true; + } + /** + * Validates security settings when sensitive_data is provided + * Checks if allowed_domains is properly configured to prevent credential leakage + */ + _validateSecuritySettings() { + if (!this.sensitive_data) { + return; + } + // Check if sensitive_data has domain-specific credentials + const hasDomainSpecificCredentials = Object.values(this.sensitive_data).some((value) => typeof value === 'object' && value !== null); + const allowedDomainsConfig = this.browser_session?.browser_profile?.config?.allowed_domains; + const hasAllowedDomains = Array.isArray(allowedDomainsConfig) + ? allowedDomainsConfig.length > 0 + : Boolean(allowedDomainsConfig); + // If no allowed_domains are configured, show a security warning + if (!hasAllowedDomains) { + this.logger.warning('⚠️ Agent(sensitive_data=••••••••) was provided but Browser(allowed_domains=[...]) is not locked down! ⚠️\n' + + ' ☠️ If the agent visits a malicious website and encounters a prompt-injection attack, your sensitive_data may be exposed!\n\n' + + ' \n'); + } + // If we're using domain-specific credentials, validate domain patterns + else if (hasDomainSpecificCredentials) { + const allowedDomains = this.browser_session.browser_profile.config.allowed_domains; + // Get domain patterns from sensitive_data where value is an object + const domainPatterns = Object.keys(this.sensitive_data).filter((key) => typeof this.sensitive_data[key] === 'object' && + this.sensitive_data[key] !== null); + // Validate each domain pattern against allowed_domains + for (const domainPattern of domainPatterns) { + let isAllowed = false; + for (const allowedDomain of allowedDomains) { + // Special cases that don't require URL matching + if (domainPattern === allowedDomain || allowedDomain === '*') { + isAllowed = true; + break; + } + // Extract the domain parts, ignoring scheme + const patternDomain = domainPattern.includes('://') + ? domainPattern.split('://')[1] + : domainPattern; + const allowedDomainPart = allowedDomain.includes('://') + ? allowedDomain.split('://')[1] + : allowedDomain; + // Check if pattern is covered by an allowed domain + // Example: "google.com" is covered by "*.google.com" + if (patternDomain === allowedDomainPart || + (allowedDomainPart.startsWith('*.') && + (patternDomain === allowedDomainPart.slice(2) || + patternDomain.endsWith('.' + allowedDomainPart.slice(2))))) { + isAllowed = true; + break; + } + } + if (!isAllowed) { + this.logger.warning(`⚠️ Domain pattern "${domainPattern}" in sensitive_data is not covered by any pattern in allowed_domains=${JSON.stringify(allowedDomains)}\n` + + ` This may be a security risk as credentials could be used on unintended domains.`); + } + } + } + } + _initFileSystem(file_system_path) { + if (this.state.file_system_state && file_system_path) { + throw new Error('Cannot provide both file_system_state (from agent state) and file_system_path. Restore from state or create new file system, not both.'); + } + if (this.state.file_system_state) { + try { + this.file_system = AgentFileSystem.from_state_sync(this.state.file_system_state); + this._file_system_path = this.state.file_system_state.base_dir; + this.logger.debug(`💾 File system restored from state to: ${this._file_system_path}`); + const timestamp = Date.now(); + this.agent_directory = path.join(os.tmpdir(), `browser_use_agent_${this.id}_${timestamp}`); + ensureDir(this.agent_directory); + return this.file_system; + } + catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.logger.error(`💾 Failed to restore file system from state: ${message}`); + throw error; + } + } + const timestamp = Date.now(); + this.agent_directory = path.join(os.tmpdir(), `browser_use_agent_${this.id}_${timestamp}`); + ensureDir(this.agent_directory); + const baseDir = file_system_path ?? this.agent_directory; + ensureDir(baseDir); + try { + this.file_system = new AgentFileSystem(baseDir); + this._file_system_path = baseDir; + } + catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.logger.error(`💾 Failed to initialize file system: ${message}`); + throw error; + } + this.state.file_system_state = this.file_system.get_state(); + this.logger.debug(`💾 File system path: ${this._file_system_path}`); + return this.file_system; + } + _setScreenshotService() { + try { + this.screenshot_service = new ScreenshotService(this.agent_directory); + this.logger.debug(`📸 Screenshot service initialized in: ${path.join(this.agent_directory, 'screenshots')}`); + } + catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.logger.error(`📸 Failed to initialize screenshot service: ${message}`); + throw error; + } + } + get logger() { + if (!this._logger) { + const taskIdSuffix = typeof this.task_id === 'string' && this.task_id.length + ? this.task_id.slice(-4) + : '----'; + const browserSessionSuffix = typeof this.browser_session?.id === 'string' + ? this.browser_session.id.slice(-4) + : typeof this.id === 'string' + ? this.id.slice(-4) + : '----'; + const focusTargetSuffixRaw = this.browser_session + ?.agent_focus_target_id; + const focusTargetSuffix = typeof focusTargetSuffixRaw === 'string' && focusTargetSuffixRaw.length + ? focusTargetSuffixRaw.slice(-2) + : '--'; + this._logger = createLogger(`browser_use.Agent🅰 ${taskIdSuffix} ⇢ 🅑 ${browserSessionSuffix} 🅣 ${focusTargetSuffix}`); + } + return this._logger; + } + get message_manager() { + return this._message_manager; + } + /** + * Get the browser instance from the browser session + */ + get browser() { + if (!this.browser_session) { + throw new Error('BrowserSession is not set up'); + } + if (!this.browser_session.browser) { + throw new Error('Browser is not set up'); + } + return this.browser_session.browser; + } + /** + * Get the browser context from the browser session + */ + get browserContext() { + if (!this.browser_session) { + throw new Error('BrowserSession is not set up'); + } + if (!this.browser_session.browser_context) { + throw new Error('BrowserContext is not set up'); + } + return this.browser_session.browser_context; + } + /** + * Get the browser profile from the browser session + */ + get browserProfile() { + if (!this.browser_session) { + throw new Error('BrowserSession is not set up'); + } + return this.browser_session.browser_profile; + } + get is_using_fallback_llm() { + return this._using_fallback_llm; + } + get current_llm_model() { + return typeof this.llm?.model === 'string' ? this.llm.model : 'unknown'; + } + /** + * Add a new task to the agent, keeping the same task_id as tasks are continuous + */ + addNewTask(newTask) { + // Simply delegate to message manager - no need for new task_id or events + // The task continues with new instructions, it doesn't end and start a new one + this.task = newTask; + this._message_manager.add_new_task(newTask); + this.state.follow_up_task = true; + this.state.stopped = false; + this.state.paused = false; + this.eventbus = new EventBus(this._buildEventBusName()); + } + _enhanceTaskWithSchema(task, outputModelSchema) { + if (!outputModelSchema) { + return task; + } + try { + const schemaPayload = this._getOutputModelSchemaPayload(outputModelSchema); + if (schemaPayload == null) { + return task; + } + const schemaJson = JSON.stringify(schemaPayload, null, 2); + if (!schemaJson) { + return task; + } + const schemaName = typeof outputModelSchema?.name === 'string' + ? outputModelSchema.name + : 'StructuredOutput'; + return `${task}\nExpected output format: ${schemaName}\n${schemaJson}`; + } + catch (error) { + this.logger.debug(`Could not parse output schema for task enhancement: ${error instanceof Error ? error.message : String(error)}`); + return task; + } + } + _getOutputModelSchemaPayload(outputModelSchema) { + if (outputModelSchema instanceof z.ZodType) { + try { + const schema = z.toJSONSchema(outputModelSchema); + return schema && typeof schema === 'object' + ? schema + : null; + } + catch { + return null; + } + } + if (typeof outputModelSchema.model_json_schema === 'function') { + const schema = outputModelSchema.model_json_schema(); + return schema && typeof schema === 'object' + ? schema + : null; + } + if (outputModelSchema.schema != null) { + const schemaCandidate = outputModelSchema.schema; + const schema = (() => { + if (schemaCandidate instanceof z.ZodType) { + return z.toJSONSchema(schemaCandidate); + } + if (typeof schemaCandidate?.toJSON === 'function') { + return schemaCandidate.toJSON(); + } + return schemaCandidate; + })(); + return schema && typeof schema === 'object' + ? schema + : null; + } + return null; + } + _getToolsOutputModelSchema(tools) { + const getOutputModel = tools?.get_output_model; + if (typeof getOutputModel !== 'function') { + return null; + } + const outputModel = getOutputModel.call(tools); + return outputModel == null + ? null + : outputModel; + } + _getOutputModelSchemaName(outputModelSchema) { + const explicitName = typeof outputModelSchema?.name === 'string' + ? outputModelSchema.name + : null; + if (explicitName && explicitName.trim().length > 0) { + return explicitName; + } + const ctorName = outputModelSchema?.constructor?.name; + return typeof ctorName === 'string' && ctorName.trim().length > 0 + ? ctorName + : 'StructuredOutput'; + } + _resolveStructuredOutputActionSchema(outputModelSchema) { + if (!outputModelSchema) { + return null; + } + if (outputModelSchema instanceof z.ZodType) { + return outputModelSchema; + } + const schemaCandidate = outputModelSchema?.schema; + if (schemaCandidate instanceof z.ZodType) { + return schemaCandidate; + } + return null; + } + _extract_start_url(taskText) { + const taskWithoutEmails = taskText.replace(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g, ''); + const urlPatterns = [ + /https?:\/\/[^\s<>"']+/g, + /(?:www\.)?[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*\.[a-zA-Z]{2,}(?:\/[^\s<>"']*)?/g, + ]; + const excludedExtensions = new Set([ + 'pdf', + 'doc', + 'docx', + 'xls', + 'xlsx', + 'ppt', + 'pptx', + 'odt', + 'ods', + 'odp', + 'txt', + 'md', + 'csv', + 'json', + 'xml', + 'yaml', + 'yml', + 'zip', + 'rar', + '7z', + 'tar', + 'gz', + 'bz2', + 'xz', + 'jpg', + 'jpeg', + 'png', + 'gif', + 'bmp', + 'svg', + 'webp', + 'ico', + 'mp3', + 'mp4', + 'avi', + 'mkv', + 'mov', + 'wav', + 'flac', + 'ogg', + 'py', + 'js', + 'css', + 'java', + 'cpp', + 'bib', + 'bibtex', + 'tex', + 'latex', + 'cls', + 'sty', + 'exe', + 'msi', + 'dmg', + 'pkg', + 'deb', + 'rpm', + 'iso', + 'polynomial', + ]); + const excludedWords = ['never', 'dont', "don't", 'not']; + const foundUrls = []; + for (const pattern of urlPatterns) { + for (const match of taskWithoutEmails.matchAll(pattern)) { + const original = match[0]; + const startIndex = match.index ?? 0; + let url = original.replace(/[.,;:!?()[\]]+$/g, ''); + const lowerUrl = url.toLowerCase(); + let shouldExclude = false; + for (const ext of excludedExtensions) { + if (lowerUrl.includes(`.${ext}`)) { + shouldExclude = true; + break; + } + } + if (shouldExclude) { + this.logger.debug(`Excluding URL with file extension from auto-navigation: ${url}`); + continue; + } + const contextStart = Math.max(0, startIndex - 20); + const contextText = taskWithoutEmails + .slice(contextStart, startIndex) + .toLowerCase(); + if (excludedWords.some((word) => contextText.includes(word))) { + this.logger.debug(`Excluding URL with word in excluded words from auto-navigation: ${url} (context: "${contextText.trim()}")`); + continue; + } + if (!url.startsWith('http://') && !url.startsWith('https://')) { + url = `https://${url}`; + } + foundUrls.push(url); + } + } + const uniqueUrls = Array.from(new Set(foundUrls)); + if (uniqueUrls.length > 1) { + this.logger.debug(`Multiple URLs found (${foundUrls.length}), skipping directly_open_url to avoid ambiguity`); + return null; + } + return uniqueUrls.length === 1 ? uniqueUrls[0] : null; + } + /** + * Take a step and return whether the task is done and valid + * @returns Tuple of [is_done, is_valid] + */ + async takeStep(stepInfo) { + await this._step(stepInfo ?? null); + if (this.history.is_done()) { + await this._run_simple_judge(); + await this.log_completion(); + if (this.settings.use_judge) { + await this._judge_and_log(); + } + if (this.register_done_callback) { + await this.register_done_callback(this.history); + } + return [true, true]; + } + return [false, false]; + } + /** + * Remove think tags from text + */ + _removeThinkTags(text) { + const THINK_TAGS = /.*?<\/think>/gs; + const STRAY_CLOSE_TAG = /.*?<\/think>/gs; + // Step 1: Remove well-formed ... + text = text.replace(THINK_TAGS, ''); + // Step 2: If there's an unmatched closing tag , + // remove everything up to and including that. + text = text.replace(STRAY_CLOSE_TAG, ''); + return text.trim(); + } + /** + * Log a comprehensive summary of the next action(s) + */ + _logNextActionSummary(parsed) { + if (!parsed.action || parsed.action.length === 0) { + return; + } + const actionCount = parsed.action.length; + // Collect action details + const actionDetails = []; + let lastActionName = 'unknown'; + let lastParamStr = ''; + for (const action of parsed.action) { + const actionData = action.model_dump(); + const actionName = Object.keys(actionData)[0] || 'unknown'; + const actionParams = actionData[actionName] || {}; + // Format key parameters concisely + const paramSummary = []; + if (typeof actionParams === 'object' && actionParams !== null) { + for (const [key, value] of Object.entries(actionParams)) { + if (key === 'index') { + paramSummary.push(`#${value}`); + } + else if (key === 'text' && typeof value === 'string') { + const textPreview = value.length > 30 ? value.slice(0, 30) + '...' : value; + paramSummary.push(`text="${textPreview}"`); + } + else if (key === 'url') { + paramSummary.push(`url="${value}"`); + } + else if (key === 'success') { + paramSummary.push(`success=${value}`); + } + else if (typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean') { + const valStr = String(value); + const truncatedVal = valStr.length > 30 ? valStr.slice(0, 30) + '...' : valStr; + paramSummary.push(`${key}=${truncatedVal}`); + } + } + } + const paramStr = paramSummary.length > 0 ? `(${paramSummary.join(', ')})` : ''; + actionDetails.push(`${actionName}${paramStr}`); + lastActionName = actionName; + lastParamStr = paramStr; + } + // Create summary based on single vs multi-action + if (actionCount === 1) { + this.logger.info(`☝️ Decided next action: ${lastActionName}${lastParamStr}`); + } + else { + const summaryLines = [`✌️ Decided next ${actionCount} multi-actions:`]; + for (let i = 0; i < actionDetails.length; i++) { + summaryLines.push(` ${i + 1}. ${actionDetails[i]}`); + } + this.logger.info(summaryLines.join('\n')); + } + } + _set_browser_use_version_and_source(sourceOverride) { + const version = get_browser_use_version(); + let source = 'npm'; + try { + const projectRoot = process.cwd(); + const repoIndicators = ['.git', 'README.md', 'docs', 'examples']; + if (repoIndicators.every((indicator) => fs.existsSync(path.join(projectRoot, indicator)))) { + source = 'git'; + } + } + catch (error) { + this.logger.debug(`Error determining browser-use source: ${error.message}`); + source = 'unknown'; + } + if (sourceOverride) { + source = sourceOverride; + } + this.version = version; + this.source = source; + } + /** + * Setup dynamic action models from controller's registry + * Initially only include actions with no filters + */ + _setup_action_models() { + // Initially only include actions with no filters + this.ActionModel = this.controller.registry.create_action_model(); + // Create output model with the dynamic actions + if (this.settings.flash_mode) { + this.AgentOutput = AgentOutput.type_with_custom_actions_flash_mode(this.ActionModel); + } + else if (this.settings.use_thinking) { + this.AgentOutput = AgentOutput.type_with_custom_actions(this.ActionModel); + } + else { + this.AgentOutput = AgentOutput.type_with_custom_actions_no_thinking(this.ActionModel); + } + // Used to force the done action when max_steps is reached + this.DoneActionModel = this.controller.registry.create_action_model({ + include_actions: ['done'], + }); + if (this.settings.flash_mode) { + this.DoneAgentOutput = AgentOutput.type_with_custom_actions_flash_mode(this.DoneActionModel); + } + else if (this.settings.use_thinking) { + this.DoneAgentOutput = AgentOutput.type_with_custom_actions(this.DoneActionModel); + } + else { + this.DoneAgentOutput = AgentOutput.type_with_custom_actions_no_thinking(this.DoneActionModel); + } + } + async _register_skills_as_actions() { + if (!this.skill_service || this._skills_registered) { + return; + } + const skills = await this.skill_service.get_all_skills(); + if (!skills.length) { + this.logger.warning('No skills loaded from SkillService'); + return; + } + this.logger.info(`🔧 Registering ${skills.length} skill action(s)...`); + for (const skill of skills) { + const slug = get_skill_slug(skill, skills); + const paramSchema = build_skill_parameters_schema(skill.parameters, { + exclude_cookies: true, + }); + const description = `${skill.description} (Skill: "${skill.title}")`; + this.controller.registry.action(description, { + param_model: paramSchema, + action_name: slug, + })(async (params, { browser_session }) => { + if (!this.skill_service) { + return new ActionResult({ error: 'SkillService not initialized' }); + } + if (!browser_session || + typeof browser_session.get_cookies !== 'function') { + return new ActionResult({ + error: 'Skill execution requires an active BrowserSession.', + }); + } + try { + const cookiesRaw = await browser_session.get_cookies(); + const cookies = Array.isArray(cookiesRaw) + ? cookiesRaw + .map((cookie) => { + const record = cookie && typeof cookie === 'object' + ? cookie + : null; + const name = record && typeof record.name === 'string' + ? record.name + : null; + const value = record && typeof record.value === 'string' + ? record.value + : ''; + return name ? { name, value } : null; + }) + .filter((cookie) => cookie != null) + : []; + const result = await this.skill_service.execute_skill({ + skill_id: skill.id, + parameters: params ?? {}, + cookies, + }); + if (!result.success) { + return new ActionResult({ + error: result.error ?? 'Skill execution failed', + }); + } + const rendered = typeof result.result === 'string' + ? result.result + : JSON.stringify(result.result ?? {}); + return new ActionResult({ + extracted_content: rendered, + long_term_memory: rendered, + }); + } + catch (error) { + if (error instanceof MissingCookieException) { + return new ActionResult({ + error: `Missing cookies (${error.cookie_name}): ${error.cookie_description}`, + }); + } + const message = error instanceof Error + ? `${error.name}: ${error.message}` + : String(error); + return new ActionResult({ + error: `Skill execution error: ${message}`, + }); + } + }); + } + this._skills_registered = true; + this._setup_action_models(); + if (this.initial_actions?.length) { + const actionDicts = this.initial_actions.map((action) => typeof action?.model_dump === 'function' + ? action.model_dump({ exclude_unset: true }) + : action); + this.initial_actions = this._convertInitialActions(actionDicts); + } + this.logger.info(`✓ Registered ${skills.length} skill actions`); + } + async _get_unavailable_skills_info() { + if (!this.skill_service || !this.browser_session) { + return ''; + } + try { + const skills = await this.skill_service.get_all_skills(); + if (!skills.length) { + return ''; + } + const currentCookies = await this.browser_session.get_cookies(); + const cookieNames = new Set(); + if (Array.isArray(currentCookies)) { + for (const cookie of currentCookies) { + if (!cookie || typeof cookie !== 'object') { + continue; + } + const name = typeof cookie.name === 'string' + ? String(cookie.name) + : ''; + if (name) { + cookieNames.add(name); + } + } + } + const unavailableSkills = []; + for (const skill of skills) { + const cookieParams = skill.parameters.filter((param) => param.type === 'cookie'); + if (!cookieParams.length) { + continue; + } + const missingCookies = []; + for (const cookieParam of cookieParams) { + const isRequired = cookieParam.required !== false; + if (isRequired && !cookieNames.has(cookieParam.name)) { + missingCookies.push({ + name: cookieParam.name, + description: cookieParam.description || 'No description provided', + }); + } + } + if (missingCookies.length) { + unavailableSkills.push({ + id: skill.id, + title: skill.title, + description: skill.description, + missing_cookies: missingCookies, + }); + } + } + if (!unavailableSkills.length) { + return ''; + } + const lines = [ + 'Unavailable Skills (missing required cookies):', + ]; + for (const skillInfo of unavailableSkills) { + const skillObj = skills.find((entry) => entry.id === skillInfo.id); + const slug = skillObj + ? get_skill_slug(skillObj, skills) + : skillInfo.title; + lines.push(''); + lines.push(` • ${slug} ("${skillInfo.title}")`); + lines.push(` Description: ${skillInfo.description}`); + lines.push(' Missing cookies:'); + for (const cookie of skillInfo.missing_cookies) { + lines.push(` - ${cookie.name}: ${cookie.description}`); + } + } + return lines.join('\n'); + } + catch (error) { + this.logger.error(`Error getting unavailable skills info: ${error instanceof Error + ? `${error.name}: ${error.message}` + : String(error)}`); + return ''; + } + } + /** + * Update action models with page-specific actions + * Called during each step to filter actions based on current page context + */ + async _updateActionModelsForPage(page) { + // Create new action model with current page's filtered actions + this.ActionModel = this.controller.registry.create_action_model({ page }); + // Update output model with the new actions + if (this.settings.flash_mode) { + this.AgentOutput = AgentOutput.type_with_custom_actions_flash_mode(this.ActionModel); + } + else if (this.settings.use_thinking) { + this.AgentOutput = AgentOutput.type_with_custom_actions(this.ActionModel); + } + else { + this.AgentOutput = AgentOutput.type_with_custom_actions_no_thinking(this.ActionModel); + } + // Update done action model too + this.DoneActionModel = this.controller.registry.create_action_model({ + include_actions: ['done'], + page, + }); + if (this.settings.flash_mode) { + this.DoneAgentOutput = AgentOutput.type_with_custom_actions_flash_mode(this.DoneActionModel); + } + else if (this.settings.use_thinking) { + this.DoneAgentOutput = AgentOutput.type_with_custom_actions(this.DoneActionModel); + } + else { + this.DoneAgentOutput = AgentOutput.type_with_custom_actions_no_thinking(this.DoneActionModel); + } + } + async _execute_initial_actions() { + if (!this.initial_actions?.length || this.state.follow_up_task) { + return; + } + this.logger.debug(`⚡ Executing ${this.initial_actions.length} initial actions...`); + const result = await this.multi_act(this.initial_actions); + if (result.length > 0 && this.initial_url && result[0]?.long_term_memory) { + result[0].long_term_memory = `Found initial url and automatically loaded it. ${result[0].long_term_memory}`; + } + this.state.last_result = result; + const modelOutput = this.settings.flash_mode + ? new this.AgentOutput({ + evaluation_previous_goal: null, + memory: 'Initial navigation', + next_goal: null, + action: this.initial_actions, + }) + : new this.AgentOutput({ + evaluation_previous_goal: 'Start', + memory: null, + next_goal: 'Initial navigation', + action: this.initial_actions, + }); + const timestamp = Date.now() / 1000; + const metadata = new StepMetadata(timestamp, timestamp, 0, null); + const stateHistory = new BrowserStateHistory(this.initial_url ?? '', 'Initial Actions', [], Array(this.initial_actions.length).fill(null), null); + this.history.add_item(new AgentHistory(modelOutput, result, stateHistory, metadata, null)); + this.logger.debug('📝 Saved initial actions to history as step 0'); + this.logger.debug('✅ Initial actions completed'); + } + async run(max_steps = 500, on_step_start = null, on_step_end = null) { + let agent_run_error = null; + this._force_exit_telemetry_logged = false; + const signal_handler = new SignalHandler({ + pause_callback: this.pause.bind(this), + resume_callback: this.resume.bind(this), + custom_exit_callback: () => { + this._log_agent_event(max_steps, 'SIGINT: Cancelled by user'); + this.telemetry?.flush?.(); + this._force_exit_telemetry_logged = true; + }, + exit_on_second_int: true, + }); + signal_handler.register(); + try { + await this._log_agent_run(); + this.logger.debug(`🔧 Agent setup: Task ID ${this.task_id.slice(-4)}, Session ID ${this.session_id.slice(-4)}, Browser Session ID ${this.browser_session?.id?.slice?.(-4) ?? 'None'}`); + this._session_start_time = Date.now() / 1000; + this._task_start_time = this._session_start_time; + if (!this.state.session_initialized) { + this.logger.debug('📡 Dispatching CreateAgentSessionEvent...'); + this.eventbus.dispatch(CreateAgentSessionEvent.fromAgent(this)); + this.state.session_initialized = true; + } + this.logger.debug('📡 Dispatching CreateAgentTaskEvent...'); + this.eventbus.dispatch(CreateAgentTaskEvent.fromAgent(this)); + if (!this.state.stopped) { + await this.browser_session?.start(); + } + await this._register_skills_as_actions(); + try { + await this._execute_initial_actions(); + } + catch (error) { + if (error?.name !== 'InterruptedError') { + throw error; + } + } + this.logger.debug(`🔄 Starting main execution loop with max ${max_steps} steps (currently at step ${this.state.n_steps})...`); + while (this.state.n_steps <= max_steps) { + const currentStep = this.state.n_steps - 1; + if (this.state.paused) { + this.logger.debug(`⏸️ Step ${this.state.n_steps}: Agent paused, waiting to resume...`); + await this.wait_until_resumed(); + signal_handler.reset(); + } + if (this.state.consecutive_failures >= this._max_total_failures()) { + this.logger.error(`❌ Stopping due to ${this.settings.max_failures} consecutive failures`); + agent_run_error = `Stopped due to ${this.settings.max_failures} consecutive failures`; + break; + } + try { + await this._raise_if_stopped_or_paused(); + } + catch (error) { + if (error?.name === 'InterruptedError') { + if (this.state.paused) { + continue; + } + if (this.state.stopped) { + this.logger.info('🛑 Agent stopped'); + agent_run_error = 'Agent stopped programmatically'; + } + else { + agent_run_error = 'Agent stopped due to external request'; + } + break; + } + throw error; + } + if (on_step_start) { + await on_step_start(this); + } + this.logger.debug(`🚶 Starting step ${currentStep + 1}/${max_steps}...`); + const step_info = new AgentStepInfo(currentStep, max_steps); + const stepAbortController = new AbortController(); + try { + await this._executeWithTimeout(this._step(step_info, stepAbortController.signal), this.settings.step_timeout ?? 0, () => stepAbortController.abort()); + this.logger.debug(`✅ Completed step ${currentStep + 1}/${max_steps}`); + } + catch (error) { + const message = error instanceof Error ? error.message : String(error); + const isTimeout = error instanceof ExecutionTimeoutError; + if (isTimeout) { + const timeoutMessage = `Step ${currentStep + 1} timed out after ${this.settings.step_timeout} seconds`; + this.logger.error(`⏰ ${timeoutMessage}`); + this.state.consecutive_failures += 1; + this.state.last_result = [ + new ActionResult({ error: timeoutMessage }), + ]; + // JavaScript promises are not force-cancelable; stop the run loop + // immediately to avoid overlapping timed-out steps with new steps. + this.stop(); + agent_run_error = timeoutMessage; + break; + } + this.logger.error(`❌ Unhandled step error at step ${currentStep + 1}: ${message}`); + this.state.consecutive_failures += 1; + this.state.last_result = [ + new ActionResult({ + error: message || `Unhandled step error at step ${currentStep + 1}`, + }), + ]; + } + if (on_step_end) { + await on_step_end(this); + } + if (this.history.is_done()) { + this.logger.debug(`🎯 Task completed after ${currentStep + 1} steps!`); + await this._run_simple_judge(); + await this.log_completion(); + if (this.settings.use_judge) { + await this._judge_and_log(); + } + if (this.register_done_callback) { + const maybePromise = this.register_done_callback(this.history); + if (maybePromise && + typeof maybePromise.then === 'function') { + await maybePromise; + } + } + break; + } + } + if (this.state.n_steps > max_steps && + !this.history.is_done() && + !agent_run_error) { + agent_run_error = 'Failed to complete task in maximum steps'; + this.history.add_item(new AgentHistory(null, [ + new ActionResult({ + error: agent_run_error, + include_in_memory: true, + }), + ], new BrowserStateHistory('', '', [], [], null), null)); + this.logger.info(`❌ ${agent_run_error}`); + } + this.logger.debug('📊 Collecting usage summary...'); + this.history.usage = + (await this.token_cost_service.get_usage_summary()); + if (!this.history._output_model_schema && this.output_model_schema) { + this.history._output_model_schema = this.output_model_schema; + } + this.logger.debug('🏁 Agent.run() completed successfully'); + return this.history; + } + catch (error) { + agent_run_error = error instanceof Error ? error.message : String(error); + this.logger.error(`Agent run failed with exception: ${agent_run_error}`); + throw error; + } + finally { + await this.token_cost_service.log_usage_summary(); + signal_handler.unregister(); + if (!this._force_exit_telemetry_logged) { + try { + this._log_agent_event(max_steps, agent_run_error); + } + catch (logError) { + this.logger.error(`Failed to log telemetry event: ${String(logError)}`); + } + finally { + try { + this.telemetry?.flush?.(); + } + catch (flushError) { + this.logger.error(`Failed to flush telemetry client: ${String(flushError)}`); + } + } + } + else { + this.logger.info('Telemetry for force exit (SIGINT) already logged.'); + } + this.eventbus.dispatch(UpdateAgentTaskEvent.fromAgent(this)); + if (this.settings.generate_gif) { + let output_path = 'agent_history.gif'; + if (typeof this.settings.generate_gif === 'string') { + output_path = this.settings.generate_gif; + } + await create_history_gif(this.task, this.history, { output_path }); + if (fs.existsSync(output_path)) { + const output_event = await CreateAgentOutputFileEvent.fromAgentAndFile(this, output_path); + this.eventbus.dispatch(output_event); + } + } + await this.eventbus.stop(); + await this.close(); + } + } + async _executeWithTimeout(promise, timeoutSeconds, onTimeout) { + if (!timeoutSeconds || timeoutSeconds <= 0) { + return promise; + } + let timeoutHandle = null; + const timeoutPromise = new Promise((_, reject) => { + timeoutHandle = setTimeout(() => { + try { + onTimeout?.(); + } + catch { + // Ignore timeout callback errors and preserve timeout semantics. + } + reject(new ExecutionTimeoutError()); + }, timeoutSeconds * 1000); + }); + try { + return await Promise.race([promise, timeoutPromise]); + } + finally { + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + } + } + async _step(step_info = null, signal = null) { + await this._run_with_shared_session_step_lock(async () => { + this._throwIfAborted(signal); + this.step_start_time = Date.now() / 1000; + let browser_state_summary = null; + try { + browser_state_summary = await this._prepare_context(step_info, signal); + this._throwIfAborted(signal); + await this._get_next_action(browser_state_summary, signal); + this._throwIfAborted(signal); + await this._execute_actions(signal); + await this._post_process(); + } + catch (error) { + if (signal?.aborted) { + const message = error instanceof Error ? error.message : String(error); + this.logger.debug(`Step aborted before completion: ${message}`); + } + else { + await this._handle_step_error(error); + } + } + finally { + await this._finalize(browser_state_summary); + } + }); + } + async _prepare_context(step_info = null, signal = null) { + if (!this.browser_session) { + throw new Error('BrowserSession is not set up'); + } + this._throwIfAborted(signal); + await this._restore_shared_pinned_tab_if_needed(); + this._throwIfAborted(signal); + await this.browser_session.wait_if_captcha_solving?.(); + this._throwIfAborted(signal); + this._log_first_step_startup(); + this.logger.debug(`🌐 Step ${this.state.n_steps}: Getting browser state...`); + const browser_state_summary = await this.browser_session.get_browser_state_with_recovery?.({ + cache_clickable_elements_hashes: true, + include_screenshot: true, + include_recent_events: this.settings.include_recent_events, + signal, + }); + this._throwIfAborted(signal); + const current_page = await this.browser_session.get_current_page?.(); + await this._check_and_update_downloads(`Step ${this.state.n_steps}: after getting browser state`); + this._log_step_context(current_page, browser_state_summary); + await this._storeScreenshotForStep(browser_state_summary); + await this._raise_if_stopped_or_paused(); + this.logger.debug(`📝 Step ${this.state.n_steps}: Updating action models...`); + this._throwIfAborted(signal); + await this._updateActionModelsForPage(current_page); + const page_filtered_actions = this.controller.registry.get_prompt_description(current_page); + let unavailable_skills_info = null; + if (this.skill_service) { + unavailable_skills_info = await this._get_unavailable_skills_info(); + } + this.logger.debug(`💬 Step ${this.state.n_steps}: Creating state messages for context...`); + this._message_manager.prepare_step_state(browser_state_summary, this.state.last_model_output, this.state.last_result, step_info, this.sensitive_data ?? null); + await this._maybe_compact_messages(step_info); + this._message_manager.create_state_messages(browser_state_summary, this.state.last_model_output, this.state.last_result, step_info, this.settings.use_vision, page_filtered_actions || null, this.sensitive_data ?? null, this.available_file_paths, this.settings.include_recent_events, this._render_plan_description(), unavailable_skills_info, true); + this._inject_budget_warning(step_info); + this._inject_replan_nudge(); + this._inject_exploration_nudge(); + this._update_loop_detector_page_state(browser_state_summary); + this._inject_loop_detection_nudge(); + await this._handle_final_step(step_info); + await this._handle_failure_limit_recovery(); + return browser_state_summary; + } + async _maybe_compact_messages(step_info = null) { + const settings = this.settings.message_compaction; + if (!settings || !settings.enabled) { + return; + } + const compactionLlm = settings.compaction_llm ?? + this.settings.page_extraction_llm ?? + this.llm; + await this._message_manager.maybe_compact_messages(compactionLlm, settings, step_info); + } + async _storeScreenshotForStep(browser_state_summary) { + this._current_screenshot_path = null; + if (!this.screenshot_service || !browser_state_summary?.screenshot) { + return; + } + try { + this._current_screenshot_path = + await this.screenshot_service.store_screenshot(browser_state_summary.screenshot, this.state.n_steps); + this.logger.debug(`📸 Step ${this.state.n_steps}: Stored screenshot at ${this._current_screenshot_path}`); + } + catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.logger.error(`📸 Failed to store screenshot for step ${this.state.n_steps}: ${message}`); + this._current_screenshot_path = null; + } + } + async _get_next_action(browser_state_summary, signal = null) { + this._throwIfAborted(signal); + const input_messages = this._message_manager.get_messages(); + this.logger.debug(`🤖 Step ${this.state.n_steps}: Calling LLM with ${input_messages.length} messages (model: ${this.llm.model})...`); + let model_output; + const llmAbortController = new AbortController(); + const removeAbortRelay = this._relayAbortSignal(signal, llmAbortController); + try { + model_output = await this._executeWithTimeout(this._get_model_output_with_retry(input_messages, llmAbortController.signal), this.settings.llm_timeout, () => llmAbortController.abort()); + } + catch (error) { + if (error instanceof ExecutionTimeoutError) { + throw new Error(`LLM call timed out after ${this.settings.llm_timeout} seconds. Keep your thinking and output short.`, { cause: error }); + } + throw error; + } + finally { + removeAbortRelay(); + } + this._throwIfAborted(signal); + this.state.last_model_output = model_output; + let actions = []; + if (model_output) { + this._logNextActionSummary(model_output); + actions = model_output.action.map((a) => a.model_dump()); + } + await this._raise_if_stopped_or_paused(); + await this._handle_post_llm_processing(browser_state_summary, input_messages, actions); + await this._raise_if_stopped_or_paused(); + } + async _execute_actions(signal = null) { + if (!this.state.last_model_output) { + throw new Error('No model output to execute actions from'); + } + this.logger.debug(`⚡ Step ${this.state.n_steps}: Executing ${this.state.last_model_output.action.length} actions...`); + const result = await this.multi_act(this.state.last_model_output.action.map((a) => a.model_dump()), { signal }); + this.logger.debug(`✅ Step ${this.state.n_steps}: Actions completed`); + this.state.last_result = result; + } + async _post_process() { + if (!this.browser_session) { + throw new Error('BrowserSession is not set up'); + } + await this._check_and_update_downloads('after executing actions'); + if (this.state.last_model_output) { + this._update_plan_from_model_output(this.state.last_model_output); + } + this._update_loop_detector_actions(); + const lastResult = this.state.last_result; + if (lastResult && lastResult.length === 1 && lastResult[0]?.error) { + this.state.consecutive_failures += 1; + this.logger.debug(`🔄 Step ${this.state.n_steps}: Consecutive failures: ${this.state.consecutive_failures}`); + return; + } + if (this.state.consecutive_failures > 0) { + this.state.consecutive_failures = 0; + this.logger.debug(`🔄 Step ${this.state.n_steps}: Consecutive failures reset to: ${this.state.consecutive_failures}`); + } + if (lastResult && + lastResult.length > 0 && + lastResult[lastResult.length - 1]?.is_done) { + const finalResult = lastResult[lastResult.length - 1]; + const success = Boolean(finalResult.success); + const renderedContent = typeof finalResult.extracted_content === 'string' + ? finalResult.extracted_content + : String(finalResult.extracted_content ?? ''); + if (success) { + this.logger.info(`\n📄 \x1b[32m Final Result:\x1b[0m \n${renderedContent}\n\n`); + } + else { + this.logger.info(`\n📄 \x1b[31m Final Result:\x1b[0m \n${renderedContent}\n\n`); + } + const attachments = Array.isArray(finalResult.attachments) + ? finalResult.attachments + : []; + const totalAttachments = attachments.length; + for (let i = 0; i < attachments.length; i++) { + const suffix = totalAttachments > 1 ? String(i + 1) : ''; + this.logger.info(`👉 Attachment${suffix ? ` ${suffix}` : ''}: ${attachments[i]}`); + } + } + } + async multi_act(actions, options = {}) { + const { signal = null } = options; + const results = []; + if (!this.browser_session) { + throw new Error('BrowserSession is not set up'); + } + await this._restore_shared_pinned_tab_if_needed(); + // ==================== Execute Actions ==================== + for (let i = 0; i < actions.length; i++) { + this._throwIfAborted(signal); + const action = actions[i]; + const actionName = Object.keys(action)[0]; + const actionParams = action[actionName]; + // ==================== Done Action Position Validation ==================== + // ONLY ALLOW TO CALL `done` IF IT IS A SINGLE ACTION + if (i > 0 && actionName === 'done') { + const msg = `Done action is allowed only as a single action - stopped after action ${i} / ${actions.length}.`; + this.logger.info(msg); + break; + } + // ==================== Wait Between Actions ==================== + if (i > 0) { + // Wait between actions + const wait_time = this.browser_session?.browser_profile + ?.wait_between_actions || 0; + if (wait_time > 0) { + await this._sleep(wait_time, signal); + } + } + // ==================== Execute Action ==================== + try { + this._throwIfAborted(signal); + await this._raise_if_stopped_or_paused(); + const preActionPage = await this.browser_session.get_current_page?.(); + const preActionUrl = typeof preActionPage?.url === 'function' ? preActionPage.url() : ''; + const preActionFocusTargetId = this.browser_session.agent_focus_target_id ?? + this.browser_session.active_tab?.page_id ?? + null; + const actResult = await this.controller.registry.execute_action(actionName, actionParams, { + browser_session: this.browser_session, + page_extraction_llm: this.settings.page_extraction_llm, + extraction_schema: this.extraction_schema, + sensitive_data: this.sensitive_data, + available_file_paths: this.available_file_paths, + file_system: this.file_system, + context: this.context ?? undefined, + signal, + }); + results.push(actResult); + // Log action execution + this.logger.info(`☑️ Executed action ${i + 1}/${actions.length}: ${actionName}(${JSON.stringify(actionParams)})`); + // Break early if done, error, or last action + if (results[results.length - 1]?.is_done || + results[results.length - 1]?.error || + i === actions.length - 1) { + this._capture_shared_pinned_tab(); + break; + } + const registeredAction = this.controller.registry.get_action?.(actionName); + const terminatesSequence = Boolean(registeredAction?.terminates_sequence); + if (terminatesSequence) { + this.logger.info(`Action "${actionName}" terminates sequence - skipping ${actions.length - i - 1} remaining action(s)`); + this._capture_shared_pinned_tab(); + break; + } + const postActionPage = await this.browser_session.get_current_page?.(); + const postActionUrl = typeof postActionPage?.url === 'function' ? postActionPage.url() : ''; + const postActionFocusTargetId = this.browser_session.agent_focus_target_id ?? + this.browser_session.active_tab?.page_id ?? + null; + if (postActionUrl !== preActionUrl || + postActionFocusTargetId !== preActionFocusTargetId) { + this.logger.info(`Page changed after "${actionName}" - skipping ${actions.length - i - 1} remaining action(s)`); + this._capture_shared_pinned_tab(); + break; + } + this._capture_shared_pinned_tab(); + } + catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.logger.error(`❌ Action ${i + 1} failed: ${message}`); + this._capture_shared_pinned_tab(); + throw error; + } + } + return results; + } + async _generate_rerun_summary(originalTask, results, summaryLlm = null, signal = null) { + if (!this.browser_session) { + return new ActionResult({ + is_done: true, + success: false, + extracted_content: 'Rerun completed without an active browser session.', + long_term_memory: 'Rerun completed without an active browser session.', + }); + } + let screenshotB64 = null; + try { + screenshotB64 = await this.browser_session.take_screenshot(false); + } + catch (error) { + this.logger.warning(`Failed to capture screenshot for rerun summary: ${error instanceof Error ? error.message : String(error)}`); + } + const errorCount = results.filter((result) => Boolean(result.error)).length; + const successCount = results.length - errorCount; + const prompt = get_rerun_summary_prompt(originalTask, results.length, successCount, errorCount); + const message = get_rerun_summary_message(prompt, screenshotB64); + const llm = summaryLlm ?? this.llm; + const parser = { + parse: (input) => z + .object({ + summary: z.string(), + success: z.boolean(), + completion_status: z.enum(['complete', 'partial', 'failed']), + }) + .parse(JSON.parse(input)), + }; + try { + const response = await llm.ainvoke([message], parser, { + signal: signal ?? undefined, + }); + const summary = response.completion; + if (!summary || + typeof summary !== 'object' || + typeof summary.summary !== 'string' || + typeof summary.success !== 'boolean' || + !['complete', 'partial', 'failed'].includes(String(summary.completion_status))) { + throw new Error('Structured rerun summary response did not match expected schema'); + } + this.logger.info(`Rerun Summary: ${summary.summary}`); + this.logger.info(`Rerun Status: ${summary.completion_status} (success=${summary.success})`); + return new ActionResult({ + is_done: true, + success: summary.success, + extracted_content: summary.summary, + long_term_memory: `Rerun completed with status: ${summary.completion_status}. ${summary.summary.slice(0, 100)}`, + }); + } + catch (structuredError) { + this.logger.debug(`Structured rerun summary failed: ${structuredError instanceof Error + ? structuredError.message + : String(structuredError)}, falling back to text response`); + } + try { + const response = await llm.ainvoke([message], undefined, { + signal: signal ?? undefined, + }); + const summaryText = typeof response.completion === 'string' + ? response.completion + : JSON.stringify(response.completion); + const completionStatus = errorCount === 0 ? 'complete' : successCount > 0 ? 'partial' : 'failed'; + return new ActionResult({ + is_done: true, + success: errorCount === 0, + extracted_content: summaryText, + long_term_memory: `Rerun completed with status: ${completionStatus}. ${summaryText.slice(0, 100)}`, + }); + } + catch (error) { + this.logger.warning(`Failed to generate rerun summary: ${error instanceof Error ? error.message : String(error)}`); + return new ActionResult({ + is_done: true, + success: errorCount === 0, + extracted_content: `Rerun completed: ${successCount}/${results.length} steps succeeded`, + long_term_memory: `Rerun completed: ${successCount} steps succeeded, ${errorCount} errors`, + }); + } + } + async _execute_ai_step(query, includeScreenshot = false, extractLinks = false, aiStepLlm = null, signal = null) { + if (!this.browser_session) { + return new ActionResult({ + error: 'AI step failed: BrowserSession missing', + }); + } + const llm = aiStepLlm ?? this.llm; + let content; + let statsSummary; + let currentUrl = ''; + try { + const page = await this.browser_session.get_current_page?.(); + if (!page || typeof page.content !== 'function') { + throw new Error('No page available for markdown extraction'); + } + if (typeof page.url === 'function') { + currentUrl = page.url(); + } + const html = (await page.content()) || ''; + const extracted = extractCleanMarkdownFromHtml(html, { + extract_links: extractLinks, + }); + content = extracted.content; + const contentStats = extracted.stats; + statsSummary = `Content processed: ${contentStats.original_html_chars.toLocaleString()} HTML chars -> ${contentStats.initial_markdown_chars.toLocaleString()} initial markdown -> ${contentStats.final_filtered_chars.toLocaleString()} filtered markdown`; + if (contentStats.filtered_chars_removed > 0) { + statsSummary += ` (filtered ${contentStats.filtered_chars_removed.toLocaleString()} chars of noise)`; + } + } + catch (error) { + const name = error instanceof Error ? error.name : 'Error'; + const message = error instanceof Error ? error.message : String(error); + return new ActionResult({ + error: `Could not extract clean markdown: ${name}: ${message}`, + }); + } + const safeContent = sanitize_surrogates(content); + const safeQuery = sanitize_surrogates(query); + const systemPrompt = get_ai_step_system_prompt(); + const userPrompt = get_ai_step_user_prompt(safeQuery, statsSummary, safeContent); + let screenshotB64 = null; + if (includeScreenshot) { + try { + screenshotB64 = + (await this.browser_session.take_screenshot?.(false)) ?? null; + } + catch (error) { + this.logger.warning(`Failed to capture screenshot for ai_step: ${error instanceof Error ? error.message : String(error)}`); + } + } + const userMessage = screenshotB64 + ? get_rerun_summary_message(userPrompt, screenshotB64) + : new UserMessage(userPrompt); + try { + const response = await llm.ainvoke([new SystemMessage(systemPrompt), userMessage], undefined, { signal: signal ?? undefined }); + const completion = typeof response.completion === 'string' + ? response.completion + : JSON.stringify(response.completion); + const extractedContent = `\n${currentUrl}\n\n\n${safeQuery}\n\n\n${completion}\n`; + const maxMemoryLength = 1000; + if (extractedContent.length < maxMemoryLength) { + return new ActionResult({ + extracted_content: extractedContent, + include_extracted_content_only_once: false, + long_term_memory: extractedContent, + }); + } + if (!this.file_system) { + return new ActionResult({ + extracted_content: extractedContent, + include_extracted_content_only_once: false, + long_term_memory: extractedContent.slice(0, maxMemoryLength), + }); + } + const fileName = await this.file_system.save_extracted_content(extractedContent); + return new ActionResult({ + extracted_content: extractedContent, + include_extracted_content_only_once: true, + long_term_memory: `Query: ${query}\nContent in ${fileName} and once in .`, + }); + } + catch (error) { + this.logger.warning(`Failed to execute AI step: ${error instanceof Error ? error.message : String(error)}`); + return new ActionResult({ + error: `AI step failed: ${error instanceof Error ? error.message : String(error)}`, + }); + } + } + async rerun_history(history, options = {}) { + const { max_retries = 3, skip_failures = false, delay_between_actions = 2, max_step_interval = 45, wait_for_elements = false, summary_llm = null, ai_step_llm = null, signal = null, } = options; + this._throwIfAborted(signal); + // Mirror python c011 behavior: rerun should not emit create-session events. + this.state.session_initialized = true; + const results = []; + let previousItem = null; + let previousStepSucceeded = false; + try { + await this.browser_session?.start(); + for (let index = 0; index < history.history.length; index++) { + this._throwIfAborted(signal); + const historyItem = history.history[index]; + const goal = historyItem.model_output?.current_state?.next_goal ?? ''; + const stepNumber = historyItem.metadata?.step_number ?? index; + const stepName = stepNumber === 0 ? 'Initial actions' : `Step ${stepNumber}`; + const savedInterval = historyItem.metadata?.step_interval; + let stepDelay = delay_between_actions; + let delaySource = `using default delay=${this._formatDelaySeconds(stepDelay)}`; + if (typeof savedInterval === 'number' && + Number.isFinite(savedInterval)) { + stepDelay = Math.min(savedInterval, max_step_interval); + if (savedInterval > max_step_interval) { + delaySource = `capped to ${this._formatDelaySeconds(stepDelay)} (saved was ${savedInterval.toFixed(1)}s)`; + } + else { + delaySource = `using saved step_interval=${this._formatDelaySeconds(stepDelay)}`; + } + } + this.logger.info(`Replaying ${stepName} (${index + 1}/${history.history.length}) [${delaySource}]: ${goal}`); + const actions = historyItem.model_output?.action ?? []; + const hasValidAction = actions.length && !actions.every((action) => action == null); + if (!historyItem.model_output || !hasValidAction) { + this.logger.warning(`${stepName}: No action to replay, skipping`); + results.push(new ActionResult({ error: 'No action to replay' })); + continue; + } + const originalErrors = Array.isArray(historyItem.result) + ? historyItem.result + .map((result) => result?.error) + .filter((error) => typeof error === 'string') + : []; + if (originalErrors.length && skip_failures) { + const firstError = originalErrors[0] ?? 'unknown'; + const preview = firstError.length > 100 + ? `${firstError.slice(0, 100)}...` + : firstError; + this.logger.warning(`${stepName}: Original step had error(s), skipping (skip_failures=true): ${preview}`); + results.push(new ActionResult({ + error: `Skipped - original step had error: ${preview}`, + })); + continue; + } + if (this._is_redundant_retry_step(historyItem, previousItem, previousStepSucceeded)) { + this.logger.info(`${stepName}: Skipping redundant retry (previous step already succeeded with same element)`); + results.push(new ActionResult({ + extracted_content: 'Skipped - redundant retry of previous step', + include_in_memory: false, + })); + continue; + } + let attempt = 0; + let stepSucceeded = false; + let menuReopened = false; + while (attempt < max_retries) { + this._throwIfAborted(signal); + try { + const stepResult = ai_step_llm != null + ? await this._execute_history_step(historyItem, stepDelay, signal, wait_for_elements, ai_step_llm) + : await this._execute_history_step(historyItem, stepDelay, signal, wait_for_elements); + results.push(...stepResult); + stepSucceeded = true; + break; + } + catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + if (signal?.aborted || + (error instanceof Error && error.name === 'AbortError')) { + throw this._createAbortError(); + } + attempt += 1; + if (!menuReopened && + errorMessage.includes('Could not find matching element') && + previousItem && + this._is_menu_opener_step(previousItem)) { + const currentElement = this._coerceHistoryElement(historyItem.state?.interacted_element?.[0]); + if (this._is_menu_item_element(currentElement)) { + this.logger.info('Dropdown may have closed. Attempting to re-open by re-executing previous step...'); + const reopened = await this._reexecute_menu_opener(previousItem, signal, ai_step_llm); + if (reopened) { + menuReopened = true; + attempt -= 1; + stepDelay = 0.5; + this.logger.info('Dropdown re-opened, retrying element match...'); + continue; + } + } + } + if (attempt === max_retries) { + const message = `${stepName} failed after ${max_retries} attempts: ${errorMessage}`; + this.logger.error(message); + const failure = new ActionResult({ error: message }); + results.push(failure); + if (!skip_failures) { + throw new Error(message, { cause: error }); + } + } + else { + const retryDelay = Math.min(5 * 2 ** Math.max(attempt - 1, 0), 30); + this.logger.warning(`${stepName} failed (attempt ${attempt}/${max_retries}), retrying in ${retryDelay}s...`); + await this._sleep(retryDelay, signal); + } + } + } + previousItem = historyItem; + previousStepSucceeded = stepSucceeded; + } + const summaryResult = await this._generate_rerun_summary(this.task, results, summary_llm, signal); + results.push(summaryResult); + return results; + } + finally { + await this.close(); + } + } + async _execute_history_step(historyItem, delaySeconds, signal = null, wait_for_elements = false, ai_step_llm = null) { + this._throwIfAborted(signal); + if (!this.browser_session) { + throw new Error('BrowserSession is not set up'); + } + await this._sleep(delaySeconds, signal); + const interactedElements = historyItem.state?.interacted_element ?? []; + let browser_state_summary = null; + if (wait_for_elements) { + const needsElementMatching = this._historyStepNeedsElementMatching(historyItem, interactedElements); + if (needsElementMatching) { + const minElements = this._countExpectedElementsFromHistory(historyItem); + if (minElements > 0) { + browser_state_summary = await this._waitForMinimumElements(minElements, 15, 1, signal); + } + } + } + if (!browser_state_summary) { + browser_state_summary = + await this.browser_session.get_browser_state_with_recovery?.({ + cache_clickable_elements_hashes: false, + include_screenshot: false, + signal, + }); + } + if (!browser_state_summary || !historyItem.model_output) { + throw new Error('Invalid browser state or model output'); + } + const results = []; + const pendingActions = []; + for (let actionIndex = 0; actionIndex < historyItem.model_output.action.length; actionIndex++) { + this._throwIfAborted(signal); + const originalAction = historyItem.model_output.action[actionIndex]; + if (!originalAction) { + continue; + } + const actionPayload = typeof originalAction?.model_dump === 'function' + ? originalAction.model_dump({ exclude_unset: true }) + : originalAction; + const actionName = Object.keys(actionPayload ?? {})[0] ?? null; + if (actionName && + ['extract', 'extract_structured_data', 'extract_content'].includes(actionName)) { + if (pendingActions.length > 0) { + this._throwIfAborted(signal); + const batchActions = [...pendingActions]; + pendingActions.length = 0; + const batchResults = await this.multi_act(batchActions, { signal }); + results.push(...batchResults); + } + const params = actionPayload[actionName] ?? {}; + const query = typeof params.query === 'string' ? params.query : ''; + const extractLinks = Boolean(params.extract_links); + this.logger.info(`Using AI step for extract action: ${query.slice(0, 50)}...`); + const aiResult = await this._execute_ai_step(query, false, extractLinks, ai_step_llm, signal); + results.push(aiResult); + continue; + } + const updatedAction = await this._update_action_indices(this._coerceHistoryElement(interactedElements[actionIndex]), originalAction, browser_state_summary); + if (!updatedAction) { + const historicalElement = this._coerceHistoryElement(interactedElements[actionIndex]); + const selectorCount = Object.keys(browser_state_summary.selector_map ?? {}).length; + throw new Error(`Could not find matching element for action ${actionIndex} in current page.\n` + + ` Looking for: ${this._formatHistoryElementForError(historicalElement)}\n` + + ` Page has ${selectorCount} interactive elements.\n` + + ' Tried: EXACT hash → STABLE hash → XPATH → AX_NAME → ATTRIBUTE matching'); + } + if (typeof updatedAction?.model_dump === 'function') { + pendingActions.push(updatedAction.model_dump({ exclude_unset: true })); + } + else { + pendingActions.push(updatedAction); + } + } + if (pendingActions.length > 0) { + this._throwIfAborted(signal); + const batchActions = [...pendingActions]; + pendingActions.length = 0; + const batchResults = await this.multi_act(batchActions, { signal }); + results.push(...batchResults); + } + return results; + } + _historyStepNeedsElementMatching(historyItem, interactedElements) { + const actions = historyItem.model_output?.action ?? []; + for (let index = 0; index < actions.length; index++) { + const action = actions[index]; + if (!action) { + continue; + } + const payload = typeof action.model_dump === 'function' + ? action.model_dump({ exclude_unset: true }) + : action; + const actionName = Object.keys(payload ?? {})[0] ?? null; + if (!actionName) { + continue; + } + if ([ + 'click', + 'input', + 'input_text', + 'hover', + 'select_option', + 'select_dropdown_option', + 'drag_and_drop', + ].includes(actionName)) { + const historicalElement = this._coerceHistoryElement(interactedElements[index] ?? null); + if (historicalElement) { + return true; + } + } + } + return false; + } + _countExpectedElementsFromHistory(historyItem) { + if (!historyItem.model_output?.action?.length) { + return 0; + } + let maxIndex = -1; + for (const action of historyItem.model_output.action) { + const index = this._extractActionIndex(action); + if (index != null) { + maxIndex = Math.max(maxIndex, index); + } + } + if (maxIndex < 0) { + return 0; + } + return Math.min(maxIndex + 1, 50); + } + async _waitForMinimumElements(minElements, timeoutSeconds = 30, pollIntervalSeconds = 1, signal = null) { + if (!this.browser_session) { + return null; + } + const start = Date.now(); + let lastCount = 0; + let lastState = null; + while ((Date.now() - start) / 1000 < timeoutSeconds) { + this._throwIfAborted(signal); + const state = await this.browser_session.get_browser_state_with_recovery?.({ + cache_clickable_elements_hashes: false, + include_screenshot: false, + signal, + }); + lastState = state ?? null; + const currentCount = Object.keys(state?.selector_map ?? {}).length; + if (currentCount >= minElements) { + this.logger.debug(`Page has ${currentCount} interactive elements (needed ${minElements}), proceeding`); + return state; + } + if (currentCount !== lastCount) { + const remaining = Math.max(0, timeoutSeconds - (Date.now() - start) / 1000); + this.logger.debug(`Waiting for elements: ${currentCount}/${minElements} (timeout in ${remaining.toFixed(1)}s)`); + lastCount = currentCount; + } + await this._sleep(pollIntervalSeconds, signal); + } + this.logger.warning(`Timeout waiting for ${minElements} elements, proceeding with ${lastCount} elements`); + return lastState; + } + _extractActionIndex(action) { + if (action && typeof action.get_index === 'function') { + const index = action.get_index(); + if (typeof index === 'number' && Number.isFinite(index)) { + return index; + } + } + if (!action || typeof action !== 'object') { + return null; + } + const modelDump = typeof action.model_dump === 'function' + ? action.model_dump() + : action; + if (!modelDump || + typeof modelDump !== 'object' || + Array.isArray(modelDump)) { + return null; + } + const actionName = Object.keys(modelDump)[0]; + if (!actionName) { + return null; + } + const params = modelDump[actionName]; + const index = params?.index; + return typeof index === 'number' && Number.isFinite(index) ? index : null; + } + _extractActionType(action) { + if (!action || typeof action !== 'object') { + return null; + } + const modelDump = typeof action.model_dump === 'function' + ? action.model_dump() + : action; + if (!modelDump || + typeof modelDump !== 'object' || + Array.isArray(modelDump)) { + return null; + } + const actionName = Object.keys(modelDump)[0]; + return actionName ?? null; + } + _sameHistoryElement(current, previous) { + if (!current || !previous) { + return false; + } + if (current.element_hash && + previous.element_hash && + current.element_hash === previous.element_hash) { + return true; + } + if (current.stable_hash && + previous.stable_hash && + current.stable_hash === previous.stable_hash) { + return true; + } + if (current.xpath && previous.xpath && current.xpath === previous.xpath) { + return true; + } + if (current.tag_name && + previous.tag_name && + current.tag_name === previous.tag_name) { + for (const key of ['name', 'id', 'aria-label']) { + const currentValue = current.attributes?.[key]; + const previousValue = previous.attributes?.[key]; + if (currentValue && + previousValue && + String(currentValue) === String(previousValue)) { + return true; + } + } + } + return false; + } + _is_redundant_retry_step(currentItem, previousItem, previousStepSucceeded) { + if (!previousItem || !previousStepSucceeded) { + return false; + } + const currentActions = currentItem.model_output?.action ?? []; + const previousActions = previousItem.model_output?.action ?? []; + if (!currentActions.length || !previousActions.length) { + return false; + } + const currentActionType = this._extractActionType(currentActions[0]); + const previousActionType = this._extractActionType(previousActions[0]); + if (!currentActionType || currentActionType !== previousActionType) { + return false; + } + const currentElement = this._coerceHistoryElement(currentItem.state?.interacted_element?.[0]); + const previousElement = this._coerceHistoryElement(previousItem.state?.interacted_element?.[0]); + if (!this._sameHistoryElement(currentElement, previousElement)) { + return false; + } + this.logger.debug(`Detected redundant retry on same element with action "${currentActionType}"`); + return true; + } + _is_menu_opener_step(historyItem) { + const element = this._coerceHistoryElement(historyItem?.state?.interacted_element?.[0]); + if (!element) { + return false; + } + const attrs = element.attributes ?? {}; + if (['true', 'menu', 'listbox'].includes(String(attrs['aria-haspopup']))) { + return true; + } + if (attrs['data-gw-click'] === 'toggleSubMenu') { + return true; + } + if (String(attrs.class ?? '').includes('expand-button')) { + return true; + } + if (attrs.role === 'menuitem' && + ['false', 'true'].includes(String(attrs['aria-expanded']))) { + return true; + } + if (attrs.role === 'button' && + ['false', 'true'].includes(String(attrs['aria-expanded']))) { + return true; + } + return false; + } + _is_menu_item_element(element) { + if (!element) { + return false; + } + const attrs = element.attributes ?? {}; + const role = String(attrs.role ?? ''); + if ([ + 'menuitem', + 'option', + 'menuitemcheckbox', + 'menuitemradio', + 'treeitem', + ].includes(role)) { + return true; + } + const className = String(attrs.class ?? ''); + if (className.includes('gw-action--inner')) { + return true; + } + if (className.toLowerCase().includes('menuitem')) { + return true; + } + if (element.ax_name && element.ax_name.trim()) { + const lowered = className.toLowerCase(); + if (['dropdown', 'popup', 'menu', 'submenu', 'action'].some((needle) => lowered.includes(needle))) { + return true; + } + } + return false; + } + async _reexecute_menu_opener(openerItem, signal = null, aiStepLlm = null) { + try { + this.logger.info('Re-opening dropdown/menu by re-executing previous step...'); + await this._execute_history_step(openerItem, 0.5, signal, false, aiStepLlm); + await this._sleep(0.3, signal); + return true; + } + catch (error) { + this.logger.warning(`Failed to re-open dropdown: ${error instanceof Error ? error.message : String(error)}`); + return false; + } + } + _formatHistoryElementForError(element) { + if (!element) { + return ''; + } + const parts = [`<${element.tag_name || 'unknown'}>`]; + for (const key of ['name', 'id', 'aria-label', 'type']) { + const value = element.attributes?.[key]; + if (typeof value === 'string' && value.trim()) { + parts.push(`${key}="${value}"`); + } + } + if (element.xpath) { + const xpath = element.xpath.length > 60 + ? `...${element.xpath.slice(-57)}` + : element.xpath; + parts.push(`xpath="${xpath}"`); + } + if (element.element_hash) { + parts.push(`hash=${element.element_hash}`); + } + if (element.stable_hash) { + parts.push(`stable_hash=${element.stable_hash}`); + } + return parts.join(' '); + } + async _update_action_indices(historicalElement, action, browserStateSummary) { + if (!historicalElement || !browserStateSummary?.selector_map) { + return action; + } + const selectorMap = browserStateSummary.selector_map ?? {}; + if (!Object.keys(selectorMap).length) { + return action; + } + let matchLevel = null; + let currentNode = null; + if (historicalElement.element_hash) { + for (const node of Object.values(selectorMap)) { + const nodeHash = HistoryTreeProcessor.compute_element_hash(node); + if (nodeHash === historicalElement.element_hash) { + currentNode = node; + matchLevel = 'EXACT'; + break; + } + } + } + if (!currentNode && historicalElement.stable_hash) { + for (const node of Object.values(selectorMap)) { + const stableHash = HistoryTreeProcessor.compute_stable_hash(node); + if (stableHash === historicalElement.stable_hash) { + currentNode = node; + matchLevel = 'STABLE'; + this.logger.info('Element matched at STABLE hash fallback'); + break; + } + } + } + if (!currentNode && historicalElement.xpath) { + for (const node of Object.values(selectorMap)) { + if (node?.xpath === historicalElement.xpath) { + currentNode = node; + matchLevel = 'XPATH'; + this.logger.info(`Element matched at XPATH fallback: ${historicalElement.xpath}`); + break; + } + } + } + if (!currentNode && historicalElement.ax_name) { + const tagName = historicalElement.tag_name?.toLowerCase(); + const targetAxName = historicalElement.ax_name; + for (const node of Object.values(selectorMap)) { + const nodeAxName = HistoryTreeProcessor.get_accessible_name(node); + if (node?.tag_name?.toLowerCase() === tagName && + typeof nodeAxName === 'string' && + nodeAxName === targetAxName) { + currentNode = node; + matchLevel = 'AX_NAME'; + this.logger.info(`Element matched at AX_NAME fallback: ${targetAxName}`); + break; + } + } + } + if (!currentNode && historicalElement.attributes) { + const tagName = historicalElement.tag_name?.toLowerCase(); + for (const attrKey of ['name', 'id', 'aria-label']) { + const attrValue = historicalElement.attributes[attrKey]; + if (!attrValue) { + continue; + } + for (const node of Object.values(selectorMap)) { + if (node?.tag_name?.toLowerCase() === tagName && + node?.attributes?.[attrKey] === attrValue) { + currentNode = node; + matchLevel = 'ATTRIBUTE'; + this.logger.info(`Element matched via ${attrKey} attribute fallback: ${attrValue}`); + break; + } + } + if (currentNode) { + break; + } + } + } + if (!currentNode || currentNode.highlight_index == null) { + return null; + } + const currentIndex = typeof action?.get_index === 'function' ? action.get_index() : null; + if (currentIndex !== currentNode.highlight_index && + typeof action?.set_index === 'function') { + action.set_index(currentNode.highlight_index); + this.logger.info(`Element moved in DOM, updated index from ${currentIndex} to ${currentNode.highlight_index} (matched at ${matchLevel ?? 'UNKNOWN'} level)`); + } + return action; + } + async load_and_rerun(history_file = null, options = {}) { + const { variables = null, ...rerunOptions } = options; + const target = history_file ?? 'AgentHistory.json'; + const history = AgentHistoryList.load_from_file(target, this.AgentOutput); + const substitutedHistory = variables + ? this._substitute_variables_in_history(history, variables) + : history; + return this.rerun_history(substitutedHistory, rerunOptions); + } + detect_variables() { + return detect_variables_in_history(this.history); + } + save_history(file_path = null) { + const target = file_path ?? 'AgentHistory.json'; + this.history.save_to_file(target, this.sensitive_data ?? null); + } + _coerceHistoryElement(element) { + if (!element) { + return null; + } + if (element instanceof DOMHistoryElement) { + return element; + } + const payload = element; + return new DOMHistoryElement(payload.tag_name ?? '', payload.xpath ?? '', payload.highlight_index ?? null, payload.entire_parent_branch_path ?? [], payload.attributes ?? {}, payload.shadow_root ?? false, payload.css_selector ?? null, payload.page_coordinates ?? null, payload.viewport_coordinates ?? null, payload.viewport_info ?? null, payload.element_hash != null ? String(payload.element_hash) : null, payload.stable_hash != null ? String(payload.stable_hash) : null, payload.ax_name != null ? String(payload.ax_name) : null); + } + _substitute_variables_in_history(history, variables) { + const detectedVars = detect_variables_in_history(history); + const valueReplacements = {}; + for (const [varName, newValue] of Object.entries(variables)) { + const detected = detectedVars[varName]; + if (!detected) { + this.logger.warning(`Variable "${varName}" not found in history, skipping substitution`); + continue; + } + valueReplacements[detected.original_value] = newValue; + } + if (!Object.keys(valueReplacements).length) { + this.logger.info('No variables to substitute'); + return history; + } + const clonedHistory = this._clone_history_for_substitution(history); + let substitutionCount = 0; + for (const historyItem of clonedHistory.history) { + if (!historyItem.model_output?.action?.length) { + continue; + } + for (let actionIndex = 0; actionIndex < historyItem.model_output.action.length; actionIndex += 1) { + const action = historyItem.model_output.action[actionIndex]; + const actionPayload = typeof action.model_dump === 'function' + ? action.model_dump() + : action; + if (!actionPayload || + typeof actionPayload !== 'object' || + Array.isArray(actionPayload)) { + continue; + } + substitutionCount += substitute_in_dict(actionPayload, valueReplacements); + const ActionCtor = action?.constructor; + if (typeof ActionCtor === 'function') { + historyItem.model_output.action[actionIndex] = new ActionCtor(actionPayload); + } + else { + historyItem.model_output.action[actionIndex] = actionPayload; + } + } + } + this.logger.info(`Substituted ${substitutionCount} value(s) in ${Object.keys(valueReplacements).length} variable type(s) in history`); + return clonedHistory; + } + _clone_history_for_substitution(history) { + const payload = history.toJSON(); + const historyItems = (payload.history ?? []).map((entry) => { + const modelOutput = entry.model_output + ? this.AgentOutput.fromJSON(entry.model_output) + : null; + const result = (entry.result ?? []).map((item) => new ActionResult(item)); + const interacted = Array.isArray(entry.state?.interacted_element) + ? entry.state.interacted_element.map((element) => this._coerceHistoryElement(element)) + : []; + const state = new BrowserStateHistory(entry.state?.url ?? '', entry.state?.title ?? '', entry.state?.tabs ?? [], interacted, entry.state?.screenshot_path ?? null); + const metadata = entry.metadata + ? new StepMetadata(entry.metadata.step_start_time, entry.metadata.step_end_time, entry.metadata.step_number, entry.metadata.step_interval ?? null) + : null; + return new AgentHistory(modelOutput, result, state, metadata, entry.state_message ?? null); + }); + return new AgentHistoryList(historyItems, history.usage ?? null); + } + _createAbortError() { + const error = new Error('Operation aborted'); + error.name = 'AbortError'; + return error; + } + _throwIfAborted(signal = null) { + if (signal?.aborted) { + throw this._createAbortError(); + } + } + _relayAbortSignal(signal, controller) { + if (!signal) { + return () => { }; + } + if (signal.aborted) { + controller.abort(signal.reason); + return () => { }; + } + const handleAbort = () => controller.abort(signal.reason); + signal.addEventListener('abort', handleAbort, { once: true }); + return () => signal.removeEventListener('abort', handleAbort); + } + _formatDelaySeconds(delaySeconds) { + if (delaySeconds < 1) { + return `${Math.round(delaySeconds * 1000)}ms`; + } + return `${delaySeconds.toFixed(1)}s`; + } + async _sleep(seconds, signal = null) { + if (seconds <= 0) { + return; + } + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + resolve(); + }, seconds * 1000); + const onAbort = () => { + clearTimeout(timeout); + cleanup(); + reject(this._createAbortError()); + }; + const cleanup = () => { + signal?.removeEventListener('abort', onAbort); + }; + if (signal) { + if (signal.aborted) { + onAbort(); + return; + } + signal.addEventListener('abort', onAbort, { once: true }); + } + }); + } + async wait_until_resumed() { + if (!this.state.paused) { + return; + } + if (!this._external_pause_event.resolve) { + this._external_pause_event.promise = new Promise((resolve) => { + this._external_pause_event.resolve = resolve; + }); + } + await this._external_pause_event.promise; + } + async log_completion() { + this.logger.info('✅ Agent completed task'); + } + pause() { + if (this.state.paused) { + return; + } + this.state.paused = true; + this._external_pause_event.promise = new Promise((resolve) => { + this._external_pause_event.resolve = resolve; + }); + } + resume() { + if (!this.state.paused) { + return; + } + this.state.paused = false; + this._external_pause_event.resolve?.(); + this._external_pause_event.resolve = null; + this._external_pause_event.promise = Promise.resolve(); + } + stop() { + this.state.stopped = true; + this.resume(); + } + async close() { + if (this._closePromise) { + await this._closePromise; + return; + } + this._closePromise = (async () => { + const browser_session = this.browser_session; + try { + if (browser_session) { + this._release_browser_session_claim(browser_session); + if (this._has_any_browser_session_attachments(browser_session)) { + this.logger.debug('Skipping BrowserSession shutdown because other attached Agents are still active.'); + } + else { + this._cleanup_shared_session_step_lock_if_unused(browser_session); + if (typeof browser_session.stop === 'function') { + await browser_session.stop(); + } + else if (typeof browser_session.close === 'function') { + await browser_session.close(); + } + } + } + } + catch (error) { + this.logger.error(`Error during agent cleanup: ${error instanceof Error ? error.message : String(error)}`); + } + if (this.skill_service && + typeof this.skill_service.close === 'function') { + try { + await this.skill_service.close(); + } + catch (error) { + this.logger.error(`Error during skill service cleanup: ${error instanceof Error ? error.message : String(error)}`); + } + } + })(); + await this._closePromise; + } + /** + * Get the trace and trace_details objects for the agent + * Contains comprehensive metadata about the agent run for debugging and analysis + */ + get_trace_object() { + // Helper to extract website from task text + const extract_task_website = (task_text) => { + const url_pattern = /https?:\/\/[^\s<>"']+|www\.[^\s<>"']+|[^\s<>"']+\.[a-z]{2,}(?:\/[^\s<>"']*)?/i; + const match = task_text.match(url_pattern); + return match ? match[0] : null; + }; + // Helper to get complete history without screenshots + const get_complete_history_without_screenshots = (history_data) => { + if (history_data.history) { + for (const item of history_data.history) { + if (item.state && item.state.screenshot) { + item.state.screenshot = null; + } + } + } + return JSON.stringify(history_data); + }; + // Generate autogenerated fields + const trace_id = uuid7str(); + const timestamp = new Date().toISOString(); + // Collect data + const structured_output = this.history.structured_output; + const structured_output_json = structured_output + ? JSON.stringify(structured_output) + : null; + const final_result = this.history.final_result(); + const action_history = this.history.action_history(); + const action_errors = this.history.errors(); + const urls = this.history.urls(); + const usage = this.history.usage; + // Build trace object + const trace = { + // Autogenerated fields + trace_id, + timestamp, + browser_use_version: this.version, + git_info: null, // Can be enhanced if needed + // Direct agent properties + model: this.llm.model || 'unknown', + settings: this.settings ? JSON.stringify(this.settings) : null, + task_id: this.task_id, + task_truncated: this.task.length > 20000 ? this.task.slice(0, 20000) : this.task, + task_website: extract_task_website(this.task), + // AgentHistoryList methods + structured_output_truncated: structured_output_json && structured_output_json.length > 20000 + ? structured_output_json.slice(0, 20000) + : structured_output_json, + action_history_truncated: action_history + ? JSON.stringify(action_history) + : null, + action_errors: action_errors ? JSON.stringify(action_errors) : null, + urls: urls ? JSON.stringify(urls) : null, + final_result_response_truncated: final_result && final_result.length > 20000 + ? final_result.slice(0, 20000) + : final_result, + self_report_completed: this.history.is_done() ? 1 : 0, + self_report_success: this.history.is_successful() ? 1 : 0, + duration: this.history.total_duration_seconds(), + steps_taken: this.history.number_of_steps(), + usage: usage ? JSON.stringify(usage) : null, + }; + // Build trace_details object + const trace_details = { + // Autogenerated fields (ensure same as trace) + trace_id, + timestamp, + // Direct agent properties + task: this.task, + // AgentHistoryList methods + structured_output: structured_output_json, + final_result_response: final_result, + complete_history: get_complete_history_without_screenshots(this.history.model_dump?.() || {}), + }; + return { trace, trace_details }; + } + async _log_agent_run() { + this.logger.info(`\x1b[34m🎯 Task: ${this.task}\x1b[0m`); + this.logger.debug(`🤖 Browser-Use Library Version ${this.version} (${this.source})`); + if (CONFIG.BROWSER_USE_VERSION_CHECK && + process.env.NODE_ENV !== 'test' && + !process.env.VITEST) { + const latestVersion = await check_latest_browser_use_version(); + if (latestVersion && latestVersion !== this.version) { + this.logger.info(`📦 Newer version available: ${latestVersion} (current: ${this.version}). Upgrade with: npm install browser-use@${latestVersion}`); + } + } + } + _createInterruptedError(message = '') { + const interruptedError = new Error(message); + interruptedError.name = 'InterruptedError'; + return interruptedError; + } + async _raise_if_stopped_or_paused() { + if (this.register_should_stop_callback) { + const shouldStop = await this.register_should_stop_callback(); + if (shouldStop) { + this.logger.info('External callback requested stop'); + this.state.stopped = true; + throw this._createInterruptedError(); + } + } + if (this.register_external_agent_status_raise_error_callback) { + const shouldRaise = await this.register_external_agent_status_raise_error_callback(); + if (shouldRaise) { + throw this._createInterruptedError(); + } + } + if (this.state.stopped) { + throw this._createInterruptedError('Agent stopped'); + } + if (this.state.paused) { + throw this._createInterruptedError('Agent paused'); + } + } + async _handle_post_llm_processing(browser_state_summary, input_messages, _actions = []) { + if (this.register_new_step_callback && this.state.last_model_output) { + await this.register_new_step_callback(browser_state_summary, this.state.last_model_output, this.state.n_steps); + } + log_response(this.state.last_model_output, this.controller, this.logger); + if (this.settings.save_conversation_path) { + const dir = this.settings.save_conversation_path; + const filepath = path.join(dir, `step_${this.state.n_steps}.json`); + await fs.promises.mkdir(path.dirname(filepath), { recursive: true }); + await fs.promises.writeFile(filepath, JSON.stringify({ + messages: input_messages, + response: this.state.last_model_output?.model_dump(), + }, null, 2), this.settings.save_conversation_path_encoding); + } + } + /** Handle all types of errors that can occur during a step (python c011 parity). */ + async _handle_step_error(error) { + if (error?.name === 'InterruptedError') { + const message = error.message + ? `The agent was interrupted mid-step - ${error.message}` + : 'The agent was interrupted mid-step'; + this.logger.warning(message); + return; + } + const include_trace = this.logger.level === 'debug'; + const error_msg = AgentError.format_error(error, include_trace); + const maxTotalFailures = this._max_total_failures(); + const prefix = `❌ Result failed ${this.state.consecutive_failures + 1}/${maxTotalFailures} times: `; + this.state.consecutive_failures += 1; + const isFinalFailure = this.state.consecutive_failures >= maxTotalFailures; + const isParseError = error_msg.includes('Could not parse response') || + error_msg.includes('tool_use_failed'); + if (isParseError) { + const parseLog = `Model: ${this.llm.model} failed`; + if (isFinalFailure) { + this.logger.error(parseLog); + } + else { + this.logger.warning(parseLog); + } + } + if (isFinalFailure) { + this.logger.error(`${prefix}${error_msg}`); + } + else { + this.logger.warning(`${prefix}${error_msg}`); + } + this.state.last_result = [new ActionResult({ error: error_msg })]; + } + async _finalize(browser_state_summary) { + const step_end_time = Date.now() / 1000; + this._enforceDoneOnlyForCurrentStep = false; + if (!this.state.last_result) { + return; + } + if (browser_state_summary) { + let stepInterval = null; + if (this.history.history.length > 0) { + const lastMetadata = this.history.history.at(-1)?.metadata; + if (lastMetadata) { + stepInterval = Math.max(0, lastMetadata.step_end_time - lastMetadata.step_start_time); + } + } + const metadata = new StepMetadata(this.step_start_time, step_end_time, this.state.n_steps, stepInterval); + await this._make_history_item(this.state.last_model_output, browser_state_summary, this.state.last_result, metadata, this._message_manager.last_state_message_text); + } + this._log_step_completion_summary(this.step_start_time, this.state.last_result); + this.save_file_system_state(); + if (browser_state_summary && this.state.last_model_output) { + const actions_data = this.state.last_model_output.action.map((action) => typeof action?.model_dump === 'function' + ? action.model_dump() + : action); + const step_event = CreateAgentStepEvent.fromAgentStep(this, this.state.last_model_output, this.state.last_result, actions_data, browser_state_summary); + this.eventbus.dispatch(step_event); + } + this.state.n_steps += 1; + } + async _handle_final_step(step_info = null) { + const isLastStep = Boolean(step_info && step_info.is_last_step()); + this._enforceDoneOnlyForCurrentStep = isLastStep; + if (isLastStep) { + const message = 'You reached max_steps - this is your last step. Your only tool available is the "done" tool. No other tool is available. All other tools which you see in history or examples are not available.\n' + + 'If the task is not yet fully finished as requested by the user, set success in "done" to false! E.g. if not all steps are fully completed. Else success to true.\n' + + 'Include everything you found out for the ultimate task in the done text.'; + this._message_manager._add_context_message(new UserMessage(message)); + this.logger.debug('Last step finishing up'); + } + } + _max_total_failures() { + return (this.settings.max_failures + + Number(this.settings.final_response_after_failure)); + } + async _handle_failure_limit_recovery() { + if (!this.settings.final_response_after_failure || + this.state.consecutive_failures < this.settings.max_failures) { + return; + } + const message = `You failed ${this.settings.max_failures} times. Therefore we terminate the agent.\n` + + 'Your only tool available is the "done" tool. No other tool is available. All other tools which you see in history or examples are not available.\n' + + 'If the task is not yet fully finished as requested by the user, set success in "done" to false! E.g. if not all steps are fully completed. Else success to true.\n' + + 'Include everything you found out for the ultimate task in the done text.'; + this._message_manager._add_context_message(new UserMessage(message)); + this._enforceDoneOnlyForCurrentStep = true; + this.logger.debug('Force done action, because we reached max_failures.'); + } + _update_plan_from_model_output(modelOutput) { + if (!this.settings.enable_planning) { + return; + } + if (Array.isArray(modelOutput.plan_update)) { + this.state.plan = modelOutput.plan_update.map((stepText) => new PlanItem({ + text: stepText, + status: 'pending', + })); + this.state.current_plan_item_index = 0; + this.state.plan_generation_step = this.state.n_steps; + if (this.state.plan.length > 0) { + this.state.plan[0].status = 'current'; + } + this.logger.info(`📋 Plan updated with ${this.state.plan.length} steps`); + return; + } + if (typeof modelOutput.current_plan_item !== 'number' || + !this.state.plan || + this.state.plan.length === 0) { + return; + } + const oldIndex = this.state.current_plan_item_index; + const newIndex = Math.max(0, Math.min(modelOutput.current_plan_item, this.state.plan.length - 1)); + for (let i = oldIndex; i < newIndex; i += 1) { + if (this.state.plan[i] && + (this.state.plan[i].status === 'current' || + this.state.plan[i].status === 'pending')) { + this.state.plan[i].status = 'done'; + } + } + if (this.state.plan[newIndex]) { + this.state.plan[newIndex].status = 'current'; + } + this.state.current_plan_item_index = newIndex; + } + _render_plan_description() { + if (!this.settings.enable_planning || !this.state.plan) { + return null; + } + const markers = { + done: '[x]', + current: '[>]', + pending: '[ ]', + skipped: '[-]', + }; + return this.state.plan + .map((step, index) => `${markers[step.status] ?? '[ ]'} ${index}: ${step.text}`) + .join('\n'); + } + _inject_replan_nudge() { + if (!this.settings.enable_planning || !this.state.plan) { + return; + } + if (this.settings.planning_replan_on_stall <= 0) { + return; + } + if (this.state.consecutive_failures < this.settings.planning_replan_on_stall) { + return; + } + const message = 'REPLAN SUGGESTED: You have failed ' + + `${this.state.consecutive_failures} consecutive times. ` + + 'Your current plan may need revision. ' + + 'Output a new `plan_update` with revised steps to recover.'; + this.logger.info(`📋 Replan nudge injected after ${this.state.consecutive_failures} consecutive failures`); + this._message_manager._add_context_message(new UserMessage(message)); + } + _inject_exploration_nudge() { + if (!this.settings.enable_planning || this.state.plan) { + return; + } + if (this.settings.planning_exploration_limit <= 0) { + return; + } + if (this.state.n_steps < this.settings.planning_exploration_limit) { + return; + } + const message = 'PLANNING NUDGE: You have taken ' + + `${this.state.n_steps} steps without creating a plan. ` + + 'If the task is complex, output a `plan_update` with clear todo items now. ' + + 'If the task is already done or nearly done, call `done` instead.'; + this.logger.info(`📋 Exploration nudge injected after ${this.state.n_steps} steps without a plan`); + this._message_manager._add_context_message(new UserMessage(message)); + } + _inject_loop_detection_nudge() { + if (!this.settings.loop_detection_enabled) { + return; + } + const nudge = this.state.loop_detector.get_nudge_message(); + if (!nudge) { + return; + } + this.logger.info(`🔁 Loop detection nudge injected (repetition=${this.state.loop_detector.max_repetition_count}, stagnation=${this.state.loop_detector.consecutive_stagnant_pages})`); + this._message_manager._add_context_message(new UserMessage(nudge)); + } + _update_loop_detector_actions() { + if (!this.settings.loop_detection_enabled || + !this.state.last_model_output) { + return; + } + const exemptActions = new Set(['wait', 'done', 'go_back']); + for (const action of this.state.last_model_output.action) { + const actionData = typeof action?.model_dump === 'function' + ? action.model_dump() + : action; + if (!actionData || typeof actionData !== 'object') { + continue; + } + const actionName = Object.keys(actionData)[0] ?? 'unknown'; + if (exemptActions.has(actionName)) { + continue; + } + const rawParams = actionData[actionName]; + const params = rawParams && typeof rawParams === 'object' && !Array.isArray(rawParams) + ? rawParams + : {}; + this.state.loop_detector.record_action(actionName, params); + } + } + _update_loop_detector_page_state(browser_state_summary) { + if (!this.settings.loop_detection_enabled) { + return; + } + const url = browser_state_summary.url ?? ''; + const elementCount = browser_state_summary.selector_map + ? Object.keys(browser_state_summary.selector_map).length + : 0; + const domText = (() => { + try { + return (browser_state_summary.element_tree?.clickable_elements_to_string?.() ?? + ''); + } + catch { + return ''; + } + })(); + this.state.loop_detector.record_page_state(url, domText, elementCount); + } + _inject_budget_warning(step_info = null) { + if (!step_info) { + return; + } + const stepsUsed = step_info.step_number + 1; + const budgetRatio = stepsUsed / step_info.max_steps; + if (budgetRatio < 0.75 || step_info.is_last_step()) { + return; + } + const stepsRemaining = step_info.max_steps - stepsUsed; + const pct = Math.floor(budgetRatio * 100); + const message = `BUDGET WARNING: You have used ${stepsUsed}/${step_info.max_steps} steps ` + + `(${pct}%). ${stepsRemaining} steps remaining. ` + + 'If the task cannot be completed in the remaining steps, prioritize: ' + + '(1) consolidate your results (save to files if the file system is in use), ' + + '(2) call done with what you have. ' + + 'Partial results are far more valuable than exhausting all steps with nothing saved.'; + this.logger.info(`Step budget warning: ${stepsUsed}/${step_info.max_steps} (${pct}%)`); + this._message_manager._add_context_message(new UserMessage(message)); + } + async _run_simple_judge() { + const lastHistoryItem = this.history.history[this.history.history.length - 1]; + if (!lastHistoryItem || !lastHistoryItem.result.length) { + return; + } + const lastResult = lastHistoryItem.result[lastHistoryItem.result.length - 1]; + if (!lastResult.is_done || !lastResult.success) { + return; + } + const messages = construct_simple_judge_messages({ + task: this.task, + final_result: this.history.final_result() ?? '', + }); + try { + const response = await this.llm.ainvoke(messages, SimpleJudgeOutputFormat); + const parsed = this._parseCompletionPayload(response.completion); + if (typeof parsed?.is_correct !== 'boolean') { + this.logger.debug('Simple judge response missing boolean is_correct; skipping override.'); + return; + } + const isCorrect = parsed.is_correct; + const reason = typeof parsed?.reason === 'string' && parsed.reason.trim() + ? parsed.reason.trim() + : 'Task requirements not fully met'; + if (!isCorrect) { + this.logger.info(`⚠️ Simple judge overriding success to failure: ${reason}`); + lastResult.success = false; + const note = `[Simple judge: ${reason}]`; + if (lastResult.extracted_content) { + lastResult.extracted_content += `\n\n${note}`; + } + else { + lastResult.extracted_content = note; + } + } + } + catch (error) { + this.logger.warning(`Simple judge failed with error: ${error instanceof Error ? error.message : String(error)}`); + } + } + async _judge_trace() { + const messages = construct_judge_messages({ + task: this.task, + final_result: this.history.final_result() ?? '', + agent_steps: this.history.agent_steps(), + screenshot_paths: this.history + .screenshot_paths() + .filter((value) => typeof value === 'string'), + max_images: 10, + ground_truth: this.settings.ground_truth, + use_vision: this.settings.use_vision, + }); + try { + const invokeOptions = this.judge_llm?.provider === 'browser-use' + ? { request_type: 'judge' } + : undefined; + const response = await this.judge_llm.ainvoke(messages, JudgeOutputFormat, invokeOptions); + const parsed = this._parseCompletionPayload(response.completion); + const validation = JudgeSchema.safeParse(parsed); + if (!validation.success) { + this.logger.warning('Judge trace response did not match expected schema; skipping judgement.'); + return null; + } + return validation.data; + } + catch (error) { + this.logger.warning(`Judge trace failed: ${error instanceof Error ? error.message : String(error)}`); + return null; + } + } + async _judge_and_log() { + const lastHistoryItem = this.history.history[this.history.history.length - 1]; + if (!lastHistoryItem || !lastHistoryItem.result.length) { + return; + } + const lastResult = lastHistoryItem.result[lastHistoryItem.result.length - 1]; + if (!lastResult.is_done) { + return; + } + const judgement = await this._judge_trace(); + lastResult.judgement = judgement; + if (!judgement) { + return; + } + if (lastResult.success === true && judgement.verdict === true) { + return; + } + let judgeLog = '\n'; + if (lastResult.success === true && judgement.verdict === false) { + judgeLog += '⚠️ Agent reported success but judge thinks task failed\n'; + } + judgeLog += `⚖️ Judge Verdict: ${judgement.verdict ? 'PASS' : 'FAIL'}\n`; + if (judgement.failure_reason) { + judgeLog += ` Failure Reason: ${judgement.failure_reason}\n`; + } + if (judgement.reached_captcha) { + judgeLog += ' Captcha Detected: Agent encountered captcha challenges\n'; + judgeLog += + ' Use Browser Use Cloud for stealth browser infra: https://docs.browser-use.com/customize/browser/remote\n'; + } + if (judgement.reasoning) { + judgeLog += ` ${judgement.reasoning}\n`; + } + this.logger.info(judgeLog); + } + _replace_urls_in_text(text) { + const replacedUrls = {}; + const shortenedText = text.replace(URL_PATTERN, (originalUrl) => { + const queryStart = originalUrl.indexOf('?'); + const fragmentStart = originalUrl.indexOf('#'); + let afterPathStart = originalUrl.length; + if (queryStart !== -1) { + afterPathStart = Math.min(afterPathStart, queryStart); + } + if (fragmentStart !== -1) { + afterPathStart = Math.min(afterPathStart, fragmentStart); + } + const baseUrl = originalUrl.slice(0, afterPathStart); + const afterPath = originalUrl.slice(afterPathStart); + if (afterPath.length <= this._url_shortening_limit) { + return originalUrl; + } + const truncatedAfterPath = afterPath.slice(0, this._url_shortening_limit); + const shortHash = createHash('md5') + .update(afterPath, 'utf8') + .digest('hex') + .slice(0, 7); + const shortened = `${baseUrl}${truncatedAfterPath}...${shortHash}`; + if (shortened.length >= originalUrl.length) { + return originalUrl; + } + replacedUrls[shortened] = originalUrl; + return shortened; + }); + return [shortenedText, replacedUrls]; + } + _process_messages_and_replace_long_urls_shorter_ones(inputMessages) { + const urlsReplaced = {}; + for (const message of inputMessages) { + if (!message || typeof message !== 'object') { + continue; + } + const role = message.role; + const isUserOrAssistant = message instanceof UserMessage || + message instanceof AssistantMessage || + role === 'user' || + role === 'assistant'; + if (!isUserOrAssistant) { + continue; + } + if (typeof message.content === 'string') { + const [updated, replaced] = this._replace_urls_in_text(message.content); + message.content = updated; + Object.assign(urlsReplaced, replaced); + continue; + } + if (!Array.isArray(message.content)) { + continue; + } + for (const part of message.content) { + if (!part || typeof part !== 'object') { + continue; + } + const isTextPart = part instanceof ContentPartTextParam || part.type === 'text'; + if (!isTextPart || typeof part.text !== 'string') { + continue; + } + const [updated, replaced] = this._replace_urls_in_text(part.text); + part.text = updated; + Object.assign(urlsReplaced, replaced); + } + } + return urlsReplaced; + } + _replace_shortened_urls_in_string(text, urlReplacements) { + let result = text; + for (const [shortenedUrl, originalUrl] of Object.entries(urlReplacements)) { + result = result.split(shortenedUrl).join(originalUrl); + } + return result; + } + _replace_shortened_urls_in_value(value, urlReplacements) { + if (typeof value === 'string') { + return this._replace_shortened_urls_in_string(value, urlReplacements); + } + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i += 1) { + value[i] = this._replace_shortened_urls_in_value(value[i], urlReplacements); + } + return value; + } + if (!value || typeof value !== 'object') { + return value; + } + for (const [key, nested] of Object.entries(value)) { + value[key] = + this._replace_shortened_urls_in_value(nested, urlReplacements); + } + return value; + } + _parseCompletionPayload(rawCompletion) { + let parsedCompletion = rawCompletion; + if (typeof parsedCompletion === 'string') { + let jsonText = this._removeThinkTags(parsedCompletion.trim()); + // Handle common markdown wrappers like ```json ... ``` + const fencedMatch = jsonText.match(/```(?:json)?\s*([\s\S]*?)```/i); + if (fencedMatch && fencedMatch[1]) { + jsonText = fencedMatch[1].trim(); + } + // If extra text surrounds JSON, try to isolate the first JSON object + const firstBrace = jsonText.indexOf('{'); + const lastBrace = jsonText.lastIndexOf('}'); + if (firstBrace !== -1 && lastBrace !== -1 && lastBrace > firstBrace) { + jsonText = jsonText.slice(firstBrace, lastBrace + 1); + } + try { + parsedCompletion = JSON.parse(jsonText); + } + catch (error) { + throw new Error(`Failed to parse LLM completion as JSON: ${String(error)}`, { cause: error }); + } + } + if (!parsedCompletion || typeof parsedCompletion !== 'object') { + throw new Error('Model completion must be a JSON object'); + } + return parsedCompletion; + } + _isModelActionMissing(actions) { + if (actions.length === 0) { + return true; + } + return actions.every((entry) => { + const candidate = entry && + typeof entry === 'object' && + typeof entry.model_dump === 'function' + ? entry.model_dump() + : entry; + if (!candidate || + typeof candidate !== 'object' || + Array.isArray(candidate)) { + return false; + } + return Object.keys(candidate).length === 0; + }); + } + _getOutputActionNames(doneOnly) { + const registryActions = this.controller.registry.get_all_actions(); + const modelForStep = doneOnly + ? this.DoneActionModel + : this.ActionModel; + const modelAvailableNames = modelForStep?.available_actions; + if (Array.isArray(modelAvailableNames) && modelAvailableNames.length > 0) { + const deduped = Array.from(new Set(modelAvailableNames.filter((name) => typeof name === 'string' && + name.trim().length > 0 && + registryActions.has(name)))); + if (deduped.length > 0) { + return deduped; + } + } + if (doneOnly && registryActions.has('done')) { + return ['done']; + } + return Array.from(registryActions.keys()); + } + _toStrictActionParamSchema(schema) { + if (schema instanceof z.ZodObject) { + return schema.strict(); + } + return schema; + } + _buildActionOutputSchema(doneOnly) { + const registryActions = this.controller.registry.get_all_actions(); + const actionSchemas = this._getOutputActionNames(doneOnly) + .map((actionName) => { + const actionInfo = registryActions.get(actionName); + if (!actionInfo) { + return null; + } + const paramSchema = this._toStrictActionParamSchema(actionInfo.paramSchema); + return z.object({ [actionName]: paramSchema }).strict(); + }) + .filter((schema) => schema != null); + if (actionSchemas.length === 0) { + const doneAction = registryActions.get('done'); + if (doneAction) { + const doneParams = this._toStrictActionParamSchema(doneAction.paramSchema); + return z.object({ done: doneParams }).strict(); + } + return z.object({ done: z.object({}).strict() }).strict(); + } + if (actionSchemas.length === 1) { + return actionSchemas[0]; + } + const [firstActionSchema, secondActionSchema, ...remainingActionSchemas] = actionSchemas; + return z.union([ + firstActionSchema, + secondActionSchema, + ...remainingActionSchemas, + ]); + } + _buildLlmOutputFormat(doneOnly) { + const schema = z.object({ + thinking: z.string().optional().nullable(), + evaluation_previous_goal: z.string().optional().nullable(), + memory: z.string().optional().nullable(), + next_goal: z.string().optional().nullable(), + current_plan_item: z.number().int().optional().nullable(), + plan_update: z.array(z.string()).optional().nullable(), + action: z + .array(this._buildActionOutputSchema(doneOnly)) + .optional() + .nullable(), + }); + const outputFormat = schema; + outputFormat.schema = schema; + return outputFormat; + } + async _get_model_output_with_retry(messages, signal = null) { + const urlReplacements = this._process_messages_and_replace_long_urls_shorter_ones(messages); + const invokeAndParse = async (inputMessages) => { + this._throwIfAborted(signal); + const outputFormat = this._buildLlmOutputFormat(this._enforceDoneOnlyForCurrentStep); + const completion = await this.llm.ainvoke(inputMessages, outputFormat, { + signal: signal ?? undefined, + session_id: this.session_id, + }); + this._throwIfAborted(signal); + const parsed = this._parseCompletionPayload(completion.completion); + if (Object.keys(urlReplacements).length) { + this._replace_shortened_urls_in_value(parsed, urlReplacements); + } + return parsed; + }; + const invokeAndParseWithFallback = async (inputMessages) => { + try { + return await invokeAndParse(inputMessages); + } + catch (error) { + if ((error instanceof ModelRateLimitError || + error instanceof ModelProviderError) && + this._try_switch_to_fallback_llm(error)) { + this._throwIfAborted(signal); + return await invokeAndParse(inputMessages); + } + throw error; + } + }; + let parsed_completion = await invokeAndParseWithFallback(messages); + let rawAction = Array.isArray(parsed_completion?.action) + ? parsed_completion.action + : []; + this.logger.debug(`✅ Step ${this.state.n_steps}: Got LLM response with ${rawAction.length} actions`); + if (this._isModelActionMissing(rawAction)) { + this._throwIfAborted(signal); + this.logger.warning('Model returned empty action. Retrying...'); + const clarificationMessage = new UserMessage('You forgot to return an action. Please respond with a valid JSON action according to the expected schema with your assessment and next actions.'); + parsed_completion = await invokeAndParseWithFallback([ + ...messages, + clarificationMessage, + ]); + rawAction = Array.isArray(parsed_completion?.action) + ? parsed_completion.action + : []; + if (this._isModelActionMissing(rawAction)) { + this.logger.warning('Model still returned empty after retry. Inserting safe noop action.'); + rawAction = [ + { + done: { + success: false, + text: 'No next action returned by LLM!', + }, + }, + ]; + } + } + const action = this._validateAndNormalizeActions(rawAction); + const toNullableString = (value) => typeof value === 'string' ? value : null; + const toNullableNumber = (value) => typeof value === 'number' && Number.isFinite(value) + ? Math.trunc(value) + : null; + const toNullablePlanUpdate = (value) => Array.isArray(value) + ? value.filter((item) => typeof item === 'string') + : null; + const AgentOutputModel = this._enforceDoneOnlyForCurrentStep + ? (this.DoneAgentOutput ?? this.AgentOutput ?? AgentOutput) + : (this.AgentOutput ?? AgentOutput); + return new AgentOutputModel({ + thinking: toNullableString(parsed_completion?.thinking), + evaluation_previous_goal: toNullableString(parsed_completion?.evaluation_previous_goal), + memory: toNullableString(parsed_completion?.memory), + next_goal: toNullableString(parsed_completion?.next_goal), + current_plan_item: toNullableNumber(parsed_completion?.current_plan_item), + plan_update: toNullablePlanUpdate(parsed_completion?.plan_update), + action, + }); + } + _try_switch_to_fallback_llm(error) { + if (this._using_fallback_llm) { + this.logger.warning(`⚠️ Fallback LLM also failed (${error.name}: ${error.message}), no more fallbacks available`); + return false; + } + const retryableStatusCodes = new Set([401, 402, 429, 500, 502, 503, 504]); + const statusCode = typeof error.statusCode === 'number' ? error.statusCode : null; + const isRetryable = error instanceof ModelRateLimitError || + (statusCode != null && retryableStatusCodes.has(statusCode)); + if (!isRetryable) { + return false; + } + if (!this._fallback_llm) { + this.logger.warning(`⚠️ LLM error (${error.name}: ${error.message}) but no fallback_llm configured`); + return false; + } + this._log_fallback_switch(error, this._fallback_llm); + this.llm = this._fallback_llm; + this._using_fallback_llm = true; + this.token_cost_service.register_llm(this._fallback_llm); + return true; + } + _log_fallback_switch(error, fallback) { + const originalModel = typeof this._original_llm?.model === 'string' + ? this._original_llm.model + : 'unknown'; + const fallbackModel = typeof fallback?.model === 'string' ? fallback.model : 'unknown'; + const errorType = error.name || 'Error'; + const statusCode = typeof error.statusCode === 'number' ? error.statusCode : 'N/A'; + this.logger.warning(`⚠️ Primary LLM (${originalModel}) failed with ${errorType} (status=${statusCode}), switching to fallback LLM (${fallbackModel})`); + } + _validateAndNormalizeActions(actions) { + const normalizedActions = []; + const registryActions = this.controller.registry.get_all_actions(); + const actionAliases = { + navigate: 'go_to_url', + input: 'input_text', + switch: 'switch_tab', + close: 'close_tab', + extract: 'extract_structured_data', + find_text: 'scroll_to_text', + dropdown_options: 'get_dropdown_options', + select_dropdown: 'select_dropdown_option', + replace_file: 'replace_file_str', + }; + const availableNames = new Set(); + const modelForStep = this._enforceDoneOnlyForCurrentStep + ? this.DoneActionModel + : this.ActionModel; + const modelAvailableNames = modelForStep?.available_actions; + if (Array.isArray(modelAvailableNames) && modelAvailableNames.length > 0) { + for (const actionName of modelAvailableNames) { + if (typeof actionName === 'string' && actionName.trim()) { + availableNames.add(actionName); + } + } + } + else { + for (const actionName of registryActions.keys()) { + availableNames.add(actionName); + } + } + for (let i = 0; i < actions.length; i++) { + const entry = actions[i]; + const candidate = entry && + typeof entry === 'object' && + typeof entry.model_dump === 'function' + ? entry.model_dump() + : entry; + if (!candidate || + typeof candidate !== 'object' || + Array.isArray(candidate)) { + throw new Error(`Invalid action at index ${i}: expected an object with exactly one action key`); + } + const actionObject = candidate; + const keys = Object.keys(actionObject); + if (keys.length !== 1) { + throw new Error(`Invalid action at index ${i}: expected exactly one action key, got ${keys.length}`); + } + const requestedActionName = keys[0]; + let actionName = requestedActionName; + if (!availableNames.has(actionName)) { + const aliasTarget = actionAliases[requestedActionName]; + if (aliasTarget && availableNames.has(aliasTarget)) { + actionName = aliasTarget; + } + } + if (!availableNames.has(actionName)) { + const available = Array.from(availableNames).sort().join(', '); + throw new Error(`Action '${requestedActionName}' is not available on the current page. Available actions: ${available}`); + } + const actionInfo = registryActions.get(actionName); + if (!actionInfo) { + throw new Error(`Action '${requestedActionName}' is not registered`); + } + const rawParams = (actionObject[requestedActionName] ?? + actionObject[actionName] ?? + {}); + const paramsResult = actionInfo.paramSchema.safeParse(rawParams); + if (!paramsResult.success) { + // Surface a human-readable issue list (zod v4 `prettifyError`) plus + // a corrective hint, rather than the default JSON dump of `.issues`. + // This Error propagates → `_handle_step_error` writes it into + // `state.last_result` → `create_state_messages` injects it into the + // next LLM turn, so the model knows exactly what shape it got wrong. + const pretty = z.prettifyError(paramsResult.error); + const sentParams = JSON.stringify(rawParams); + throw new Error(`Schema validation failed for action '${requestedActionName}'. ` + + `You sent: ${sentParams}. Issues:\n${pretty}\n` + + `Please retry with parameters matching the action's schema exactly.`); + } + normalizedActions.push(new modelForStep({ + [actionName]: paramsResult.data, + })); + } + if (normalizedActions.length === 0) { + throw new Error('Model output must contain at least one action'); + } + if (normalizedActions.length > this.settings.max_actions_per_step) { + this.logger.warning(`Model returned ${normalizedActions.length} actions, trimming to max_actions_per_step=${this.settings.max_actions_per_step}`); + return normalizedActions.slice(0, this.settings.max_actions_per_step); + } + return normalizedActions; + } + async _update_action_models_for_page(page) { + await this._updateActionModelsForPage(page); + } + async _check_and_update_downloads(context = '') { + if (!this.has_downloads_path || !this.browser_session) { + return; + } + try { + const current_downloads = Array.isArray(this.browser_session.downloaded_files) + ? [...this.browser_session.downloaded_files] + : []; + const changed = current_downloads.length !== this._last_known_downloads.length || + current_downloads.some((value, index) => value !== this._last_known_downloads[index]); + if (changed) { + this._update_available_file_paths(current_downloads); + this._last_known_downloads = current_downloads; + if (context) { + this.logger.debug(`📁 ${context}: Updated available files`); + } + } + } + catch (error) { + const errorType = error instanceof Error + ? error.name || 'Error' + : typeof error === 'object' && error !== null + ? (error.constructor?.name ?? 'Error') + : 'Error'; + const message = error instanceof Error ? error.message : String(error); + const errorContext = context ? ` ${context}` : ''; + this.logger.debug(`📁 Failed to check for downloads${errorContext}: ${errorType}: ${message}`); + } + } + _update_available_file_paths(downloads) { + if (!this.has_downloads_path) { + return; + } + const existing = this.available_file_paths + ? [...this.available_file_paths] + : []; + const known = new Set(existing); + const new_files = downloads.filter((pathValue) => !known.has(pathValue)); + if (new_files.length) { + const updated = existing.concat(new_files); + this.available_file_paths = updated; + this.logger.info(`📁 Added ${new_files.length} downloaded files to available_file_paths (total: ${updated.length} files)`); + for (const file_path of new_files) { + this.logger.info(`📄 New file available: ${file_path}`); + } + } + else { + this.logger.debug(`📁 No new downloads detected (tracking ${existing.length} files)`); + } + } + _log_step_context(current_page, browser_state_summary) { + const url = browser_state_summary?.url ?? ''; + const url_short = url.length > 50 ? `${url.slice(0, 50)}...` : url; + const interactive_count = browser_state_summary?.selector_map + ? Object.keys(browser_state_summary.selector_map).length + : 0; + this.logger.info('\n'); + this.logger.info(`📍 Step ${this.state.n_steps}:`); + this.logger.debug(`Evaluating page with ${interactive_count} interactive elements on: ${url_short}`); + } + _log_first_step_startup() { + if (this.history.history.length !== 0) { + return; + } + this.logger.info(`Starting a browser-use agent with version ${this.version}, with provider=${this.llm.provider ?? 'unknown'} and model=${this.llm.model}`); + } + _log_step_completion_summary(step_start_time, result) { + if (!result.length) { + return; + } + const step_duration = Date.now() / 1000 - step_start_time; + const action_count = result.length; + const success_count = result.filter((r) => !r.error).length; + const failure_count = action_count - success_count; + const success_indicator = success_count ? `✅ ${success_count}` : ''; + const failure_indicator = failure_count ? `❌ ${failure_count}` : ''; + const status_parts = [success_indicator, failure_indicator].filter(Boolean); + const status_str = status_parts.length ? status_parts.join(' | ') : '✅ 0'; + this.logger.info(`📍 Step ${this.state.n_steps}: Ran ${action_count} actions in ${step_duration.toFixed(2)}s: ${status_str}`); + } + _log_agent_event(max_steps, agent_run_error) { + if (!this.telemetry) { + return; + } + const token_summary = this.token_cost_service?.get_usage_tokens_for_model?.(this.llm.model) ?? { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + }; + const action_history_data = this.history.history.map((historyItem) => { + if (!historyItem.model_output) { + return null; + } + return historyItem.model_output.action.map((action) => { + if (typeof action?.model_dump === 'function') { + return action.model_dump({ exclude_unset: true }); + } + return action; + }); + }); + const final_result = this.history.final_result(); + const final_result_str = final_result != null ? JSON.stringify(final_result) : null; + const judgement_data = this.history.judgement(); + const judge_verdict = judgement_data && typeof judgement_data.verdict === 'boolean' + ? judgement_data.verdict + : null; + const judge_reasoning = judgement_data && typeof judgement_data.reasoning === 'string' + ? judgement_data.reasoning + : null; + const judge_failure_reason = judgement_data && typeof judgement_data.failure_reason === 'string' + ? judgement_data.failure_reason + : null; + const judge_reached_captcha = judgement_data && typeof judgement_data.reached_captcha === 'boolean' + ? judgement_data.reached_captcha + : null; + const judge_impossible_task = judgement_data && typeof judgement_data.impossible_task === 'boolean' + ? judgement_data.impossible_task + : null; + let cdpHost = null; + const cdpUrl = this.browser_session?.cdp_url; + if (typeof cdpUrl === 'string' && cdpUrl) { + try { + const parsed = new URL(cdpUrl); + cdpHost = parsed.hostname || cdpUrl; + } + catch { + cdpHost = cdpUrl; + } + } + this.telemetry.capture(new AgentTelemetryEvent({ + task: this.task, + model: this.llm.model, + model_provider: this.llm.provider ?? 'unknown', + max_steps: max_steps, + max_actions_per_step: this.settings.max_actions_per_step, + use_vision: this.settings.use_vision, + version: this.version, + source: this.source, + cdp_url: cdpHost, + agent_type: null, + action_errors: this.history.errors(), + action_history: action_history_data, + urls_visited: this.history.urls(), + steps: this.state.n_steps, + total_input_tokens: token_summary.prompt_tokens ?? 0, + total_output_tokens: token_summary.completion_tokens ?? 0, + prompt_cached_tokens: token_summary.prompt_cached_tokens ?? 0, + total_tokens: token_summary.total_tokens ?? 0, + total_duration_seconds: this.history.total_duration_seconds(), + success: this.history.is_successful(), + final_result_response: final_result_str, + error_message: agent_run_error, + judge_verdict, + judge_reasoning, + judge_failure_reason, + judge_reached_captcha, + judge_impossible_task, + })); + } + async _make_history_item(model_output, browser_state_summary, result, metadata, state_message = null) { + const interacted_elements = model_output + ? AgentHistory.get_interacted_element(model_output, browser_state_summary.selector_map) + : []; + const state = new BrowserStateHistory(browser_state_summary.url, browser_state_summary.title, browser_state_summary.tabs, interacted_elements, this._current_screenshot_path); + this.history.add_item(new AgentHistory(model_output, result, state, metadata, state_message)); + } + save_file_system_state() { + if (!this.file_system) { + this.logger.error('💾 File system is not set up. Cannot save state.'); + throw new Error('File system is not set up. Cannot save state.'); + } + this.state.file_system_state = this.file_system.get_state(); + } +} diff --git a/dist/agent/system_prompt.md b/dist/agent/system_prompt.md new file mode 100644 index 00000000..22b18a03 --- /dev/null +++ b/dist/agent/system_prompt.md @@ -0,0 +1,269 @@ +You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in . + +You excel at following tasks: +1. Navigating complex websites and extracting precise information +2. Automating form submissions and interactive web actions +3. Gathering and saving information +4. Using your filesystem effectively to decide what to keep in your context +5. Operate effectively in an agent loop +6. Efficiently performing diverse web tasks + + +- Default working language: **English** +- Always respond in the same language as the user request + + +At every step, your input will consist of: +1. : A chronological event stream including your previous actions and their results. +2. : Current , summary of , , and . +3. : Current URL, open tabs, interactive elements indexed for actions, and visible page content. +4. : Screenshot of the browser with bounding boxes around interactive elements. If you used screenshot before, this will contain a screenshot. +5. This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step. + + +Agent history will be given as a list of step information as follows: +: +Evaluation of Previous Step: Assessment of last action +Memory: Your memory of this step +Next Goal: Your goal for this step +Action Results: Your actions and their results + +and system messages wrapped in tag. + + +USER REQUEST: This is your ultimate objective and always remains visible. +- This has the highest priority. Make the user happy. +- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps. +- If the task is open ended you can plan yourself how to get it done. + + +1. Browser State will be given as: +Current URL: URL of the page you are currently viewing. +Open Tabs: Open tabs with their ids. +Interactive Elements: All interactive elements will be provided in a tree-style XML format: +- Format: `[index]` for interactive elements +- Text content appears as child nodes on separate lines (not inside tags) +- Indentation with tabs shows parent/child relationships +Examples: +[33]
+ User form + [35] + *[38] +Note that: +- Only elements with numeric indexes in [] are interactive +- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above +- Elements tagged with a star `*[` are the new interactive elements that appeared since the last step +- Pure text elements without [] are not interactive +- The index numbers may change between steps as the page updates + + +If you used screenshot before, you will be provided with a screenshot of the current page with bounding boxes around interactive elements. This is your GROUND TRUTH: use it to evaluate your progress. +If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot. +Use screenshot if you are unsure or simply want more information about the current page state. +The screenshot shows exactly what a human user would see, making it invaluable for understanding complex layouts, images, or visual content. + +You must call the AgentOutput tool with the following schema for the arguments: + +{{ + "memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its obvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.", + "action": [ + {{ + "action_name": {{ + "parameter1": "value1", + "parameter2": "value2" + }} + }} + ] +}} + +Always put `memory` field before the `action` field. + + +Your memory field should include your reasoning. Apply these patterns: +- Did the previous action succeed? Verify using screenshot as ground truth. +- What is the current state relative to the user request? +- Are there any obstacles (popups, login walls)? CAPTCHAs are solved automatically. +- What specific next step will make progress toward the goal? +- If stuck, what alternative approach should you try? +- What information should be remembered for later steps? +Never assume an action succeeded just because you attempted it. Always verify from the screenshot or browser state. +Track important data points like prices, names, counts, and URLs that will be needed later. + + +Here are examples of good output patterns. Use them as reference but never copy them directly. + +"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison." +"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports." +"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter." +"memory": "Popup appeared blocking the page. Need to close it first before continuing with search." +"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead." +"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation." +"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying." +"memory": "Form submission failed - screenshot shows error message about invalid email format. Need to correct the email field." +"memory": "Successfully added item to cart. Screenshot confirms cart count is now 1. Next step is to proceed to checkout." +"memory": "Dropdown menu appeared after clicking. Need to select the 'Electronics' category from the options shown." +"memory": "Page loaded but content is different from expected. URL shows login redirect. Will look for alternative access or report limitation." +"memory": "Scrolled through first 10 results, found 3 matching items. Need to continue scrolling to find more options." + + + "write_file": {{ + "file_name": "todo.md", + "content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion" + }} + + + +Common actions you can use: +- navigate: Go to a specific URL +- click: Click on an element by index +- input: Type text into an input field +- scroll: Scroll the page up or down +- wait: Wait for the page to load +- extract: Extract structured information from the page +- screenshot: Take a screenshot for visual verification +- switch_tab: Switch between browser tabs +- go_back: Navigate back in browser history +- done: Complete the task and report results +- write_file: Write content to a file +- read_file: Read content from a file +- replace_file_str: Replace text in a file +Each action has specific parameters - refer to the action schema for details. + + +When encountering errors or unexpected states: +1. First, verify the current state using screenshot as ground truth +2. Check if a popup, modal, or overlay is blocking interaction +3. If an element is not found, scroll to reveal more content +4. If an action fails repeatedly (2-3 times), try an alternative approach +5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically. +6. If the page structure is different than expected, re-analyze and adapt +7. If stuck in a loop, explicitly acknowledge it in memory and change strategy +8. If max_steps is approaching, prioritize completing the most important parts of the task + + +1. ALWAYS verify action success using the screenshot before proceeding +2. ALWAYS handle popups/modals/cookie banners before other actions +3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.) +4. NEVER repeat the same failing action more than 2-3 times - try alternatives +5. NEVER assume success - always verify from screenshot or browser state +6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying +7. Put ALL relevant findings in done action's text field +8. Match user's requested output format exactly +9. Track progress in memory to avoid loops +10. When at max_steps, call done with whatever results you have +11. Always compare current trajectory against the user's original request +12. Be efficient - combine actions when possible but verify results between major steps + diff --git a/dist/agent/system_prompt_browser_use.md b/dist/agent/system_prompt_browser_use.md new file mode 100644 index 00000000..372771e0 --- /dev/null +++ b/dist/agent/system_prompt_browser_use.md @@ -0,0 +1,18 @@ +You are a browser-use agent operating in thinking mode. You automate browser tasks by outputting structured JSON actions. + + +Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative. + + + +You must ALWAYS respond with a valid JSON in this exact format: +{{ + "thinking": "A structured reasoning block analyzing: current page state, what was attempted, what worked/failed, and strategic planning for next steps.", + "evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.", + "memory": "1-3 sentences of specific memory of this step and overall progress. Track items found, pages visited, forms filled, etc.", + "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.", + "action": [{{"action_name": {{...params...}}}}] +}} +Action list should NEVER be empty. +DATA GROUNDING: Only report data observed in browser state or tool outputs. Never fabricate URLs, prices, or values. If not found, say so. + diff --git a/dist/agent/system_prompt_browser_use_flash.md b/dist/agent/system_prompt_browser_use_flash.md new file mode 100644 index 00000000..0a04cf5e --- /dev/null +++ b/dist/agent/system_prompt_browser_use_flash.md @@ -0,0 +1,15 @@ +You are a browser-use agent operating in flash mode. You automate browser tasks by outputting structured JSON actions. + + +Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative. + + + +You must respond with a valid JSON in this exact format: +{{ + "memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer.", + "action": [{{"action_name": {{...params...}}}}] +}} +Action list should NEVER be empty. +DATA GROUNDING: Only report data observed in browser state or tool outputs. Never fabricate URLs, prices, or values. If not found, say so. + diff --git a/dist/agent/system_prompt_browser_use_no_thinking.md b/dist/agent/system_prompt_browser_use_no_thinking.md new file mode 100644 index 00000000..ca5be037 --- /dev/null +++ b/dist/agent/system_prompt_browser_use_no_thinking.md @@ -0,0 +1,17 @@ +You are a browser-use agent. You automate browser tasks by outputting structured JSON actions. + + +Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative. + + + +You must ALWAYS respond with a valid JSON in this exact format: +{{ + "evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.", + "memory": "1-3 sentences of specific memory of this step and overall progress. Track items found, pages visited, forms filled, etc.", + "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.", + "action": [{{"action_name": {{...params...}}}}] +}} +Action list should NEVER be empty. +DATA GROUNDING: Only report data observed in browser state or tool outputs. Never fabricate URLs, prices, or values. If not found, say so. + diff --git a/dist/agent/system_prompt_flash.md b/dist/agent/system_prompt_flash.md new file mode 100644 index 00000000..706c6362 --- /dev/null +++ b/dist/agent/system_prompt_flash.md @@ -0,0 +1,16 @@ +You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in . +Default: English. Match user's language. +Ultimate objective. Specific tasks: follow each step. Open-ended: plan approach. +Elements: [index]text. Only [indexed] are interactive. Indentation=child. *[=new. +- PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. When writing CSV, use double quotes for commas. In available_file_paths, you can read downloaded files and user attachment files. + +You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred. + +You must respond with a valid JSON in this exact format: +{{ + "memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its opvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.", + "action":[{{"navigate": {{ "url": "url_value"}}}}] +}} +Before calling `done` with `success=true`: re-read the user request, verify every requirement is met (correct count, filters applied, format matched), confirm actions actually completed via page state/screenshot, and ensure no data was fabricated. If anything is unmet or uncertain, set `success` to `false`. BLOCKING ERROR CHECK: if you encountered an unresolved blocking error (payment declined, login failed with no credentials, email verification wall, access denied not bypassed, required paywall) you MUST set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups) do not count. +DATA GROUNDING: Only report data observed in browser state or tool outputs. Never fabricate URLs, prices, or values — including "representative" ones. If not found, say so. + diff --git a/dist/agent/system_prompt_flash_anthropic.md b/dist/agent/system_prompt_flash_anthropic.md new file mode 100644 index 00000000..fa0291b1 --- /dev/null +++ b/dist/agent/system_prompt_flash_anthropic.md @@ -0,0 +1,30 @@ +You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in . + +User request is the ultimate objective. For tasks with specific instructions, follow each step. For open-ended tasks, plan your own approach. + + +Elements: [index]text. Only [indexed] are interactive. Indentation=child. *[=new. + + +PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking and saving data. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. In available_file_paths, you can read downloaded files and user attachment files. + + +You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred. + +You must call the AgentOutput tool with the following schema for the arguments: + +{{ + "memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its obvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.", + "action": [ + {{ + "action_name": {{ + "parameter1": "value1", + "parameter2": "value2" + }} + }} + ] +}} + +Always put `memory` field before the `action` field. +Before calling `done` with `success=true`: re-read the user request, verify every requirement is met (correct count, filters applied, format matched), confirm actions actually completed via page state/screenshot, and ensure no data was fabricated. If anything is unmet or uncertain, set `success` to `false`. + diff --git a/dist/agent/system_prompt_no_thinking.md b/dist/agent/system_prompt_no_thinking.md new file mode 100644 index 00000000..8371fa1e --- /dev/null +++ b/dist/agent/system_prompt_no_thinking.md @@ -0,0 +1,245 @@ +You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in . + +You excel at following tasks: +1. Navigating complex websites and extracting precise information +2. Automating form submissions and interactive web actions +3. Gathering and saving information +4. Using your filesystem effectively to decide what to keep in your context +5. Operate effectively in an agent loop +6. Efficiently performing diverse web tasks + + +- Default working language: **English** +- Always respond in the same language as the user request + + +At every step, your input will consist of: +1. : A chronological event stream including your previous actions and their results. +2. : Current , summary of , , and . +3. : Current URL, open tabs, interactive elements indexed for actions, and visible page content. +4. : Screenshot of the browser with bounding boxes around interactive elements. If you used screenshot before, this will contain a screenshot. +5. This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step. + + +Agent history will be given as a list of step information as follows: +: +Evaluation of Previous Step: Assessment of last action +Memory: Your memory of this step +Next Goal: Your goal for this step +Action Results: Your actions and their results + +and system messages wrapped in tag. + + +USER REQUEST: This is your ultimate objective and always remains visible. +- This has the highest priority. Make the user happy. +- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps. +- If the task is open ended you can plan yourself how to get it done. + + +1. Browser State will be given as: +Current URL: URL of the page you are currently viewing. +Open Tabs: Open tabs with their ids. +Interactive Elements: All interactive elements will be provided in format as [index]text where +- index: Numeric identifier for interaction +- type: HTML element type (button, input, etc.) +- text: Element description +Examples: +[33]
User form
+\t*[35] +Note that: +- Only elements with numeric indexes in [] are interactive +- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index) +- Elements tagged with a star `*[` are the new interactive elements that appeared on the website since the last step - if url has not changed. Your previous actions caused that change. Think if you need to interact with them, e.g. after input you might need to select the right option from the list. +- Pure text elements without [] are not interactive. + + +If you used screenshot before, you will be provided with a screenshot of the current page with bounding boxes around interactive elements. This is your GROUND TRUTH: reason about the image in your thinking to evaluate your progress. +If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot. +Use screenshot if you are unsure or simply want more information. + + +Strictly follow these rules while using the browser and navigating the web: +- Only interact with elements that have a numeric [index] assigned. +- Only use indexes that are explicitly provided. +- If research is needed, open a **new tab** instead of reusing the current one. +- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list. +- By default, only elements in the visible viewport are listed. +- CAPTCHAs are automatically solved by the browser. If you encounter a CAPTCHA, it will be handled for you and you will be notified of the result. Do not attempt to solve CAPTCHAs manually — just continue with your task after the CAPTCHA is resolved. +- If the page is not fully loaded, use the wait action. +- You can call extract on specific pages to gather structured semantic information from the entire page, including parts not currently visible. +- Call extract only if the information you are looking for is not visible in your otherwise always just use the needed text from the . +- Calling the extract tool is expensive! DO NOT query the same page with the same extract query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool. +- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field. +- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step. +- If the includes specific page information such as product type, rating, price, location, etc., ALWAYS look for filter/sort options FIRST before browsing results. Apply all relevant filters before scrolling through results. +- The is the ultimate goal. If the user specifies explicit steps, they have always the highest priority. +- If you input into a field, you might need to press enter, click the search button, or select from dropdown for completion. +- For autocomplete/combobox fields (e.g. search boxes with suggestions, fields with role="combobox"): type your search text, then WAIT for the suggestions dropdown to appear in the next step. If suggestions appear (new elements marked with *[), click the correct one instead of pressing Enter. If no suggestions appear after one step, you may press Enter or submit normally. +- Don't login into a page if you don't have to. Don't login if you don't have the credentials. +- There are 2 types of tasks always first think which type of request you are dealing with: +1. Very specific step by step instructions: +- Follow them as very precise and don't skip steps. Try to complete everything as requested. +2. Open ended tasks. Plan yourself, be creative in achieving them. +- If you get stuck e.g. with logins in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search. CAPTCHAs are handled automatically. +- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in . You can either read the file or scroll in the page to see more. +- Handle popups, modals, cookie banners, and overlays immediately before attempting other actions. Look for close buttons (X, Close, Dismiss, No thanks, Skip) or accept/reject options. If a popup blocks interaction with the main page, handle it first. +- If you encounter access denied (403), bot detection, or rate limiting, do NOT repeatedly retry the same URL. Try alternative approaches or report the limitation. +- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without meaningful progress, or the same action fails 2-3 times, try a different approach. Track what you have tried in memory to avoid repeating failed approaches. + + +- You have access to a persistent file system which you can use to track progress, store results, and manage long tasks. +- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks. Use `replace_file` tool to update markers in `todo.md` as first action whenever you complete an item. This file should guide your step-by-step execution when you have a long running task. +- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas. +- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary. +- If exists, includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access. +- If the task is really long, initialize a `results.md` file to accumulate your results. +- DO NOT use the file system if the task is less than 10 steps! + + +Decide whether to plan based on task complexity: +- Simple task (1-3 actions, e.g. "go to X and click Y"): Act directly. Do NOT output `plan_update`. +- Complex but clear task (multi-step, known approach): Output `plan_update` immediately with 3-10 todo items. +- Complex and unclear task (unfamiliar site, vague goal): Explore for a few steps first, then output `plan_update` once you understand the landscape. +When a plan exists, `` in your input shows status markers: [x]=done, [>]=current, [ ]=pending, [-]=skipped. +Output `current_plan_item` (0-indexed) to indicate which item you are working on. +Output `plan_update` again only to revise the plan after unexpected obstacles or after exploration. +Completing all plan items does NOT mean the task is done. Always verify against the original before calling `done`. + + +You must call the `done` action in one of two cases: +- When you have fully completed the USER REQUEST. +- When you reach the final allowed step (`max_steps`), even if the task is incomplete. +- If it is ABSOLUTELY IMPOSSIBLE to continue. +The `done` action is your opportunity to terminate and share your findings with the user. +- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components. +- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`. +- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`. +- Put ALL the relevant information you found so far in the `text` field when you call `done` action. +- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST. +- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions. +- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer. +- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task! + +BEFORE calling `done` with `success=true`, you MUST perform this verification: +1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply). +2. **Check each requirement against your results:** + - Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them) + - Did you apply ALL specified filters/criteria? (e.g., price range, date, location) + - Does your output match the requested format exactly? +3. **Verify actions actually completed:** + - If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened. + - If you took a screenshot or downloaded a file — verify it exists in your file system. +4. **Verify data grounding:** Every URL, price, name, and value must be observed in your tool outputs, browser_state, or browser_vision (screenshot). Derived values (counts, totals, computed results) from observed data are allowed. Never fabricate URLs, invent values, or use "representative" placeholders — if not found, say so. +5. **Blocking error check:** If you hit an unresolved blocker (payment declined, login failed without credentials, email/verification wall, required paywall, access denied not bypassed) → set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups, retried errors) do NOT count. +6. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.** + Partial results with `success=false` are more valuable than overclaiming success. + + + +- You are allowed to use a maximum of {max_actions} actions per step. +If you are allowed multiple actions, you can specify multiple actions in the list to be executed sequentially (one after another). +- If the page changes after an action, the sequence is interrupted and you get the new state. You can see this in your agent history when this happens. +Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred. + + +You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page. +**Recommended Action Combinations:** +- `input` + `click` → Fill form field and submit/search in one step +- `input` + `input` → Fill multiple form fields +- `click` + `click` → Navigate through multi-step flows (when the page does not navigate between clicks) +- File operations + browser actions +Do not try multiple different paths in one step. Always have one clear goal per step. +Its important that you see in the next step if your action was successful, so do not chain actions which change the browser state multiple times, e.g. +- do not use click and then navigate, because you would not see if the click was successful or not. +- or do not use switch and switch together, because you would not see the state in between. +- do not use input and then scroll, because you would not see if the input was successful or not. + + +Be clear and concise in your decision-making. Exhibit the following reasoning patterns to successfully achieve the : +- Reason about to track progress and context toward . +- Analyze the most recent "Next Goal" and "Action Result" in and clearly state what you previously tried to achieve. +- Analyze all relevant items in , , , , and the screenshot to understand your state. +- Explicitly judge success/failure/uncertainty of the last action. Never assume an action succeeded just because it appears to be executed in your last step in . For example, you might have "Action 1/1: Input '2025-05-05' into element 3." in your history even though inputting text failed. Always verify using (screenshot) as the primary ground truth. If a screenshot is unavailable, fall back to . If the expected change is missing, mark the last action as failed (or uncertain) and plan a recovery. +- If todo.md is empty and the task is multi-step, generate a stepwise plan in todo.md using file tools. +- Analyze `todo.md` to guide and track your progress. +- If any todo.md items are finished, mark them as complete in the file. +- Analyze whether you are stuck, e.g. when you repeat the same actions multiple times without any progress. Then consider alternative approaches. +- Analyze the where one-time information are displayed due to your previous action. Reason about whether you want to keep this information in memory and plan writing them into a file if applicable using the file tools. +- If you see information relevant to , plan saving the information into a file. +- Before writing data into a file, analyze the and check if the file already has some content to avoid overwriting. +- Decide what concise, actionable context should be stored in memory to inform future reasoning. +- When ready to finish, state you are preparing to call done and communicate completion/results to the user. +- Before done, use read_file to verify file contents intended for user output. +- Always reason about the . Make sure to carefully analyze the specific steps and information required. E.g. specific filters, specific form fields, specific information to search. Make sure to always compare the current trajectory with the user request. + + +Here are examples of good output patterns. Use them as reference but never copy them directly. + + "write_file": {{ + "file_name": "todo.md", + "content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion" + }} + + +- Positive Examples: +"evaluation_previous_goal": "Successfully navigated to the product page and found the target information. Verdict: Success" +"evaluation_previous_goal": "Clicked the login button and user authentication form appeared. Verdict: Success" +- Negative Examples: +"evaluation_previous_goal": "Failed to input text into the search bar as I cannot see it in the image. Verdict: Failure" +"evaluation_previous_goal": "Clicked the submit button with index 15 but the form was not submitted successfully. Verdict: Failure" + + +"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison." +"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports." +"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter." +"memory": "Popup appeared blocking the page. Need to close it first before continuing with search." +"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead." +"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation." +"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying." + + +"next_goal": "Click on the 'Add to Cart' button to proceed with the purchase flow." +"next_goal": "Extract details from the first item on the page." +"next_goal": "Close the popup that appeared blocking the main content." +"next_goal": "Apply price filter to narrow results to items under $50." + + + +You must ALWAYS respond with a valid JSON in this exact format: +{{ + "evaluation_previous_goal": "One-sentence analysis of your last action. Clearly state success, failure, or uncertain.", + "memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.", + "next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.", + "current_plan_item": 0, + "plan_update": ["Todo item 1", "Todo item 2", "Todo item 3"], + "action":[{{"navigate": {{ "url": "url_value"}}}}, // ... more actions in sequence] +}} +Action list should NEVER be empty. +`current_plan_item` and `plan_update` are optional. See for details. + + +1. ALWAYS verify action success using the screenshot before proceeding +2. ALWAYS handle popups/modals/cookie banners before other actions +3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.) +4. NEVER repeat the same failing action more than 2-3 times - try alternatives +5. NEVER assume success - always verify from screenshot or browser state +6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying +7. Put ALL relevant findings in done action's text field +8. Match user's requested output format exactly +9. Track progress in memory to avoid loops +10. When at max_steps, call done with whatever results you have +11. Always compare current trajectory against the user's original request +12. Be efficient - combine actions when possible but verify results between major steps + + +When encountering errors or unexpected states: +1. First, verify the current state using screenshot as ground truth +2. Check if a popup, modal, or overlay is blocking interaction +3. If an element is not found, scroll to reveal more content +4. If an action fails repeatedly (2-3 times), try an alternative approach +5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically. +6. If the page structure is different than expected, re-analyze and adapt +7. If stuck in a loop, explicitly acknowledge it in memory and change strategy +8. If max_steps is approaching, prioritize completing the most important parts of the task + diff --git a/dist/agent/variable-detector.d.ts b/dist/agent/variable-detector.d.ts new file mode 100644 index 00000000..6faf6101 --- /dev/null +++ b/dist/agent/variable-detector.d.ts @@ -0,0 +1,12 @@ +import type { AgentHistoryList, DetectedVariable } from './views.js'; +import type { DOMHistoryElement } from '../dom/history-tree-processor/view.js'; +export declare const detect_variables_in_history: (history: AgentHistoryList | { + history: any[]; +}) => Record; +export declare const substitute_in_dict: (data: Record, replacements: Record) => number; +export declare const _private_for_tests: { + detectFromAttributes: (attributes: Record) => [string, string | null] | null; + detectFromValuePattern: (value: string) => [string, string | null] | null; + detectVariableType: (value: string, element: DOMHistoryElement | null) => [string, string | null] | null; + ensureUniqueName: (baseName: string, existing: Record) => string; +}; diff --git a/dist/agent/variable-detector.js b/dist/agent/variable-detector.js new file mode 100644 index 00000000..7f272fcc --- /dev/null +++ b/dist/agent/variable-detector.js @@ -0,0 +1,211 @@ +import { DetectedVariable as DetectedVariableModel } from './views.js'; +const ACTION_FIELDS_TO_CHECK = ['text', 'query']; +const getActionPayload = (action) => { + if (action && typeof action.model_dump === 'function') { + return action.model_dump(); + } + if (action && typeof action === 'object' && !Array.isArray(action)) { + return action; + } + return {}; +}; +const ensureUniqueName = (baseName, existing) => { + if (!(baseName in existing)) { + return baseName; + } + let counter = 2; + while (`${baseName}_${counter}` in existing) { + counter += 1; + } + return `${baseName}_${counter}`; +}; +const detectFromAttributes = (attributes) => { + const inputType = String(attributes.type ?? '').toLowerCase(); + if (inputType === 'email') + return ['email', 'email']; + if (inputType === 'tel') + return ['phone', 'phone']; + if (inputType === 'date') + return ['date', 'date']; + if (inputType === 'number') + return ['number', 'number']; + if (inputType === 'url') + return ['url', 'url']; + const semanticFields = [ + attributes.id ?? '', + attributes.name ?? '', + attributes.placeholder ?? '', + attributes['aria-label'] ?? '', + ]; + const combined = semanticFields.join(' ').toLowerCase(); + if (['address', 'street', 'addr'].some((needle) => combined.includes(needle))) { + if (combined.includes('billing')) + return ['billing_address', null]; + if (combined.includes('shipping')) + return ['shipping_address', null]; + return ['address', null]; + } + if (['comment', 'note', 'message', 'description'].some((needle) => combined.includes(needle))) { + return ['comment', null]; + } + if (combined.includes('email') || combined.includes('e-mail')) { + return ['email', 'email']; + } + if (['phone', 'tel', 'mobile', 'cell'].some((needle) => combined.includes(needle))) { + return ['phone', 'phone']; + } + if (combined.includes('first') && combined.includes('name')) { + return ['first_name', null]; + } + if (combined.includes('last') && combined.includes('name')) { + return ['last_name', null]; + } + if (combined.includes('full') && combined.includes('name')) { + return ['full_name', null]; + } + if (combined.includes('name')) { + return ['name', null]; + } + if (['date', 'dob', 'birth'].some((needle) => combined.includes(needle))) { + return ['date', 'date']; + } + if (combined.includes('city')) + return ['city', null]; + if (combined.includes('state') || combined.includes('province')) { + return ['state', null]; + } + if (combined.includes('country')) + return ['country', null]; + if (['zip', 'postal', 'postcode'].some((needle) => combined.includes(needle))) { + return ['zip_code', 'postal_code']; + } + if (combined.includes('company') || combined.includes('organization')) { + return ['company', null]; + } + return null; +}; +const detectFromValuePattern = (value) => { + if (/^[\w.-]+@[\w.-]+\.\w+$/.test(value)) { + return ['email', 'email']; + } + if (/^[\d\s\-()+]+$/.test(value)) { + const digitsOnly = value.replace(/[\s\-()+]/g, ''); + if (digitsOnly.length >= 10) { + return ['phone', 'phone']; + } + } + if (/^\d{4}-\d{2}-\d{2}$/.test(value)) { + return ['date', 'date']; + } + const stripped = value.replace(/[\s-]/g, ''); + if (value.length >= 2 && + value.length <= 30 && + value[0] === value[0].toUpperCase() && + /^[A-Za-z]+$/.test(stripped)) { + const words = value.trim().split(/\s+/); + if (words.length === 1) + return ['first_name', null]; + if (words.length === 2) + return ['full_name', null]; + return ['name', null]; + } + if (/^\d{1,9}$/.test(value)) { + return ['number', 'number']; + } + return null; +}; +const detectVariableType = (value, element) => { + const attrs = element?.attributes; + if (attrs && typeof attrs === 'object') { + const normalizedAttrs = Object.fromEntries(Object.entries(attrs).map(([key, attrValue]) => [key, String(attrValue)])); + const attrMatch = detectFromAttributes(normalizedAttrs); + if (attrMatch) { + return attrMatch; + } + } + return detectFromValuePattern(value); +}; +const detectInAction = (actionPayload, element, detected, detectedValues) => { + for (const params of Object.values(actionPayload)) { + if (!params || typeof params !== 'object' || Array.isArray(params)) { + continue; + } + for (const field of ACTION_FIELDS_TO_CHECK) { + const value = params[field]; + if (typeof value !== 'string' || !value.trim()) { + continue; + } + if (detectedValues.has(value)) { + continue; + } + const variableInfo = detectVariableType(value, element); + if (!variableInfo) { + continue; + } + const [baseName, format] = variableInfo; + const uniqueName = ensureUniqueName(baseName, detected); + detected[uniqueName] = new DetectedVariableModel(uniqueName, value, 'string', format); + detectedValues.add(value); + } + } +}; +export const detect_variables_in_history = (history) => { + const detected = {}; + const detectedValues = new Set(); + const historyItems = Array.isArray(history?.history) + ? history.history + : []; + for (const historyItem of historyItems) { + const actions = historyItem?.model_output?.action; + if (!Array.isArray(actions)) { + continue; + } + const interactedElements = Array.isArray(historyItem?.state?.interacted_element) + ? historyItem.state.interacted_element + : []; + for (let actionIndex = 0; actionIndex < actions.length; actionIndex += 1) { + const actionPayload = getActionPayload(actions[actionIndex]); + const element = interactedElements[actionIndex] ?? null; + detectInAction(actionPayload, element, detected, detectedValues); + } + } + return detected; +}; +export const substitute_in_dict = (data, replacements) => { + let count = 0; + for (const [key, value] of Object.entries(data)) { + if (typeof value === 'string') { + if (value in replacements) { + data[key] = replacements[value]; + count += 1; + } + continue; + } + if (value && typeof value === 'object' && !Array.isArray(value)) { + count += substitute_in_dict(value, replacements); + continue; + } + if (Array.isArray(value)) { + for (let index = 0; index < value.length; index += 1) { + const item = value[index]; + if (typeof item === 'string') { + if (item in replacements) { + value[index] = replacements[item]; + count += 1; + } + continue; + } + if (item && typeof item === 'object' && !Array.isArray(item)) { + count += substitute_in_dict(item, replacements); + } + } + } + } + return count; +}; +export const _private_for_tests = { + detectFromAttributes, + detectFromValuePattern, + detectVariableType, + ensureUniqueName, +}; diff --git a/dist/agent/views.d.ts b/dist/agent/views.d.ts new file mode 100644 index 00000000..343ecff5 --- /dev/null +++ b/dist/agent/views.d.ts @@ -0,0 +1,961 @@ +import { ActionModel } from '../controller/registry/views.js'; +import { BrowserStateHistory } from '../browser/views.js'; +import type { DOMHistoryElement } from '../dom/history-tree-processor/view.js'; +import { type SelectorMap } from '../dom/views.js'; +import type { FileSystemState } from '../filesystem/file-system.js'; +import type { BaseChatModel } from '../llm/base.js'; +import { MessageManagerState } from './message-manager/views.js'; +import type { UsageSummary } from '../tokens/views.js'; +export { ActionModel }; +export interface StructuredOutputParser { + parse?: (input: string) => T; + model_validate_json?: (input: string) => T; + model_json_schema?: () => unknown; + schema?: unknown; +} +export interface ActionResultInit { + is_done?: boolean | null; + success?: boolean | null; + judgement?: Record | null; + error?: string | null; + attachments?: string[] | null; + images?: Array> | null; + metadata?: Record | null; + long_term_memory?: string | null; + extracted_content?: string | null; + include_extracted_content_only_once?: boolean; + include_in_memory?: boolean; +} +export declare class ActionResult { + is_done: boolean | null; + success: boolean | null; + judgement: Record | null; + error: string | null; + attachments: string[] | null; + images: Array> | null; + metadata: Record | null; + long_term_memory: string | null; + extracted_content: string | null; + include_extracted_content_only_once: boolean; + include_in_memory: boolean; + constructor(init?: ActionResultInit); + private validate; + toJSON(): { + is_done: boolean | null; + success: boolean | null; + judgement: Record | null; + error: string | null; + attachments: string[] | null; + images: Record[] | null; + metadata: Record | null; + long_term_memory: string | null; + extracted_content: string | null; + include_extracted_content_only_once: boolean; + include_in_memory: boolean; + }; + model_dump(): { + is_done: boolean | null; + success: boolean | null; + judgement: Record | null; + error: string | null; + attachments: string[] | null; + images: Record[] | null; + metadata: Record | null; + long_term_memory: string | null; + extracted_content: string | null; + include_extracted_content_only_once: boolean; + include_in_memory: boolean; + }; + model_dump_json(): string; +} +export declare class PageFingerprint { + readonly url: string; + readonly element_count: number; + readonly text_hash: string; + constructor(url: string, element_count: number, text_hash: string); + static from_browser_state(url: string, dom_text: string, element_count: number): PageFingerprint; + equals(other: PageFingerprint): boolean; +} +export declare const compute_action_hash: (action_name: string, params: Record) => string; +export declare class ActionLoopDetector { + window_size: number; + recent_action_hashes: string[]; + recent_page_fingerprints: PageFingerprint[]; + max_repetition_count: number; + most_repeated_hash: string | null; + consecutive_stagnant_pages: number; + constructor(init?: Partial); + record_action(action_name: string, params: Record): void; + record_page_state(url: string, dom_text: string, element_count: number): void; + private update_repetition_stats; + get_nudge_message(): string | null; +} +export interface MessageCompactionSettings { + enabled: boolean; + compact_every_n_steps: number; + trigger_char_count: number | null; + trigger_token_count: number | null; + chars_per_token: number; + keep_last_items: number; + summary_max_chars: number; + include_read_state: boolean; + compaction_llm: BaseChatModel | null; +} +export declare const defaultMessageCompactionSettings: () => MessageCompactionSettings; +export declare const normalizeMessageCompactionSettings: (settings: Partial | MessageCompactionSettings) => MessageCompactionSettings; +export interface AgentSettings { + session_attachment_mode: 'copy' | 'strict' | 'shared'; + use_vision: boolean | 'auto'; + include_recent_events: boolean; + vision_detail_level: 'auto' | 'low' | 'high'; + save_conversation_path: string | null; + save_conversation_path_encoding: string | null; + max_failures: number; + generate_gif: boolean | string; + override_system_message: string | null; + extend_system_message: string | null; + include_attributes: string[]; + max_actions_per_step: number; + use_thinking: boolean; + flash_mode: boolean; + use_judge: boolean; + ground_truth: string | null; + max_history_items: number | null; + page_extraction_llm: unknown | null; + enable_planning: boolean; + planning_replan_on_stall: number; + planning_exploration_limit: number; + calculate_cost: boolean; + include_tool_call_examples: boolean; + llm_timeout: number; + step_timeout: number; + final_response_after_failure: boolean; + message_compaction: MessageCompactionSettings | null; + loop_detection_window: number; + loop_detection_enabled: boolean; +} +export declare const defaultAgentSettings: () => AgentSettings; +export declare class AgentState { + agent_id: string; + n_steps: number; + consecutive_failures: number; + last_result: ActionResult[] | null; + last_plan: string | null; + plan: PlanItem[] | null; + current_plan_item_index: number; + plan_generation_step: number | null; + last_model_output: AgentOutput | null; + paused: boolean; + stopped: boolean; + session_initialized: boolean; + follow_up_task: boolean; + message_manager_state: MessageManagerState; + file_system_state: FileSystemState | null; + loop_detector: ActionLoopDetector; + constructor(init?: Partial); + model_dump(): Record; + toJSON(): Record; +} +export declare class AgentStepInfo { + step_number: number; + max_steps: number; + constructor(step_number: number, max_steps: number); + is_last_step(): boolean; +} +export declare class StepMetadata { + step_start_time: number; + step_end_time: number; + step_number: number; + step_interval: number | null; + constructor(step_start_time: number, step_end_time: number, step_number: number, step_interval?: number | null); + get duration_seconds(): number; +} +export type PlanItemStatus = 'pending' | 'current' | 'done' | 'skipped'; +export declare class PlanItem { + text: string; + status: PlanItemStatus; + constructor(init?: Partial); + model_dump(): { + text: string; + status: PlanItemStatus; + }; +} +export interface AgentBrain { + thinking: string | null; + evaluation_previous_goal: string; + memory: string; + next_goal: string; +} +export declare class AgentOutput { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + constructor(init?: Partial); + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + static fromJSON(data: any): AgentOutput; + static type_with_custom_actions(custom_actions: new (initialData?: Record) => T): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T_1): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T_1): /*elided*/ any; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T_2): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T_1): /*elided*/ any; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T_2): /*elided*/ any; + }; + }; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T_1): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T_2): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T_2): /*elided*/ any; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T_1): /*elided*/ any; + }; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T_1): /*elided*/ any; + }; + }; + static type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T_1): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T_1): /*elided*/ any; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T_2): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T_1): /*elided*/ any; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T_2): /*elided*/ any; + }; + }; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T_1): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T_2): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T_2): /*elided*/ any; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T_1): /*elided*/ any; + }; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T_1): /*elided*/ any; + }; + }; + static type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T_1): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T_1): /*elided*/ any; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T_2): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T_1): /*elided*/ any; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T_2): /*elided*/ any; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + }; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + }; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T_1): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T_2): { + new (init?: Partial): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: ActionModel[]; + get current_state(): AgentBrain; + model_dump(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + model_dump_json(): string; + toJSON(): { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + }; + }; + fromJSON(data: any): AgentOutput; + type_with_custom_actions(custom_actions: new (initialData?: Record) => T_2): /*elided*/ any; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T_1): /*elided*/ any; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + }; + type_with_custom_actions_no_thinking(custom_actions: new (initialData?: Record) => T_1): /*elided*/ any; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + }; + type_with_custom_actions_flash_mode(custom_actions: new (initialData?: Record) => T): /*elided*/ any; + }; +} +export declare class AgentHistory { + model_output: AgentOutput | null; + result: ActionResult[]; + state: BrowserStateHistory; + metadata: StepMetadata | null; + state_message: string | null; + constructor(model_output: AgentOutput | null, result: ActionResult[], state: BrowserStateHistory, metadata?: StepMetadata | null, state_message?: string | null); + static get_interacted_element(model_output: AgentOutput, selector_map: SelectorMap): (DOMHistoryElement | null)[]; + private static _filterSensitiveDataFromString; + private static _filterSensitiveDataFromDict; + toJSON(sensitive_data?: Record> | null): { + model_output: { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + } | null; + result: { + is_done: boolean | null; + success: boolean | null; + judgement: Record | null; + error: string | null; + attachments: string[] | null; + images: Record[] | null; + metadata: Record | null; + long_term_memory: string | null; + extracted_content: string | null; + include_extracted_content_only_once: boolean; + include_in_memory: boolean; + }[]; + state: { + tabs: import("../index.js").TabInfo[]; + screenshot_path: string | null; + interacted_element: ({ + tag_name: string; + xpath: string; + highlight_index: number | null; + entire_parent_branch_path: string[]; + attributes: Record; + shadow_root: boolean; + css_selector: string | null; + page_coordinates: import("../index.js").CoordinateSet | null; + viewport_coordinates: import("../index.js").CoordinateSet | null; + viewport_info: import("../index.js").ViewportInfo | null; + element_hash: string | null; + stable_hash: string | null; + ax_name: string | null; + } | null)[]; + url: string; + title: string; + }; + metadata: { + step_start_time: number; + step_end_time: number; + step_number: number; + step_interval: number | null; + } | null; + state_message: string | null; + }; +} +export declare class AgentHistoryList { + history: AgentHistory[]; + usage: UsageSummary | null; + _output_model_schema: StructuredOutputParser | null; + constructor(history?: AgentHistory[], usage?: UsageSummary | null); + total_duration_seconds(): number; + add_item(history_item: AgentHistory): void; + last_action(): any; + errors(): (string | null)[]; + final_result(): string | null; + is_done(): boolean; + is_successful(): boolean | null; + judgement(): Record | null; + is_judged(): boolean; + is_validated(): boolean | null; + has_errors(): boolean; + urls(): string[]; + screenshot_paths(n_last?: number | null, return_none_if_not_screenshot?: boolean): (string | null)[]; + screenshots(n_last?: number | null, return_none_if_not_screenshot?: boolean): (string | null)[]; + action_names(): string[]; + model_thoughts(): AgentBrain[]; + model_outputs(): AgentOutput[]; + model_actions(): Record[]; + action_history(): Record[][]; + action_results(): ActionResult[]; + extracted_content(): (string | null)[]; + model_actions_filtered(include?: string[]): Record[]; + number_of_steps(): number; + agent_steps(): string[]; + get structured_output(): TStructured | null; + get_structured_output(outputModel: StructuredOutputParser): TStructured | null; + save_to_file(filepath: string, sensitive_data?: Record> | null): void; + static load_from_file(filepath: string, outputModel: typeof AgentOutput): AgentHistoryList; + static load_from_dict(payload: Record, outputModel: typeof AgentOutput): AgentHistoryList; + toJSON(sensitive_data?: Record> | null): { + history: { + model_output: { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + } | null; + result: { + is_done: boolean | null; + success: boolean | null; + judgement: Record | null; + error: string | null; + attachments: string[] | null; + images: Record[] | null; + metadata: Record | null; + long_term_memory: string | null; + extracted_content: string | null; + include_extracted_content_only_once: boolean; + include_in_memory: boolean; + }[]; + state: { + tabs: import("../index.js").TabInfo[]; + screenshot_path: string | null; + interacted_element: ({ + tag_name: string; + xpath: string; + highlight_index: number | null; + entire_parent_branch_path: string[]; + attributes: Record; + shadow_root: boolean; + css_selector: string | null; + page_coordinates: import("../index.js").CoordinateSet | null; + viewport_coordinates: import("../index.js").CoordinateSet | null; + viewport_info: import("../index.js").ViewportInfo | null; + element_hash: string | null; + stable_hash: string | null; + ax_name: string | null; + } | null)[]; + url: string; + title: string; + }; + metadata: { + step_start_time: number; + step_end_time: number; + step_number: number; + step_interval: number | null; + } | null; + state_message: string | null; + }[]; + }; + model_dump(sensitive_data?: Record> | null): { + history: { + model_output: { + thinking: string | null; + evaluation_previous_goal: string | null; + memory: string | null; + next_goal: string | null; + current_plan_item: number | null; + plan_update: string[] | null; + action: any[]; + } | null; + result: { + is_done: boolean | null; + success: boolean | null; + judgement: Record | null; + error: string | null; + attachments: string[] | null; + images: Record[] | null; + metadata: Record | null; + long_term_memory: string | null; + extracted_content: string | null; + include_extracted_content_only_once: boolean; + include_in_memory: boolean; + }[]; + state: { + tabs: import("../index.js").TabInfo[]; + screenshot_path: string | null; + interacted_element: ({ + tag_name: string; + xpath: string; + highlight_index: number | null; + entire_parent_branch_path: string[]; + attributes: Record; + shadow_root: boolean; + css_selector: string | null; + page_coordinates: import("../index.js").CoordinateSet | null; + viewport_coordinates: import("../index.js").CoordinateSet | null; + viewport_info: import("../index.js").ViewportInfo | null; + element_hash: string | null; + stable_hash: string | null; + ax_name: string | null; + } | null)[]; + url: string; + title: string; + }; + metadata: { + step_start_time: number; + step_end_time: number; + step_number: number; + step_interval: number | null; + } | null; + state_message: string | null; + }[]; + }; +} +export declare class DetectedVariable { + name: string; + original_value: string; + type: string; + format: string | null; + constructor(name: string, original_value: string, type?: string, format?: string | null); + model_dump(): { + name: string; + original_value: string; + type: string; + format: string | null; + }; +} +export declare class VariableMetadata { + detected_variables: Record; + constructor(detected_variables?: Record); +} +export declare class AgentError extends Error { + static VALIDATION_ERROR: string; + static RATE_LIMIT_ERROR: string; + static NO_VALID_ACTION: string; + static format_error(error: Error, include_trace?: boolean): string; +} diff --git a/dist/agent/views.js b/dist/agent/views.js new file mode 100644 index 00000000..5c01ceb0 --- /dev/null +++ b/dist/agent/views.js @@ -0,0 +1,951 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import { ActionModel } from '../controller/registry/views.js'; +import { BrowserStateHistory } from '../browser/views.js'; +import { HistoryTreeProcessor } from '../dom/history-tree-processor/service.js'; +import { DEFAULT_INCLUDE_ATTRIBUTES, } from '../dom/views.js'; +import { MessageManagerState } from './message-manager/views.js'; +// Re-export ActionModel for agent/service.ts +export { ActionModel }; +const parseStructuredOutput = (schema, value) => { + if (!schema) { + return null; + } + if (schema.parse) { + return schema.parse(value); + } + if (schema.model_validate_json) { + return schema.model_validate_json(value); + } + return null; +}; +export class ActionResult { + is_done; + success; + judgement; + error; + attachments; + images; + metadata; + long_term_memory; + extracted_content; + include_extracted_content_only_once; + include_in_memory; + constructor(init = {}) { + this.is_done = init.is_done ?? false; + this.success = init.success ?? null; + this.judgement = init.judgement ?? null; + this.error = init.error ?? null; + this.attachments = init.attachments ?? null; + this.images = init.images ?? null; + this.metadata = init.metadata ?? null; + this.long_term_memory = init.long_term_memory ?? null; + this.extracted_content = init.extracted_content ?? null; + this.include_extracted_content_only_once = + init.include_extracted_content_only_once ?? false; + this.include_in_memory = init.include_in_memory ?? false; + this.validate(); + } + validate() { + if (this.success === true && this.is_done !== true) { + throw new Error('success=True can only be set when is_done=True. For regular actions that succeed, leave success as None. Use success=False only for actions that fail.'); + } + } + toJSON() { + return { + is_done: this.is_done, + success: this.success, + judgement: this.judgement, + error: this.error, + attachments: this.attachments, + images: this.images, + metadata: this.metadata, + long_term_memory: this.long_term_memory, + extracted_content: this.extracted_content, + include_extracted_content_only_once: this.include_extracted_content_only_once, + include_in_memory: this.include_in_memory, + }; + } + model_dump() { + return this.toJSON(); + } + model_dump_json() { + return JSON.stringify(this.toJSON()); + } +} +export class PageFingerprint { + url; + element_count; + text_hash; + constructor(url, element_count, text_hash) { + this.url = url; + this.element_count = element_count; + this.text_hash = text_hash; + } + static from_browser_state(url, dom_text, element_count) { + const text_hash = createHash('sha256') + .update(dom_text, 'utf8') + .digest('hex') + .slice(0, 16); + return new PageFingerprint(url, element_count, text_hash); + } + equals(other) { + return (this.url === other.url && + this.element_count === other.element_count && + this.text_hash === other.text_hash); + } +} +const stableSerialize = (value) => { + if (value === null || value === undefined) { + return ''; + } + if (typeof value !== 'object') { + return String(value); + } + if (Array.isArray(value)) { + return `[${value.map((item) => stableSerialize(item)).join(',')}]`; + } + const entries = Object.entries(value) + .filter(([, entryValue]) => entryValue !== null && entryValue !== undefined) + .sort(([a], [b]) => a.localeCompare(b)); + return `{${entries + .map(([key, entryValue]) => `${key}:${stableSerialize(entryValue)}`) + .join(',')}}`; +}; +const normalizeActionForHash = (action_name, params) => { + if (action_name === 'search' || action_name === 'search_google') { + const query = String(params.query ?? ''); + const tokens = Array.from(new Set(query + .toLowerCase() + .replace(/[^\w\s]/g, ' ') + .split(/\s+/) + .filter(Boolean))).sort(); + const engine = typeof params.engine === 'string' && params.engine.trim() + ? params.engine.trim().toLowerCase() + : action_name === 'search_google' + ? 'google' + : 'google'; + return `search|${engine}|${tokens.join('|')}`; + } + if (action_name === 'click' || + action_name === 'click_element' || + action_name === 'click_element_by_index') { + return `click|${String(params.index ?? '')}`; + } + if (action_name === 'input' || action_name === 'input_text') { + const text = String(params.text ?? '') + .trim() + .toLowerCase(); + return `input|${String(params.index ?? '')}|${text}`; + } + if (action_name === 'navigate' || action_name === 'go_to_url') { + return `navigate|${String(params.url ?? '')}`; + } + if (action_name.startsWith('scroll')) { + const direction = typeof params.down === 'boolean' + ? params.down + ? 'down' + : 'up' + : action_name.includes('up') + ? 'up' + : 'down'; + const index = String(params.index ?? ''); + return `scroll|${direction}|${index}`; + } + return `${action_name}|${stableSerialize(params)}`; +}; +export const compute_action_hash = (action_name, params) => { + const normalized = normalizeActionForHash(action_name, params); + return createHash('sha256') + .update(normalized, 'utf8') + .digest('hex') + .slice(0, 12); +}; +export class ActionLoopDetector { + window_size; + recent_action_hashes; + recent_page_fingerprints; + max_repetition_count; + most_repeated_hash; + consecutive_stagnant_pages; + constructor(init) { + this.window_size = init?.window_size ?? 20; + this.recent_action_hashes = init?.recent_action_hashes ?? []; + this.recent_page_fingerprints = init?.recent_page_fingerprints ?? []; + this.max_repetition_count = init?.max_repetition_count ?? 0; + this.most_repeated_hash = init?.most_repeated_hash ?? null; + this.consecutive_stagnant_pages = init?.consecutive_stagnant_pages ?? 0; + } + record_action(action_name, params) { + const hash = compute_action_hash(action_name, params); + this.recent_action_hashes.push(hash); + if (this.recent_action_hashes.length > this.window_size) { + this.recent_action_hashes = this.recent_action_hashes.slice(-this.window_size); + } + this.update_repetition_stats(); + } + record_page_state(url, dom_text, element_count) { + const fp = PageFingerprint.from_browser_state(url, dom_text, element_count); + const last = this.recent_page_fingerprints.at(-1); + if (last && last.equals(fp)) { + this.consecutive_stagnant_pages += 1; + } + else { + this.consecutive_stagnant_pages = 0; + } + this.recent_page_fingerprints.push(fp); + if (this.recent_page_fingerprints.length > 5) { + this.recent_page_fingerprints = this.recent_page_fingerprints.slice(-5); + } + } + update_repetition_stats() { + if (!this.recent_action_hashes.length) { + this.max_repetition_count = 0; + this.most_repeated_hash = null; + return; + } + const counts = new Map(); + for (const hash of this.recent_action_hashes) { + counts.set(hash, (counts.get(hash) ?? 0) + 1); + } + let maxHash = null; + let maxCount = 0; + for (const [hash, count] of counts.entries()) { + if (count > maxCount) { + maxHash = hash; + maxCount = count; + } + } + this.most_repeated_hash = maxHash; + this.max_repetition_count = maxCount; + } + get_nudge_message() { + const messages = []; + if (this.max_repetition_count >= 12) { + messages.push(`Heads up: you have repeated a similar action ${this.max_repetition_count} times in the last ${this.recent_action_hashes.length} actions. If you are making progress with each repetition, keep going. If not, a different approach might get you there faster.`); + } + else if (this.max_repetition_count >= 8) { + messages.push(`Heads up: you have repeated a similar action ${this.max_repetition_count} times in the last ${this.recent_action_hashes.length} actions. Are you still making progress with each attempt? If so, carry on. Otherwise, it might be worth trying a different approach.`); + } + else if (this.max_repetition_count >= 5) { + messages.push(`Heads up: you have repeated a similar action ${this.max_repetition_count} times in the last ${this.recent_action_hashes.length} actions. If this is intentional and making progress, carry on. If not, it might be worth reconsidering your approach.`); + } + if (this.consecutive_stagnant_pages >= 5) { + messages.push(`The page content has not changed across ${this.consecutive_stagnant_pages} consecutive actions. Your actions might not be having the intended effect. It could be worth trying a different element or approach.`); + } + if (!messages.length) { + return null; + } + return messages.join('\n\n'); + } +} +export const defaultMessageCompactionSettings = () => ({ + enabled: true, + compact_every_n_steps: 15, + trigger_char_count: 40000, + trigger_token_count: null, + chars_per_token: 4, + keep_last_items: 6, + summary_max_chars: 6000, + include_read_state: false, + compaction_llm: null, +}); +export const normalizeMessageCompactionSettings = (settings) => { + const merged = { + ...defaultMessageCompactionSettings(), + ...settings, + }; + if (merged.trigger_char_count != null && merged.trigger_token_count != null) { + throw new Error('Set trigger_char_count or trigger_token_count for message_compaction, not both.'); + } + if (merged.trigger_token_count != null) { + merged.trigger_char_count = Math.floor(merged.trigger_token_count * merged.chars_per_token); + } + else if (merged.trigger_char_count == null) { + merged.trigger_char_count = 40000; + } + return merged; +}; +export const defaultAgentSettings = () => ({ + session_attachment_mode: 'copy', + use_vision: true, + include_recent_events: false, + vision_detail_level: 'auto', + save_conversation_path: null, + save_conversation_path_encoding: 'utf-8', + max_failures: 3, + generate_gif: false, + override_system_message: null, + extend_system_message: null, + include_attributes: [...DEFAULT_INCLUDE_ATTRIBUTES], + max_actions_per_step: 5, + use_thinking: true, + flash_mode: false, + use_judge: true, + ground_truth: null, + max_history_items: null, + page_extraction_llm: null, + enable_planning: true, + planning_replan_on_stall: 3, + planning_exploration_limit: 5, + calculate_cost: false, + include_tool_call_examples: false, + llm_timeout: 60, + step_timeout: 180, + final_response_after_failure: true, + message_compaction: null, + loop_detection_window: 20, + loop_detection_enabled: true, +}); +export class AgentState { + agent_id; + n_steps; + consecutive_failures; + last_result; + last_plan; + plan; + current_plan_item_index; + plan_generation_step; + last_model_output; + paused; + stopped; + session_initialized; + follow_up_task; + message_manager_state; + file_system_state; + loop_detector; + constructor(init) { + this.agent_id = init?.agent_id ?? ''; + this.n_steps = init?.n_steps ?? 1; + this.consecutive_failures = init?.consecutive_failures ?? 0; + this.last_result = init?.last_result ?? null; + this.last_plan = init?.last_plan ?? null; + this.plan = + init?.plan?.map((item) => item instanceof PlanItem ? item : new PlanItem(item)) ?? null; + this.current_plan_item_index = init?.current_plan_item_index ?? 0; + this.plan_generation_step = init?.plan_generation_step ?? null; + this.last_model_output = init?.last_model_output ?? null; + this.paused = init?.paused ?? false; + this.stopped = init?.stopped ?? false; + this.session_initialized = init?.session_initialized ?? false; + this.follow_up_task = init?.follow_up_task ?? false; + if (init?.message_manager_state instanceof MessageManagerState) { + this.message_manager_state = init.message_manager_state; + } + else if (init?.message_manager_state) { + this.message_manager_state = Object.assign(new MessageManagerState(), init.message_manager_state); + } + else { + this.message_manager_state = new MessageManagerState(); + } + this.file_system_state = init?.file_system_state ?? null; + if (init?.loop_detector instanceof ActionLoopDetector) { + this.loop_detector = init.loop_detector; + } + else if (init?.loop_detector) { + this.loop_detector = Object.assign(new ActionLoopDetector(), init.loop_detector); + } + else { + this.loop_detector = new ActionLoopDetector(); + } + } + model_dump() { + return { + agent_id: this.agent_id, + n_steps: this.n_steps, + consecutive_failures: this.consecutive_failures, + last_result: this.last_result?.map((result) => result.model_dump()) ?? null, + last_plan: this.last_plan, + plan: this.plan?.map((item) => item.model_dump()) ?? null, + current_plan_item_index: this.current_plan_item_index, + plan_generation_step: this.plan_generation_step, + last_model_output: this.last_model_output?.model_dump() ?? null, + paused: this.paused, + stopped: this.stopped, + session_initialized: this.session_initialized, + follow_up_task: this.follow_up_task, + message_manager_state: JSON.parse(JSON.stringify(this.message_manager_state)), + file_system_state: this.file_system_state, + loop_detector: JSON.parse(JSON.stringify(this.loop_detector)), + }; + } + toJSON() { + return this.model_dump(); + } +} +export class AgentStepInfo { + step_number; + max_steps; + constructor(step_number, max_steps) { + this.step_number = step_number; + this.max_steps = max_steps; + } + is_last_step() { + return this.step_number >= this.max_steps - 1; + } +} +export class StepMetadata { + step_start_time; + step_end_time; + step_number; + step_interval; + constructor(step_start_time, step_end_time, step_number, step_interval = null) { + this.step_start_time = step_start_time; + this.step_end_time = step_end_time; + this.step_number = step_number; + this.step_interval = step_interval; + } + get duration_seconds() { + return this.step_end_time - this.step_start_time; + } +} +export class PlanItem { + text; + status; + constructor(init) { + this.text = init?.text ?? ''; + this.status = init?.status ?? 'pending'; + } + model_dump() { + return { + text: this.text, + status: this.status, + }; + } +} +export class AgentOutput { + thinking; + evaluation_previous_goal; + memory; + next_goal; + current_plan_item; + plan_update; + action; + constructor(init) { + this.thinking = init?.thinking ?? null; + this.evaluation_previous_goal = init?.evaluation_previous_goal ?? null; + this.memory = init?.memory ?? null; + this.next_goal = init?.next_goal ?? null; + this.current_plan_item = init?.current_plan_item ?? null; + this.plan_update = init?.plan_update ?? null; + this.action = (init?.action ?? []).map((entry) => entry instanceof ActionModel ? entry : new ActionModel(entry)); + } + get current_state() { + return { + thinking: this.thinking, + evaluation_previous_goal: this.evaluation_previous_goal ?? '', + memory: this.memory ?? '', + next_goal: this.next_goal ?? '', + }; + } + model_dump() { + return { + thinking: this.thinking, + evaluation_previous_goal: this.evaluation_previous_goal, + memory: this.memory, + next_goal: this.next_goal, + current_plan_item: this.current_plan_item, + plan_update: this.plan_update, + action: this.action.map((action) => action.model_dump?.() ?? action), + }; + } + model_dump_json() { + return JSON.stringify(this.model_dump()); + } + toJSON() { + return this.model_dump(); + } + static fromJSON(data) { + if (!data) { + return new AgentOutput(); + } + const actions = Array.isArray(data.action) + ? data.action.map((item) => new ActionModel(item)) + : []; + return new AgentOutput({ + thinking: data.thinking ?? null, + evaluation_previous_goal: data.evaluation_previous_goal ?? null, + memory: data.memory ?? null, + next_goal: data.next_goal ?? null, + current_plan_item: typeof data.current_plan_item === 'number' + ? data.current_plan_item + : null, + plan_update: Array.isArray(data.plan_update) + ? data.plan_update.filter((item) => typeof item === 'string') + : null, + action: actions, + }); + } + static type_with_custom_actions(custom_actions) { + const CustomActionModel = custom_actions; + return class AgentOutputWithCustomActions extends AgentOutput { + constructor(init) { + super(init); + this.action = (init?.action ?? []).map((entry) => entry instanceof CustomActionModel + ? entry + : new CustomActionModel(entry?.model_dump?.() ?? entry)); + } + }; + } + static type_with_custom_actions_no_thinking(custom_actions) { + const BaseModel = AgentOutput.type_with_custom_actions(custom_actions); + return class AgentOutputWithoutThinking extends BaseModel { + constructor(init) { + super(init); + this.thinking = null; + } + }; + } + static type_with_custom_actions_flash_mode(custom_actions) { + const BaseModel = AgentOutput.type_with_custom_actions(custom_actions); + return class AgentOutputFlashMode extends BaseModel { + constructor(init) { + super(init); + this.thinking = null; + this.evaluation_previous_goal = null; + this.next_goal = null; + this.current_plan_item = null; + this.plan_update = null; + } + }; + } +} +export class AgentHistory { + model_output; + result; + state; + metadata; + state_message; + constructor(model_output, result, state, metadata = null, state_message = null) { + this.model_output = model_output; + this.result = result; + this.state = state; + this.metadata = metadata; + this.state_message = state_message; + } + static get_interacted_element(model_output, selector_map) { + const elements = []; + for (const action of model_output.action) { + const index = typeof action.get_index === 'function' ? action.get_index() : null; + if (index != null && selector_map[index]) { + const node = selector_map[index]; + elements.push(HistoryTreeProcessor.convert_dom_element_to_history_element(node)); + } + else { + elements.push(null); + } + } + return elements; + } + static _filterSensitiveDataFromString(value, sensitive_data) { + if (!sensitive_data) { + return value; + } + const placeholders = {}; + for (const [keyOrDomain, content] of Object.entries(sensitive_data)) { + if (typeof content === 'string' && content) { + placeholders[keyOrDomain] = content; + } + else if (content && typeof content === 'object') { + for (const [key, val] of Object.entries(content)) { + if (val) { + placeholders[key] = val; + } + } + } + } + if (!Object.keys(placeholders).length) { + return value; + } + let filtered = value; + for (const [key, secret] of Object.entries(placeholders)) { + filtered = filtered.split(secret).join(`${key}`); + } + return filtered; + } + static _filterSensitiveDataFromDict(data, sensitive_data) { + if (!sensitive_data) { + return data; + } + const filtered = {}; + for (const [key, value] of Object.entries(data)) { + if (typeof value === 'string') { + filtered[key] = this._filterSensitiveDataFromString(value, sensitive_data); + } + else if (value && typeof value === 'object' && !Array.isArray(value)) { + filtered[key] = this._filterSensitiveDataFromDict(value, sensitive_data); + } + else if (Array.isArray(value)) { + filtered[key] = value.map((item) => { + if (typeof item === 'string') { + return this._filterSensitiveDataFromString(item, sensitive_data); + } + if (item && typeof item === 'object' && !Array.isArray(item)) { + return this._filterSensitiveDataFromDict(item, sensitive_data); + } + return item; + }); + } + else { + filtered[key] = value; + } + } + return filtered; + } + toJSON(sensitive_data = null) { + let modelOutput = this.model_output?.toJSON() ?? null; + if (modelOutput && + Array.isArray(modelOutput.action) && + sensitive_data) { + modelOutput.action = modelOutput.action.map((action) => Object.prototype.hasOwnProperty.call(action, 'input') + ? AgentHistory._filterSensitiveDataFromDict(action, sensitive_data) + : action); + } + return { + model_output: modelOutput, + result: this.result.map((r) => r.toJSON()), + state: this.state.to_dict(), + metadata: this.metadata + ? { + step_start_time: this.metadata.step_start_time, + step_end_time: this.metadata.step_end_time, + step_number: this.metadata.step_number, + step_interval: this.metadata.step_interval, + } + : null, + state_message: this.state_message, + }; + } +} +export class AgentHistoryList { + history; + usage; + _output_model_schema = null; + constructor(history = [], usage = null) { + this.history = history; + this.usage = usage ?? null; + } + total_duration_seconds() { + return this.history.reduce((sum, item) => sum + (item.metadata?.duration_seconds ?? 0), 0); + } + add_item(history_item) { + this.history.push(history_item); + } + last_action() { + if (!this.history.length) { + return null; + } + const last = this.history[this.history.length - 1]; + if (!last.model_output || !last.model_output.action.length) { + return null; + } + const action = last.model_output.action[last.model_output.action.length - 1]; + if (typeof action?.model_dump === 'function') { + return action.model_dump(); + } + return action; + } + errors() { + return this.history.map((historyItem) => { + const error = historyItem.result.find((result) => result.error); + return error?.error ?? null; + }); + } + final_result() { + if (!this.history.length) { + return null; + } + const last = this.history[this.history.length - 1]; + const result = last.result[last.result.length - 1]; + return result?.extracted_content ?? null; + } + is_done() { + if (!this.history.length) { + return false; + } + const last = this.history[this.history.length - 1]; + const result = last.result[last.result.length - 1]; + return result?.is_done === true; + } + is_successful() { + if (!this.history.length) { + return null; + } + const last = this.history[this.history.length - 1]; + const result = last.result[last.result.length - 1]; + if (result?.is_done) { + return result.success ?? null; + } + return null; + } + judgement() { + if (!this.history.length) { + return null; + } + const last = this.history[this.history.length - 1]; + const result = last.result[last.result.length - 1]; + if (result?.judgement && typeof result.judgement === 'object') { + return result.judgement; + } + return null; + } + is_judged() { + return this.judgement() != null; + } + is_validated() { + const judgement = this.judgement(); + if (!judgement) { + return null; + } + return judgement.verdict === true; + } + has_errors() { + return this.errors().some((error) => error != null); + } + urls() { + return this.history.map((item) => item.state.url ?? null); + } + screenshot_paths(n_last = null, return_none_if_not_screenshot = true) { + if (n_last === 0) { + return []; + } + const items = n_last == null ? this.history : this.history.slice(-n_last); + return items + .map((item) => item.state.screenshot_path ?? null) + .filter((pathValue) => return_none_if_not_screenshot || pathValue !== null); + } + screenshots(n_last = null, return_none_if_not_screenshot = true) { + if (n_last === 0) { + return []; + } + const items = n_last == null ? this.history : this.history.slice(-n_last); + const screenshots = []; + for (const item of items) { + const screenshot = item.state.get_screenshot(); + if (screenshot) { + screenshots.push(screenshot); + } + else if (return_none_if_not_screenshot) { + screenshots.push(null); + } + } + return screenshots; + } + action_names() { + const names = []; + for (const action of this.model_actions()) { + const [name] = Object.keys(action); + if (name) { + names.push(name); + } + } + return names; + } + model_thoughts() { + return this.history + .filter((item) => item.model_output) + .map((item) => item.model_output.current_state); + } + model_outputs() { + return (this.history + .filter((item) => item.model_output) + .map((item) => item.model_output) ?? []); + } + model_actions() { + const outputs = []; + for (const item of this.history) { + if (!item.model_output) { + continue; + } + const interacted = item.state.interacted_element ?? []; + for (let index = 0; index < item.model_output.action.length; index += 1) { + const action = item.model_output.action[index]; + const interactedElement = interacted[index] ?? null; + const payload = typeof action?.model_dump === 'function' + ? action.model_dump() + : action; + if (payload && typeof payload === 'object' && interactedElement) { + payload.interacted_element = + interactedElement; + } + else if (payload && typeof payload === 'object') { + payload.interacted_element = + interactedElement; + } + outputs.push(payload); + } + } + return outputs; + } + action_history() { + const history = []; + for (const item of this.history) { + const stepActions = []; + if (item.model_output) { + const interacted = item.state.interacted_element ?? []; + for (let index = 0; index < item.model_output.action.length; index += 1) { + const action = item.model_output.action[index]; + const interactedElement = interacted[index] ?? null; + const result = item.result[index]; + const payload = typeof action?.model_dump === 'function' + ? action.model_dump() + : action; + const enriched = payload && typeof payload === 'object' + ? { ...payload } + : { action: payload }; + enriched.interacted_element = interactedElement; + enriched.result = result?.long_term_memory ?? null; + stepActions.push(enriched); + } + } + history.push(stepActions); + } + return history; + } + action_results() { + return this.history.flatMap((item) => item.result); + } + extracted_content() { + return this.history.flatMap((item) => item.result.map((result) => result.extracted_content).filter(Boolean)); + } + model_actions_filtered(include = []) { + if (!include.length) { + return this.model_actions(); + } + return this.model_actions().filter((action) => { + const [name] = Object.keys(action); + return include.includes(name); + }); + } + number_of_steps() { + return this.history.length; + } + agent_steps() { + const steps = []; + for (let stepIndex = 0; stepIndex < this.history.length; stepIndex += 1) { + const historyItem = this.history[stepIndex]; + let stepText = `Step ${stepIndex + 1}:\n`; + if (historyItem.model_output?.action?.length) { + const actions = historyItem.model_output.action.map((action) => typeof action?.model_dump === 'function' + ? action.model_dump() + : action); + stepText += `Actions: ${JSON.stringify(actions, null, 1)}\n`; + } + if (historyItem.result?.length) { + for (let resultIndex = 0; resultIndex < historyItem.result.length; resultIndex += 1) { + const result = historyItem.result[resultIndex]; + if (result?.extracted_content) { + stepText += `Result ${resultIndex + 1}: ${String(result.extracted_content)}\n`; + } + if (result?.error) { + stepText += `Error ${resultIndex + 1}: ${String(result.error)}\n`; + } + } + } + steps.push(stepText); + } + return steps; + } + get structured_output() { + const final_result = this.final_result(); + if (!final_result || !this._output_model_schema) { + return null; + } + return parseStructuredOutput(this._output_model_schema, final_result); + } + get_structured_output(outputModel) { + const finalResult = this.final_result(); + if (!finalResult) { + return null; + } + return parseStructuredOutput(outputModel, finalResult); + } + save_to_file(filepath, sensitive_data = null) { + const dir = path.dirname(filepath); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(filepath, JSON.stringify(this.toJSON(sensitive_data), null, 2), 'utf-8'); + } + static load_from_file(filepath, outputModel) { + const content = fs.readFileSync(filepath, 'utf-8'); + const payload = JSON.parse(content); + return AgentHistoryList.load_from_dict(payload, outputModel); + } + static load_from_dict(payload, outputModel) { + const historyItems = (payload.history ?? []).map((entry) => { + const modelOutput = entry.model_output + ? outputModel.fromJSON(entry.model_output) + : null; + const result = (entry.result ?? []).map((item) => new ActionResult(item)); + const state = new BrowserStateHistory(entry.state?.url ?? '', entry.state?.title ?? '', entry.state?.tabs ?? [], entry.state?.interacted_element ?? [], entry.state?.screenshot_path ?? null); + const metadata = entry.metadata + ? new StepMetadata(entry.metadata.step_start_time, entry.metadata.step_end_time, entry.metadata.step_number, entry.metadata.step_interval ?? null) + : null; + return new AgentHistory(modelOutput, result, state, metadata, entry.state_message ?? null); + }); + return new AgentHistoryList(historyItems); + } + toJSON(sensitive_data = null) { + return { + history: this.history.map((item) => item.toJSON(sensitive_data)), + }; + } + model_dump(sensitive_data = null) { + return this.toJSON(sensitive_data); + } +} +export class DetectedVariable { + name; + original_value; + type; + format; + constructor(name, original_value, type = 'string', format = null) { + this.name = name; + this.original_value = original_value; + this.type = type; + this.format = format; + } + model_dump() { + return { + name: this.name, + original_value: this.original_value, + type: this.type, + format: this.format, + }; + } +} +export class VariableMetadata { + detected_variables; + constructor(detected_variables = {}) { + this.detected_variables = detected_variables; + } +} +export class AgentError extends Error { + static VALIDATION_ERROR = 'Invalid model output format. Please follow the correct schema.'; + static RATE_LIMIT_ERROR = 'Rate limit reached. Waiting before retry.'; + static NO_VALID_ACTION = 'No valid action found'; + static format_error(error, include_trace = false) { + if (error?.name === 'ValidationError') { + return `${AgentError.VALIDATION_ERROR}\nDetails: ${error.message}`; + } + if (error.name === 'RateLimitError') { + return AgentError.RATE_LIMIT_ERROR; + } + const errorStr = error.message ?? String(error); + if (errorStr.includes('LLM response missing required fields') || + errorStr.includes('Expected format: AgentOutput')) { + const [mainError] = errorStr.split('\n'); + let helpfulMessage = `${mainError}\n\n` + + 'The previous response had an invalid output structure. ' + + 'Please stick to the required output format. \n\n'; + if (include_trace && error?.stack) { + helpfulMessage += `\n\nFull stacktrace:\n${error.stack}`; + } + return helpfulMessage; + } + if (include_trace && error?.stack) { + return `${error.message}\nStacktrace:\n${error.stack}`; + } + return error.message; + } +} diff --git a/dist/browser/browser.d.ts b/dist/browser/browser.d.ts new file mode 100644 index 00000000..a5f3186e --- /dev/null +++ b/dist/browser/browser.d.ts @@ -0,0 +1,7 @@ +import { BrowserProfile } from './profile.js'; +import { BrowserSession } from './session.js'; +export type BrowserConfig = BrowserProfile; +export type BrowserContextConfig = BrowserProfile; +export declare const Browser: typeof BrowserSession; +export { BrowserProfile }; +export { BrowserSession }; diff --git a/dist/browser/browser.js b/dist/browser/browser.js new file mode 100644 index 00000000..5b275c67 --- /dev/null +++ b/dist/browser/browser.js @@ -0,0 +1,5 @@ +import { BrowserProfile } from './profile.js'; +import { BrowserSession } from './session.js'; +export const Browser = BrowserSession; +export { BrowserProfile }; +export { BrowserSession }; diff --git a/dist/browser/cloud/cloud.d.ts b/dist/browser/cloud/cloud.d.ts new file mode 100644 index 00000000..40d1ab9c --- /dev/null +++ b/dist/browser/cloud/cloud.d.ts @@ -0,0 +1,20 @@ +import { CloudBrowserResponse, type CreateBrowserRequest } from './views.js'; +export interface CloudBrowserClientOptions { + api_base_url?: string; + api_key?: string | null; + fetch_impl?: typeof fetch; +} +export declare class CloudBrowserClient { + private readonly api_base_url; + private readonly explicit_api_key; + private readonly fetch_impl; + current_session_id: string | null; + constructor(options?: CloudBrowserClientOptions); + private _resolve_api_key; + private _auth_headers; + private _create_request_body; + private _request_json; + create_browser(request: CreateBrowserRequest, extra_headers?: Record): Promise; + stop_browser(session_id?: string | null, extra_headers?: Record): Promise; + close(): Promise; +} diff --git a/dist/browser/cloud/cloud.js b/dist/browser/cloud/cloud.js new file mode 100644 index 00000000..d7394f0e --- /dev/null +++ b/dist/browser/cloud/cloud.js @@ -0,0 +1,129 @@ +import { CONFIG } from '../../config.js'; +import { createLogger } from '../../logging-config.js'; +import { DeviceAuthClient } from '../../sync/auth.js'; +import { CloudBrowserAuthError, CloudBrowserError, CloudBrowserResponse, MAX_PAID_USER_SESSION_TIMEOUT, } from './views.js'; +const logger = createLogger('browser_use.browser.cloud'); +const stripTrailingSlash = (input) => input.replace(/\/+$/, ''); +const normalizeTimeout = (timeout) => { + if (timeout == null) { + return null; + } + const integerTimeout = Math.floor(timeout); + if (!Number.isFinite(integerTimeout) || integerTimeout < 1) { + return null; + } + return Math.min(integerTimeout, MAX_PAID_USER_SESSION_TIMEOUT); +}; +export class CloudBrowserClient { + api_base_url; + explicit_api_key; + fetch_impl; + current_session_id = null; + constructor(options = {}) { + this.api_base_url = stripTrailingSlash(options.api_base_url ?? CONFIG.BROWSER_USE_CLOUD_API_URL); + this.explicit_api_key = options.api_key ?? null; + this.fetch_impl = options.fetch_impl ?? fetch; + } + _resolve_api_key() { + if (this.explicit_api_key && this.explicit_api_key.trim()) { + return this.explicit_api_key.trim(); + } + const fromEnv = process.env.BROWSER_USE_API_KEY?.trim(); + if (fromEnv) { + return fromEnv; + } + const authClient = new DeviceAuthClient(this.api_base_url); + const fromAuthConfig = authClient.api_token?.trim(); + if (fromAuthConfig) { + return fromAuthConfig; + } + return null; + } + _auth_headers(extra_headers = {}) { + const api_key = this._resolve_api_key(); + if (!api_key) { + throw new CloudBrowserAuthError('No authentication token found. Set BROWSER_USE_API_KEY to use cloud browser.'); + } + return { + 'X-Browser-Use-API-Key': api_key, + 'Content-Type': 'application/json', + ...extra_headers, + }; + } + _create_request_body(request) { + const profile_id = request.profile_id ?? request.cloud_profile_id ?? null; + const proxy_country_code = request.proxy_country_code ?? request.cloud_proxy_country_code ?? null; + const timeout = normalizeTimeout(request.timeout ?? request.cloud_timeout ?? null); + return { + ...(profile_id ? { profile_id: String(profile_id) } : {}), + ...(proxy_country_code + ? { proxy_country_code: String(proxy_country_code) } + : {}), + ...(timeout ? { timeout } : {}), + }; + } + async _request_json(path, init, extra_headers = {}) { + const response = await this.fetch_impl(`${this.api_base_url}${path}`, { + ...init, + headers: this._auth_headers(extra_headers), + }); + const text = await response.text(); + let payload = null; + if (text) { + try { + payload = JSON.parse(text); + } + catch { + payload = text; + } + } + if (!response.ok) { + const errorDetails = payload && typeof payload === 'object' + ? JSON.stringify(payload) + : String(payload ?? ''); + if (response.status === 401 || response.status === 403) { + throw new CloudBrowserAuthError(`Cloud browser authentication failed (${response.status})`); + } + throw new CloudBrowserError(`Cloud browser request failed (${response.status}): ${errorDetails}`); + } + return payload; + } + async create_browser(request, extra_headers = {}) { + logger.info('🌤️ Creating cloud browser instance...'); + const payload = await this._request_json('/api/v2/browsers', { + method: 'POST', + body: JSON.stringify(this._create_request_body(request)), + }, extra_headers); + const browser_response = new CloudBrowserResponse(payload); + this.current_session_id = browser_response.id; + logger.info(`🌤️ Cloud browser created: ${browser_response.id}`); + return browser_response; + } + async stop_browser(session_id = null, extra_headers = {}) { + const target_session_id = session_id ?? this.current_session_id; + if (!target_session_id) { + throw new CloudBrowserError('No session ID provided and no active cloud browser session found'); + } + const payload = await this._request_json(`/api/v2/browsers/${encodeURIComponent(target_session_id)}`, { + method: 'PATCH', + body: JSON.stringify({ action: 'stop' }), + }, extra_headers); + const browser_response = new CloudBrowserResponse(payload); + if (browser_response.id === this.current_session_id) { + this.current_session_id = null; + } + logger.info(`🌤️ Cloud browser stopped: ${browser_response.id}`); + return browser_response; + } + async close() { + if (!this.current_session_id) { + return; + } + try { + await this.stop_browser(this.current_session_id); + } + catch (error) { + logger.debug(`Failed to stop cloud browser during close: ${error.message}`); + } + } +} diff --git a/dist/browser/cloud/index.d.ts b/dist/browser/cloud/index.d.ts new file mode 100644 index 00000000..8925f590 --- /dev/null +++ b/dist/browser/cloud/index.d.ts @@ -0,0 +1,3 @@ +export * from './views.js'; +export * from './cloud.js'; +export * from './management.js'; diff --git a/dist/browser/cloud/index.js b/dist/browser/cloud/index.js new file mode 100644 index 00000000..8925f590 --- /dev/null +++ b/dist/browser/cloud/index.js @@ -0,0 +1,3 @@ +export * from './views.js'; +export * from './cloud.js'; +export * from './management.js'; diff --git a/dist/browser/cloud/management.d.ts b/dist/browser/cloud/management.d.ts new file mode 100644 index 00000000..f79f64c5 --- /dev/null +++ b/dist/browser/cloud/management.d.ts @@ -0,0 +1,130 @@ +export interface CloudManagementClientOptions { + api_base_url?: string; + api_key?: string | null; + fetch_impl?: typeof fetch; +} +export interface CloudTaskView { + id: string; + sessionId: string; + llm?: string | null; + task: string; + status: string; + createdAt: string; + startedAt?: string | null; + finishedAt?: string | null; + metadata?: Record | null; + output?: string | null; + browserUseVersion?: string | null; + isSuccess?: boolean | null; + judgement?: string | null; + judgeVerdict?: boolean | null; + steps?: Array>; + outputFiles?: Array>; +} +export interface CloudSessionView { + id: string; + status: string; + startedAt: string; + liveUrl?: string | null; + finishedAt?: string | null; + tasks?: CloudTaskView[]; + publicShareUrl?: string | null; +} +export interface CloudProfileView { + id: string; + createdAt: string; + updatedAt: string; + name?: string | null; + lastUsedAt?: string | null; + cookieDomains?: string[] | null; +} +export interface CloudShareView { + shareToken: string; + shareUrl: string; + viewCount: number; + lastViewedAt?: string | null; +} +export interface PaginatedResponse { + items: T[]; + totalItems: number; + pageNumber: number; + pageSize: number; +} +export interface CreateTaskRequest { + task: string; + llm?: string | null; + startUrl?: string | null; + maxSteps?: number | null; + structuredOutput?: string | null; + sessionId?: string | null; + metadata?: Record | null; + secrets?: Record | null; + allowedDomains?: string[] | null; + opVaultId?: string | null; + highlightElements?: boolean; + flashMode?: boolean; + thinking?: boolean; + vision?: boolean | 'auto' | null; + systemPromptExtension?: string | null; + judge?: boolean; + judgeGroundTruth?: string | null; + judgeLlm?: string | null; + skillIds?: string[] | null; +} +export interface CreateSessionRequest { + profileId?: string | null; + proxyCountryCode?: string | null; + startUrl?: string | null; + browserScreenWidth?: number | null; + browserScreenHeight?: number | null; +} +export declare class CloudManagementClient { + private readonly api_base_url; + private readonly explicit_api_key; + private readonly fetch_impl; + constructor(options?: CloudManagementClientOptions); + private resolve_api_key; + private auth_headers; + private request_json; + private build_query; + list_tasks(options?: { + pageSize?: number; + pageNumber?: number; + sessionId?: string | null; + filterBy?: string | null; + after?: string | null; + before?: string | null; + }): Promise>; + create_task(request: CreateTaskRequest): Promise<{ + id: string; + sessionId: string; + }>; + get_task(task_id: string): Promise; + update_task(task_id: string, action: 'stop' | 'stop_task_and_session'): Promise; + get_task_logs(task_id: string): Promise<{ + downloadUrl: string; + }>; + list_sessions(options?: { + pageSize?: number; + pageNumber?: number; + filterBy?: string | null; + }): Promise>; + create_session(request: CreateSessionRequest): Promise; + get_session(session_id: string): Promise; + update_session(session_id: string, action: 'stop'): Promise; + delete_session(session_id: string): Promise; + create_session_public_share(session_id: string): Promise; + delete_session_public_share(session_id: string): Promise; + list_profiles(options?: { + pageSize?: number; + pageNumber?: number; + }): Promise>; + create_profile(request?: { + name?: string | null; + }): Promise; + get_profile(profile_id: string): Promise; + update_profile(profile_id: string, request?: { + name?: string | null; + }): Promise; + delete_profile(profile_id: string): Promise; +} diff --git a/dist/browser/cloud/management.js b/dist/browser/cloud/management.js new file mode 100644 index 00000000..74dcd76a --- /dev/null +++ b/dist/browser/cloud/management.js @@ -0,0 +1,140 @@ +import { CONFIG } from '../../config.js'; +import { DeviceAuthClient } from '../../sync/auth.js'; +import { CloudBrowserAuthError, CloudBrowserError } from './views.js'; +const stripTrailingSlash = (input) => input.replace(/\/+$/, ''); +export class CloudManagementClient { + api_base_url; + explicit_api_key; + fetch_impl; + constructor(options = {}) { + this.api_base_url = stripTrailingSlash(options.api_base_url ?? CONFIG.BROWSER_USE_CLOUD_API_URL); + this.explicit_api_key = options.api_key ?? null; + this.fetch_impl = options.fetch_impl ?? fetch; + } + resolve_api_key() { + if (this.explicit_api_key?.trim()) { + return this.explicit_api_key.trim(); + } + if (process.env.BROWSER_USE_API_KEY?.trim()) { + return process.env.BROWSER_USE_API_KEY.trim(); + } + const savedToken = new DeviceAuthClient(this.api_base_url).api_token?.trim(); + return savedToken || null; + } + auth_headers(extra_headers = {}) { + const api_key = this.resolve_api_key(); + if (!api_key) { + throw new CloudBrowserAuthError('No authentication token found. Set BROWSER_USE_API_KEY to use cloud APIs.'); + } + return { + 'X-Browser-Use-API-Key': api_key, + 'Content-Type': 'application/json', + ...extra_headers, + }; + } + async request_json(path, init) { + const response = await this.fetch_impl(`${this.api_base_url}${path}`, { + ...init, + headers: this.auth_headers(init.headers), + }); + const text = await response.text(); + let payload = null; + if (text) { + try { + payload = JSON.parse(text); + } + catch { + payload = text; + } + } + if (!response.ok) { + const details = payload && typeof payload === 'object' + ? JSON.stringify(payload) + : String(payload ?? ''); + if (response.status === 401 || response.status === 403) { + throw new CloudBrowserAuthError(`Cloud API authentication failed (${response.status})`); + } + throw new CloudBrowserError(`Cloud API request failed (${response.status}): ${details}`); + } + return payload; + } + build_query(params) { + const query = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + if (value !== undefined && value !== null && String(value).length > 0) { + query.set(key, String(value)); + } + } + const rendered = query.toString(); + return rendered ? `?${rendered}` : ''; + } + async list_tasks(options = {}) { + return await this.request_json(`/api/v2/tasks${this.build_query(options)}`, { method: 'GET' }); + } + async create_task(request) { + return await this.request_json('/api/v2/tasks', { + method: 'POST', + body: JSON.stringify(request), + }); + } + async get_task(task_id) { + return await this.request_json(`/api/v2/tasks/${encodeURIComponent(task_id)}`, { method: 'GET' }); + } + async update_task(task_id, action) { + return await this.request_json(`/api/v2/tasks/${encodeURIComponent(task_id)}`, { + method: 'PATCH', + body: JSON.stringify({ action }), + }); + } + async get_task_logs(task_id) { + return await this.request_json(`/api/v2/tasks/${encodeURIComponent(task_id)}/logs`, { method: 'GET' }); + } + async list_sessions(options = {}) { + return await this.request_json(`/api/v2/sessions${this.build_query(options)}`, { method: 'GET' }); + } + async create_session(request) { + return await this.request_json('/api/v2/sessions', { + method: 'POST', + body: JSON.stringify(request), + }); + } + async get_session(session_id) { + return await this.request_json(`/api/v2/sessions/${encodeURIComponent(session_id)}`, { method: 'GET' }); + } + async update_session(session_id, action) { + return await this.request_json(`/api/v2/sessions/${encodeURIComponent(session_id)}`, { + method: 'PATCH', + body: JSON.stringify({ action }), + }); + } + async delete_session(session_id) { + await this.request_json(`/api/v2/sessions/${encodeURIComponent(session_id)}`, { method: 'DELETE' }); + } + async create_session_public_share(session_id) { + return await this.request_json(`/api/v2/sessions/${encodeURIComponent(session_id)}/public-share`, { method: 'POST' }); + } + async delete_session_public_share(session_id) { + await this.request_json(`/api/v2/sessions/${encodeURIComponent(session_id)}/public-share`, { method: 'DELETE' }); + } + async list_profiles(options = {}) { + return await this.request_json(`/api/v2/profiles${this.build_query(options)}`, { method: 'GET' }); + } + async create_profile(request = {}) { + return await this.request_json('/api/v2/profiles', { + method: 'POST', + body: JSON.stringify(request), + }); + } + async get_profile(profile_id) { + return await this.request_json(`/api/v2/profiles/${encodeURIComponent(profile_id)}`, { method: 'GET' }); + } + async update_profile(profile_id, request = {}) { + return await this.request_json(`/api/v2/profiles/${encodeURIComponent(profile_id)}`, { + method: 'PATCH', + body: JSON.stringify(request), + }); + } + async delete_profile(profile_id) { + await this.request_json(`/api/v2/profiles/${encodeURIComponent(profile_id)}`, { method: 'DELETE' }); + } +} diff --git a/dist/browser/cloud/views.d.ts b/dist/browser/cloud/views.d.ts new file mode 100644 index 00000000..ad596bd0 --- /dev/null +++ b/dist/browser/cloud/views.d.ts @@ -0,0 +1,41 @@ +export type ProxyCountryCode = 'us' | 'uk' | 'fr' | 'it' | 'jp' | 'au' | 'de' | 'fi' | 'ca' | 'in' | string; +export declare const MAX_FREE_USER_SESSION_TIMEOUT = 15; +export declare const MAX_PAID_USER_SESSION_TIMEOUT = 240; +export interface CreateBrowserRequest { + cloud_profile_id?: string | null; + cloud_proxy_country_code?: ProxyCountryCode | null; + cloud_timeout?: number | null; + profile_id?: string | null; + proxy_country_code?: ProxyCountryCode | null; + timeout?: number | null; +} +export interface CloudBrowserResponsePayload { + id: string; + status: string; + liveUrl?: string; + live_url?: string; + cdpUrl?: string; + cdp_url?: string; + timeoutAt?: string; + timeout_at?: string; + startedAt?: string; + started_at?: string; + finishedAt?: string | null; + finished_at?: string | null; +} +export declare class CloudBrowserResponse { + id: string; + status: string; + liveUrl: string; + cdpUrl: string; + timeoutAt: string; + startedAt: string; + finishedAt: string | null; + constructor(payload: CloudBrowserResponsePayload); +} +export declare class CloudBrowserError extends Error { + constructor(message: string); +} +export declare class CloudBrowserAuthError extends CloudBrowserError { + constructor(message: string); +} diff --git a/dist/browser/cloud/views.js b/dist/browser/cloud/views.js new file mode 100644 index 00000000..ff9a99c1 --- /dev/null +++ b/dist/browser/cloud/views.js @@ -0,0 +1,35 @@ +export const MAX_FREE_USER_SESSION_TIMEOUT = 15; +export const MAX_PAID_USER_SESSION_TIMEOUT = 240; +export class CloudBrowserResponse { + id; + status; + liveUrl; + cdpUrl; + timeoutAt; + startedAt; + finishedAt; + constructor(payload) { + if (!payload?.id || !payload?.status) { + throw new CloudBrowserError('Invalid cloud browser response: missing id or status'); + } + this.id = String(payload.id); + this.status = String(payload.status); + this.liveUrl = String(payload.liveUrl ?? payload.live_url ?? ''); + this.cdpUrl = String(payload.cdpUrl ?? payload.cdp_url ?? ''); + this.timeoutAt = String(payload.timeoutAt ?? payload.timeout_at ?? ''); + this.startedAt = String(payload.startedAt ?? payload.started_at ?? ''); + this.finishedAt = payload.finishedAt ?? payload.finished_at ?? null; + } +} +export class CloudBrowserError extends Error { + constructor(message) { + super(message); + this.name = 'CloudBrowserError'; + } +} +export class CloudBrowserAuthError extends CloudBrowserError { + constructor(message) { + super(message); + this.name = 'CloudBrowserAuthError'; + } +} diff --git a/dist/browser/context.d.ts b/dist/browser/context.d.ts new file mode 100644 index 00000000..cf837550 --- /dev/null +++ b/dist/browser/context.d.ts @@ -0,0 +1,8 @@ +import { BrowserProfile } from './profile.js'; +import { BrowserSession } from './session.js'; +export type Browser = BrowserSession; +export type BrowserConfig = BrowserProfile; +export type BrowserContext = BrowserSession; +export type BrowserContextConfig = BrowserProfile; +export { BrowserProfile }; +export { BrowserSession }; diff --git a/dist/browser/context.js b/dist/browser/context.js new file mode 100644 index 00000000..dd02a732 --- /dev/null +++ b/dist/browser/context.js @@ -0,0 +1,4 @@ +import { BrowserProfile } from './profile.js'; +import { BrowserSession } from './session.js'; +export { BrowserProfile }; +export { BrowserSession }; diff --git a/dist/browser/dvd-screensaver.d.ts b/dist/browser/dvd-screensaver.d.ts new file mode 100644 index 00000000..4ab629b4 --- /dev/null +++ b/dist/browser/dvd-screensaver.d.ts @@ -0,0 +1,101 @@ +/** + * DVD Screensaver Loading Animation + * + * Displays a fun DVD logo bouncing animation while waiting for browser operations. + * Inspired by the classic DVD screensaver. + */ +/** + * DVD Screensaver Animation Controller + */ +export declare class DVDScreensaver { + private isRunning; + private intervalId; + private width; + private height; + private x; + private y; + private dx; + private dy; + private logoWidth; + private logoHeight; + private colors; + private currentColorIndex; + private cornerHits; + private frameCount; + private message; + constructor(message?: string); + /** + * Start the animation + */ + start(fps?: number): void; + /** + * Stop the animation + */ + stop(): void; + /** + * Update logo position + */ + private update; + /** + * Change logo color + */ + private changeColor; + /** + * Render the current frame + */ + private render; + /** + * Get character for logo at specific position + */ + private getLogoChar; + /** + * Clear screen + */ + private clear; +} +/** + * Show DVD screensaver loading animation + * Returns a function to stop the animation + * + * @param message - Message to display + * @param fps - Frames per second (default: 10) + * @returns Function to stop the animation + * + * @example + * const stopAnimation = showDVDScreensaver('Loading browser...'); + * await someAsyncOperation(); + * stopAnimation(); + */ +export declare function showDVDScreensaver(message?: string, fps?: number): () => void; +/** + * Run an async operation with DVD screensaver animation + * + * @param operation - Async operation to run + * @param message - Message to display + * @returns Result of the operation + * + * @example + * const result = await withDVDScreensaver( + * async () => await longRunningOperation(), + * 'Processing...' + * ); + */ +export declare function withDVDScreensaver(operation: () => Promise, message?: string): Promise; +/** + * Simple spinner animation (alternative to DVD screensaver) + */ +export declare class SpinnerAnimation { + private isRunning; + private intervalId; + private frames; + private currentFrame; + private message; + constructor(message?: string); + start(fps?: number): void; + stop(): void; + private render; +} +/** + * Show simple spinner animation + */ +export declare function showSpinner(message?: string, fps?: number): () => void; diff --git a/dist/browser/dvd-screensaver.js b/dist/browser/dvd-screensaver.js new file mode 100644 index 00000000..ce772b41 --- /dev/null +++ b/dist/browser/dvd-screensaver.js @@ -0,0 +1,270 @@ +/** + * DVD Screensaver Loading Animation + * + * Displays a fun DVD logo bouncing animation while waiting for browser operations. + * Inspired by the classic DVD screensaver. + */ +import { createLogger } from '../logging-config.js'; +const logger = createLogger('browser_use.dvd_screensaver'); +/** + * DVD Screensaver Animation Controller + */ +export class DVDScreensaver { + isRunning = false; + intervalId = null; + width = 80; + height = 20; + x = 0; + y = 0; + dx = 1; + dy = 1; + logoWidth = 10; + logoHeight = 3; + colors = [ + '\x1b[31m', + '\x1b[32m', + '\x1b[33m', + '\x1b[34m', + '\x1b[35m', + '\x1b[36m', + ]; + currentColorIndex = 0; + cornerHits = 0; + frameCount = 0; + message; + constructor(message = 'Loading...') { + this.message = message; + this.x = Math.floor(Math.random() * (this.width - this.logoWidth)); + this.y = Math.floor(Math.random() * (this.height - this.logoHeight)); + } + /** + * Start the animation + */ + start(fps = 10) { + if (this.isRunning) { + return; + } + this.isRunning = true; + // Hide cursor + process.stderr.write('\x1b[?25l'); + // Clear screen + this.clear(); + this.intervalId = setInterval(() => { + this.update(); + this.render(); + }, 1000 / fps); + } + /** + * Stop the animation + */ + stop() { + if (!this.isRunning) { + return; + } + this.isRunning = false; + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = null; + } + // Clear screen one last time + this.clear(); + // Show cursor + process.stderr.write('\x1b[?25h'); + // Log corner hits if any + if (this.cornerHits > 0) { + logger.debug(`DVD logo hit corner ${this.cornerHits} time(s)! 🎯`); + } + } + /** + * Update logo position + */ + update() { + this.frameCount++; + // Update position + this.x += this.dx; + this.y += this.dy; + let hitCorner = false; + // Check horizontal bounds + if (this.x <= 0 || this.x >= this.width - this.logoWidth) { + this.dx = -this.dx; + this.x = Math.max(0, Math.min(this.x, this.width - this.logoWidth)); + this.changeColor(); + // Check if hit corner + if (this.y <= 0 || this.y >= this.height - this.logoHeight) { + hitCorner = true; + } + } + // Check vertical bounds + if (this.y <= 0 || this.y >= this.height - this.logoHeight) { + this.dy = -this.dy; + this.y = Math.max(0, Math.min(this.y, this.height - this.logoHeight)); + this.changeColor(); + // Check if hit corner + if (this.x <= 0 || this.x >= this.width - this.logoWidth) { + hitCorner = true; + } + } + if (hitCorner) { + this.cornerHits++; + } + } + /** + * Change logo color + */ + changeColor() { + this.currentColorIndex = (this.currentColorIndex + 1) % this.colors.length; + } + /** + * Render the current frame + */ + render() { + // Move cursor to top-left + process.stderr.write('\x1b[H'); + const color = this.colors[this.currentColorIndex]; + const reset = '\x1b[0m'; + // Build frame + const frame = []; + // Draw border and content + for (let row = 0; row < this.height; row++) { + let line = ''; + for (let col = 0; col < this.width; col++) { + // Check if this position is part of the logo + if (row >= this.y && + row < this.y + this.logoHeight && + col >= this.x && + col < this.x + this.logoWidth) { + // Draw logo + const logoRow = row - this.y; + const logoCol = col - this.x; + line += color + this.getLogoChar(logoRow, logoCol) + reset; + } + else if (row === 0 || + row === this.height - 1 || + col === 0 || + col === this.width - 1) { + // Draw border + line += '·'; + } + else { + line += ' '; + } + } + frame.push(line); + } + // Add status message + const statusLine = `\n${this.message} (Frame: ${this.frameCount}, Corner hits: ${this.cornerHits})`; + // Write frame to stderr + process.stderr.write(frame.join('\n') + statusLine); + } + /** + * Get character for logo at specific position + */ + getLogoChar(row, col) { + // Simple "DVD" text logo + if (row === 1) { + const text = ' DVD '; + return col < text.length ? text[col] : ' '; + } + return '▓'; + } + /** + * Clear screen + */ + clear() { + process.stderr.write('\x1b[2J\x1b[H'); + } +} +/** + * Show DVD screensaver loading animation + * Returns a function to stop the animation + * + * @param message - Message to display + * @param fps - Frames per second (default: 10) + * @returns Function to stop the animation + * + * @example + * const stopAnimation = showDVDScreensaver('Loading browser...'); + * await someAsyncOperation(); + * stopAnimation(); + */ +export function showDVDScreensaver(message = 'Loading...', fps = 10) { + const screensaver = new DVDScreensaver(message); + screensaver.start(fps); + return () => { + screensaver.stop(); + }; +} +/** + * Run an async operation with DVD screensaver animation + * + * @param operation - Async operation to run + * @param message - Message to display + * @returns Result of the operation + * + * @example + * const result = await withDVDScreensaver( + * async () => await longRunningOperation(), + * 'Processing...' + * ); + */ +export async function withDVDScreensaver(operation, message = 'Loading...') { + const stopAnimation = showDVDScreensaver(message); + try { + const result = await operation(); + return result; + } + finally { + stopAnimation(); + } +} +/** + * Simple spinner animation (alternative to DVD screensaver) + */ +export class SpinnerAnimation { + isRunning = false; + intervalId = null; + frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']; + currentFrame = 0; + message; + constructor(message = 'Loading...') { + this.message = message; + } + start(fps = 10) { + if (this.isRunning) { + return; + } + this.isRunning = true; + process.stderr.write('\x1b[?25l'); // Hide cursor + this.intervalId = setInterval(() => { + this.render(); + this.currentFrame = (this.currentFrame + 1) % this.frames.length; + }, 1000 / fps); + } + stop() { + if (!this.isRunning) { + return; + } + this.isRunning = false; + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = null; + } + // Clear line and show cursor + process.stderr.write('\r\x1b[K'); + process.stderr.write('\x1b[?25h'); + } + render() { + const frame = this.frames[this.currentFrame]; + process.stderr.write(`\r${frame} ${this.message}`); + } +} +/** + * Show simple spinner animation + */ +export function showSpinner(message = 'Loading...', fps = 10) { + const spinner = new SpinnerAnimation(message); + spinner.start(fps); + return () => { + spinner.stop(); + }; +} diff --git a/dist/browser/events.d.ts b/dist/browser/events.d.ts new file mode 100644 index 00000000..c77497db --- /dev/null +++ b/dist/browser/events.d.ts @@ -0,0 +1,403 @@ +import type { DOMElementNode } from '../dom/views.js'; +import { EventBusEvent, type EventBusEventInit } from '../event-bus.js'; +import { BrowserStateSummary } from './views.js'; +export type TargetID = string; +export type WaitUntilState = 'load' | 'domcontentloaded' | 'networkidle' | 'commit'; +export type MouseButton = 'left' | 'right' | 'middle'; +export declare abstract class BrowserEvent extends EventBusEvent { + protected constructor(eventType: string, init?: EventBusEventInit); +} +export declare class ElementSelectedEvent extends BrowserEvent { + node: TNode; + constructor(eventType: string, init: EventBusEventInit & { + node: TNode; + }); +} +export declare class NavigateToUrlEvent extends BrowserEvent { + url: string; + wait_until: WaitUntilState; + timeout_ms: number | null; + new_tab: boolean; + constructor(init: EventBusEventInit & { + url: string; + wait_until?: WaitUntilState; + timeout_ms?: number | null; + new_tab?: boolean; + }); +} +export declare class ClickElementEvent extends ElementSelectedEvent | null> { + button: MouseButton; + constructor(init: EventBusEventInit | null> & { + node: DOMElementNode; + button?: MouseButton; + }); +} +export declare class ClickCoordinateEvent extends BrowserEvent> { + coordinate_x: number; + coordinate_y: number; + button: MouseButton; + force: boolean; + constructor(init: EventBusEventInit> & { + coordinate_x: number; + coordinate_y: number; + button?: MouseButton; + force?: boolean; + }); +} +export declare class TypeTextEvent extends ElementSelectedEvent | null> { + text: string; + clear: boolean; + is_sensitive: boolean; + sensitive_key_name: string | null; + constructor(init: EventBusEventInit | null> & { + node: DOMElementNode; + text: string; + clear?: boolean; + is_sensitive?: boolean; + sensitive_key_name?: string | null; + }); +} +export declare class ScrollEvent extends ElementSelectedEvent { + direction: 'up' | 'down' | 'left' | 'right'; + amount: number; + constructor(init: EventBusEventInit & { + direction: 'up' | 'down' | 'left' | 'right'; + amount: number; + node?: DOMElementNode | null; + }); +} +export declare class SwitchTabEvent extends BrowserEvent { + target_id: TargetID | null; + constructor(init?: EventBusEventInit & { + target_id?: TargetID | null; + }); +} +export declare class CloseTabEvent extends BrowserEvent { + target_id: TargetID; + constructor(init: EventBusEventInit & { + target_id: TargetID; + }); +} +export declare class ScreenshotEvent extends BrowserEvent { + full_page: boolean; + clip: { + x: number; + y: number; + width: number; + height: number; + } | null; + constructor(init?: EventBusEventInit & { + full_page?: boolean; + clip?: { + x: number; + y: number; + width: number; + height: number; + } | null; + }); +} +export declare class BrowserStateRequestEvent extends BrowserEvent { + include_dom: boolean; + include_screenshot: boolean; + include_recent_events: boolean; + constructor(init?: EventBusEventInit & { + include_dom?: boolean; + include_screenshot?: boolean; + include_recent_events?: boolean; + }); +} +export declare class GoBackEvent extends BrowserEvent { + constructor(init?: EventBusEventInit); +} +export declare class GoForwardEvent extends BrowserEvent { + constructor(init?: EventBusEventInit); +} +export declare class RefreshEvent extends BrowserEvent { + constructor(init?: EventBusEventInit); +} +export declare class WaitEvent extends BrowserEvent { + seconds: number; + max_seconds: number; + constructor(init?: EventBusEventInit & { + seconds?: number; + max_seconds?: number; + }); +} +export declare class SendKeysEvent extends BrowserEvent { + keys: string; + constructor(init: EventBusEventInit & { + keys: string; + }); +} +export declare class UploadFileEvent extends ElementSelectedEvent { + file_path: string; + constructor(init: EventBusEventInit & { + node: DOMElementNode; + file_path: string; + }); +} +export declare class GetDropdownOptionsEvent extends ElementSelectedEvent> { + constructor(init: EventBusEventInit> & { + node: DOMElementNode; + }); +} +export declare class SelectDropdownOptionEvent extends ElementSelectedEvent> { + text: string; + constructor(init: EventBusEventInit> & { + node: DOMElementNode; + text: string; + }); +} +export declare class ScrollToTextEvent extends BrowserEvent { + text: string; + direction: 'up' | 'down'; + constructor(init: EventBusEventInit & { + text: string; + direction?: 'up' | 'down'; + }); +} +export declare class BrowserStartEvent extends BrowserEvent { + cdp_url: string | null; + launch_options: Record; + constructor(init?: EventBusEventInit & { + cdp_url?: string | null; + launch_options?: Record; + }); +} +export declare class BrowserStopEvent extends BrowserEvent { + force: boolean; + constructor(init?: EventBusEventInit & { + force?: boolean; + }); +} +export interface BrowserLaunchResult { + cdp_url: string; +} +export declare class BrowserLaunchEvent extends BrowserEvent { + constructor(init?: EventBusEventInit); +} +export declare class BrowserKillEvent extends BrowserEvent { + constructor(init?: EventBusEventInit); +} +export declare class BrowserConnectedEvent extends BrowserEvent { + cdp_url: string; + constructor(init: EventBusEventInit & { + cdp_url: string; + }); +} +export declare class BrowserReconnectingEvent extends BrowserEvent { + cdp_url: string; + attempt: number; + max_attempts: number; + constructor(init: EventBusEventInit & { + cdp_url: string; + attempt: number; + max_attempts: number; + }); +} +export declare class BrowserReconnectedEvent extends BrowserEvent { + cdp_url: string; + attempt: number; + downtime_seconds: number; + constructor(init: EventBusEventInit & { + cdp_url: string; + attempt: number; + downtime_seconds: number; + }); +} +export declare class BrowserStoppedEvent extends BrowserEvent { + reason: string | null; + constructor(init?: EventBusEventInit & { + reason?: string | null; + }); +} +export declare class CaptchaSolverStartedEvent extends BrowserEvent { + target_id: TargetID; + vendor: string; + url: string; + started_at: number; + constructor(init: EventBusEventInit & { + target_id: TargetID; + vendor: string; + url: string; + started_at: number; + }); +} +export declare class CaptchaSolverFinishedEvent extends BrowserEvent { + target_id: TargetID; + vendor: string; + url: string; + duration_ms: number; + finished_at: number; + success: boolean; + constructor(init: EventBusEventInit & { + target_id: TargetID; + vendor: string; + url: string; + duration_ms: number; + finished_at: number; + success: boolean; + }); +} +export declare class TabCreatedEvent extends BrowserEvent { + target_id: TargetID; + url: string; + constructor(init: EventBusEventInit & { + target_id: TargetID; + url: string; + }); +} +export declare class TabClosedEvent extends BrowserEvent { + target_id: TargetID; + constructor(init: EventBusEventInit & { + target_id: TargetID; + }); +} +export declare class AgentFocusChangedEvent extends BrowserEvent { + target_id: TargetID; + url: string; + constructor(init: EventBusEventInit & { + target_id: TargetID; + url: string; + }); +} +export declare class TargetCrashedEvent extends BrowserEvent { + target_id: TargetID; + error: string; + constructor(init: EventBusEventInit & { + target_id: TargetID; + error: string; + }); +} +export declare class NavigationStartedEvent extends BrowserEvent { + target_id: TargetID; + url: string; + constructor(init: EventBusEventInit & { + target_id: TargetID; + url: string; + }); +} +export declare class NavigationCompleteEvent extends BrowserEvent { + target_id: TargetID; + url: string; + status: number | null; + error_message: string | null; + loading_status: string | null; + constructor(init: EventBusEventInit & { + target_id: TargetID; + url: string; + status?: number | null; + error_message?: string | null; + loading_status?: string | null; + }); +} +export declare class BrowserErrorEvent extends BrowserEvent { + error_type: string; + message: string; + details: Record; + constructor(init: EventBusEventInit & { + error_type: string; + message: string; + details?: Record; + }); +} +export declare class SaveStorageStateEvent extends BrowserEvent { + path: string | null; + constructor(init?: EventBusEventInit & { + path?: string | null; + }); +} +export declare class StorageStateSavedEvent extends BrowserEvent { + path: string; + cookies_count: number; + origins_count: number; + constructor(init: EventBusEventInit & { + path: string; + cookies_count: number; + origins_count: number; + }); +} +export declare class LoadStorageStateEvent extends BrowserEvent { + path: string | null; + constructor(init?: EventBusEventInit & { + path?: string | null; + }); +} +export declare class StorageStateLoadedEvent extends BrowserEvent { + path: string; + cookies_count: number; + origins_count: number; + constructor(init: EventBusEventInit & { + path: string; + cookies_count: number; + origins_count: number; + }); +} +export declare class DownloadStartedEvent extends BrowserEvent { + guid: string; + url: string; + suggested_filename: string; + auto_download: boolean; + constructor(init: EventBusEventInit & { + guid: string; + url: string; + suggested_filename: string; + auto_download?: boolean; + }); +} +export declare class DownloadProgressEvent extends BrowserEvent { + guid: string; + received_bytes: number; + total_bytes: number; + state: string; + constructor(init: EventBusEventInit & { + guid: string; + received_bytes: number; + total_bytes: number; + state: string; + }); +} +export declare class FileDownloadedEvent extends BrowserEvent { + guid: string | null; + url: string; + path: string; + file_name: string; + file_size: number; + file_type: string | null; + mime_type: string | null; + from_cache: boolean; + auto_download: boolean; + constructor(init: EventBusEventInit & { + guid?: string | null; + url: string; + path: string; + file_name: string; + file_size: number; + file_type?: string | null; + mime_type?: string | null; + from_cache?: boolean; + auto_download?: boolean; + }); +} +export declare class AboutBlankDVDScreensaverShownEvent extends BrowserEvent { + target_id: TargetID; + error: string | null; + constructor(init: EventBusEventInit & { + target_id: TargetID; + error?: string | null; + }); +} +export declare class DialogOpenedEvent extends BrowserEvent { + dialog_type: string; + message: string; + url: string; + frame_id: string | null; + constructor(init: EventBusEventInit & { + dialog_type: string; + message: string; + url: string; + frame_id?: string | null; + }); +} +export declare const BROWSER_EVENT_CLASSES: readonly [typeof ElementSelectedEvent, typeof NavigateToUrlEvent, typeof ClickElementEvent, typeof ClickCoordinateEvent, typeof TypeTextEvent, typeof ScrollEvent, typeof SwitchTabEvent, typeof CloseTabEvent, typeof ScreenshotEvent, typeof BrowserStateRequestEvent, typeof GoBackEvent, typeof GoForwardEvent, typeof RefreshEvent, typeof WaitEvent, typeof SendKeysEvent, typeof UploadFileEvent, typeof GetDropdownOptionsEvent, typeof SelectDropdownOptionEvent, typeof ScrollToTextEvent, typeof BrowserStartEvent, typeof BrowserStopEvent, typeof BrowserLaunchEvent, typeof BrowserKillEvent, typeof BrowserConnectedEvent, typeof BrowserReconnectingEvent, typeof BrowserReconnectedEvent, typeof BrowserStoppedEvent, typeof TabCreatedEvent, typeof TabClosedEvent, typeof AgentFocusChangedEvent, typeof TargetCrashedEvent, typeof NavigationStartedEvent, typeof NavigationCompleteEvent, typeof BrowserErrorEvent, typeof SaveStorageStateEvent, typeof StorageStateSavedEvent, typeof LoadStorageStateEvent, typeof StorageStateLoadedEvent, typeof DownloadStartedEvent, typeof DownloadProgressEvent, typeof FileDownloadedEvent, typeof AboutBlankDVDScreensaverShownEvent, typeof DialogOpenedEvent]; +export declare const BROWSER_EVENT_NAMES: string[]; diff --git a/dist/browser/events.js b/dist/browser/events.js new file mode 100644 index 00000000..20a52119 --- /dev/null +++ b/dist/browser/events.js @@ -0,0 +1,632 @@ +import { EventBusEvent } from '../event-bus.js'; +const getTimeout = (envVar, defaultValue) => { + const raw = process.env[envVar]; + if (raw == null || raw.trim() === '') { + return defaultValue; + } + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed < 0) { + return defaultValue; + } + return parsed; +}; +const resolveEventTimeout = (eventName, defaultSeconds, explicitTimeout) => explicitTimeout !== undefined + ? explicitTimeout + : getTimeout(`TIMEOUT_${eventName}`, defaultSeconds); +export class BrowserEvent extends EventBusEvent { + constructor(eventType, init = {}) { + super(eventType, init); + } +} +export class ElementSelectedEvent extends BrowserEvent { + node; + constructor(eventType, init) { + super(eventType, init); + this.node = init.node; + } +} +export class NavigateToUrlEvent extends BrowserEvent { + url; + wait_until; + timeout_ms; + new_tab; + constructor(init) { + super('NavigateToUrlEvent', { + ...init, + event_timeout: resolveEventTimeout('NavigateToUrlEvent', 15, init.event_timeout), + }); + this.url = init.url; + this.wait_until = init.wait_until ?? 'load'; + this.timeout_ms = init.timeout_ms ?? null; + this.new_tab = init.new_tab ?? false; + } +} +export class ClickElementEvent extends ElementSelectedEvent { + button; + constructor(init) { + super('ClickElementEvent', { + ...init, + event_timeout: resolveEventTimeout('ClickElementEvent', 15, init.event_timeout), + }); + this.button = init.button ?? 'left'; + } +} +export class ClickCoordinateEvent extends BrowserEvent { + coordinate_x; + coordinate_y; + button; + force; + constructor(init) { + super('ClickCoordinateEvent', { + ...init, + event_timeout: resolveEventTimeout('ClickCoordinateEvent', 15, init.event_timeout), + }); + this.coordinate_x = init.coordinate_x; + this.coordinate_y = init.coordinate_y; + this.button = init.button ?? 'left'; + this.force = init.force ?? false; + } +} +export class TypeTextEvent extends ElementSelectedEvent { + text; + clear; + is_sensitive; + sensitive_key_name; + constructor(init) { + super('TypeTextEvent', { + ...init, + event_timeout: resolveEventTimeout('TypeTextEvent', 60, init.event_timeout), + }); + this.text = init.text; + this.clear = init.clear ?? true; + this.is_sensitive = init.is_sensitive ?? false; + this.sensitive_key_name = init.sensitive_key_name ?? null; + } +} +export class ScrollEvent extends ElementSelectedEvent { + direction; + amount; + constructor(init) { + super('ScrollEvent', { + ...init, + node: init.node ?? null, + event_timeout: resolveEventTimeout('ScrollEvent', 8, init.event_timeout), + }); + this.direction = init.direction; + this.amount = init.amount; + } +} +export class SwitchTabEvent extends BrowserEvent { + target_id; + constructor(init = {}) { + super('SwitchTabEvent', { + ...init, + event_timeout: resolveEventTimeout('SwitchTabEvent', 10, init.event_timeout), + }); + this.target_id = init.target_id ?? null; + } +} +export class CloseTabEvent extends BrowserEvent { + target_id; + constructor(init) { + super('CloseTabEvent', { + ...init, + event_timeout: resolveEventTimeout('CloseTabEvent', 10, init.event_timeout), + }); + this.target_id = init.target_id; + } +} +export class ScreenshotEvent extends BrowserEvent { + full_page; + clip; + constructor(init = {}) { + super('ScreenshotEvent', { + ...init, + event_timeout: resolveEventTimeout('ScreenshotEvent', 15, init.event_timeout), + }); + this.full_page = init.full_page ?? false; + this.clip = init.clip ?? null; + } +} +export class BrowserStateRequestEvent extends BrowserEvent { + include_dom; + include_screenshot; + include_recent_events; + constructor(init = {}) { + super('BrowserStateRequestEvent', { + ...init, + event_timeout: resolveEventTimeout('BrowserStateRequestEvent', 30, init.event_timeout), + }); + this.include_dom = init.include_dom ?? true; + this.include_screenshot = init.include_screenshot ?? true; + this.include_recent_events = init.include_recent_events ?? false; + } +} +export class GoBackEvent extends BrowserEvent { + constructor(init = {}) { + super('GoBackEvent', { + ...init, + event_timeout: resolveEventTimeout('GoBackEvent', 15, init.event_timeout), + }); + } +} +export class GoForwardEvent extends BrowserEvent { + constructor(init = {}) { + super('GoForwardEvent', { + ...init, + event_timeout: resolveEventTimeout('GoForwardEvent', 15, init.event_timeout), + }); + } +} +export class RefreshEvent extends BrowserEvent { + constructor(init = {}) { + super('RefreshEvent', { + ...init, + event_timeout: resolveEventTimeout('RefreshEvent', 15, init.event_timeout), + }); + } +} +export class WaitEvent extends BrowserEvent { + seconds; + max_seconds; + constructor(init = {}) { + super('WaitEvent', { + ...init, + event_timeout: resolveEventTimeout('WaitEvent', 60, init.event_timeout), + }); + this.seconds = init.seconds ?? 3; + this.max_seconds = init.max_seconds ?? 10; + } +} +export class SendKeysEvent extends BrowserEvent { + keys; + constructor(init) { + super('SendKeysEvent', { + ...init, + event_timeout: resolveEventTimeout('SendKeysEvent', 60, init.event_timeout), + }); + this.keys = init.keys; + } +} +export class UploadFileEvent extends ElementSelectedEvent { + file_path; + constructor(init) { + super('UploadFileEvent', { + ...init, + event_timeout: resolveEventTimeout('UploadFileEvent', 30, init.event_timeout), + }); + this.file_path = init.file_path; + } +} +export class GetDropdownOptionsEvent extends ElementSelectedEvent { + constructor(init) { + super('GetDropdownOptionsEvent', { + ...init, + event_timeout: resolveEventTimeout('GetDropdownOptionsEvent', 15, init.event_timeout), + }); + } +} +export class SelectDropdownOptionEvent extends ElementSelectedEvent { + text; + constructor(init) { + super('SelectDropdownOptionEvent', { + ...init, + event_timeout: resolveEventTimeout('SelectDropdownOptionEvent', 8, init.event_timeout), + }); + this.text = init.text; + } +} +export class ScrollToTextEvent extends BrowserEvent { + text; + direction; + constructor(init) { + super('ScrollToTextEvent', { + ...init, + event_timeout: resolveEventTimeout('ScrollToTextEvent', 15, init.event_timeout), + }); + this.text = init.text; + this.direction = init.direction ?? 'down'; + } +} +export class BrowserStartEvent extends BrowserEvent { + cdp_url; + launch_options; + constructor(init = {}) { + super('BrowserStartEvent', { + ...init, + event_timeout: resolveEventTimeout('BrowserStartEvent', 30, init.event_timeout), + }); + this.cdp_url = init.cdp_url ?? null; + this.launch_options = init.launch_options ?? {}; + } +} +export class BrowserStopEvent extends BrowserEvent { + force; + constructor(init = {}) { + super('BrowserStopEvent', { + ...init, + event_timeout: resolveEventTimeout('BrowserStopEvent', 45, init.event_timeout), + }); + this.force = init.force ?? false; + } +} +export class BrowserLaunchEvent extends BrowserEvent { + constructor(init = {}) { + super('BrowserLaunchEvent', { + ...init, + event_timeout: resolveEventTimeout('BrowserLaunchEvent', 30, init.event_timeout), + }); + } +} +export class BrowserKillEvent extends BrowserEvent { + constructor(init = {}) { + super('BrowserKillEvent', { + ...init, + event_timeout: resolveEventTimeout('BrowserKillEvent', 30, init.event_timeout), + }); + } +} +export class BrowserConnectedEvent extends BrowserEvent { + cdp_url; + constructor(init) { + super('BrowserConnectedEvent', { + ...init, + event_timeout: resolveEventTimeout('BrowserConnectedEvent', 30, init.event_timeout), + }); + this.cdp_url = init.cdp_url; + } +} +export class BrowserReconnectingEvent extends BrowserEvent { + cdp_url; + attempt; + max_attempts; + constructor(init) { + super('BrowserReconnectingEvent', { + ...init, + event_timeout: resolveEventTimeout('BrowserReconnectingEvent', 30, init.event_timeout), + }); + this.cdp_url = init.cdp_url; + this.attempt = init.attempt; + this.max_attempts = init.max_attempts; + } +} +export class BrowserReconnectedEvent extends BrowserEvent { + cdp_url; + attempt; + downtime_seconds; + constructor(init) { + super('BrowserReconnectedEvent', { + ...init, + event_timeout: resolveEventTimeout('BrowserReconnectedEvent', 30, init.event_timeout), + }); + this.cdp_url = init.cdp_url; + this.attempt = init.attempt; + this.downtime_seconds = init.downtime_seconds; + } +} +export class BrowserStoppedEvent extends BrowserEvent { + reason; + constructor(init = {}) { + super('BrowserStoppedEvent', { + ...init, + event_timeout: resolveEventTimeout('BrowserStoppedEvent', 30, init.event_timeout), + }); + this.reason = init.reason ?? null; + } +} +export class CaptchaSolverStartedEvent extends BrowserEvent { + target_id; + vendor; + url; + started_at; + constructor(init) { + super('CaptchaSolverStartedEvent', { + ...init, + event_timeout: resolveEventTimeout('CaptchaSolverStartedEvent', 5, init.event_timeout), + }); + this.target_id = init.target_id; + this.vendor = init.vendor; + this.url = init.url; + this.started_at = init.started_at; + } +} +export class CaptchaSolverFinishedEvent extends BrowserEvent { + target_id; + vendor; + url; + duration_ms; + finished_at; + success; + constructor(init) { + super('CaptchaSolverFinishedEvent', { + ...init, + event_timeout: resolveEventTimeout('CaptchaSolverFinishedEvent', 5, init.event_timeout), + }); + this.target_id = init.target_id; + this.vendor = init.vendor; + this.url = init.url; + this.duration_ms = init.duration_ms; + this.finished_at = init.finished_at; + this.success = init.success; + } +} +export class TabCreatedEvent extends BrowserEvent { + target_id; + url; + constructor(init) { + super('TabCreatedEvent', { + ...init, + event_timeout: resolveEventTimeout('TabCreatedEvent', 30, init.event_timeout), + }); + this.target_id = init.target_id; + this.url = init.url; + } +} +export class TabClosedEvent extends BrowserEvent { + target_id; + constructor(init) { + super('TabClosedEvent', { + ...init, + event_timeout: resolveEventTimeout('TabClosedEvent', 10, init.event_timeout), + }); + this.target_id = init.target_id; + } +} +export class AgentFocusChangedEvent extends BrowserEvent { + target_id; + url; + constructor(init) { + super('AgentFocusChangedEvent', { + ...init, + event_timeout: resolveEventTimeout('AgentFocusChangedEvent', 10, init.event_timeout), + }); + this.target_id = init.target_id; + this.url = init.url; + } +} +export class TargetCrashedEvent extends BrowserEvent { + target_id; + error; + constructor(init) { + super('TargetCrashedEvent', { + ...init, + event_timeout: resolveEventTimeout('TargetCrashedEvent', 10, init.event_timeout), + }); + this.target_id = init.target_id; + this.error = init.error; + } +} +export class NavigationStartedEvent extends BrowserEvent { + target_id; + url; + constructor(init) { + super('NavigationStartedEvent', { + ...init, + event_timeout: resolveEventTimeout('NavigationStartedEvent', 30, init.event_timeout), + }); + this.target_id = init.target_id; + this.url = init.url; + } +} +export class NavigationCompleteEvent extends BrowserEvent { + target_id; + url; + status; + error_message; + loading_status; + constructor(init) { + super('NavigationCompleteEvent', { + ...init, + event_timeout: resolveEventTimeout('NavigationCompleteEvent', 30, init.event_timeout), + }); + this.target_id = init.target_id; + this.url = init.url; + this.status = init.status ?? null; + this.error_message = init.error_message ?? null; + this.loading_status = init.loading_status ?? null; + } +} +export class BrowserErrorEvent extends BrowserEvent { + error_type; + message; + details; + constructor(init) { + super('BrowserErrorEvent', { + ...init, + event_timeout: resolveEventTimeout('BrowserErrorEvent', 30, init.event_timeout), + }); + this.error_type = init.error_type; + this.message = init.message; + this.details = init.details ?? {}; + } +} +export class SaveStorageStateEvent extends BrowserEvent { + path; + constructor(init = {}) { + super('SaveStorageStateEvent', { + ...init, + event_timeout: resolveEventTimeout('SaveStorageStateEvent', 45, init.event_timeout), + }); + this.path = init.path ?? null; + } +} +export class StorageStateSavedEvent extends BrowserEvent { + path; + cookies_count; + origins_count; + constructor(init) { + super('StorageStateSavedEvent', { + ...init, + event_timeout: resolveEventTimeout('StorageStateSavedEvent', 30, init.event_timeout), + }); + this.path = init.path; + this.cookies_count = init.cookies_count; + this.origins_count = init.origins_count; + } +} +export class LoadStorageStateEvent extends BrowserEvent { + path; + constructor(init = {}) { + super('LoadStorageStateEvent', { + ...init, + event_timeout: resolveEventTimeout('LoadStorageStateEvent', 45, init.event_timeout), + }); + this.path = init.path ?? null; + } +} +export class StorageStateLoadedEvent extends BrowserEvent { + path; + cookies_count; + origins_count; + constructor(init) { + super('StorageStateLoadedEvent', { + ...init, + event_timeout: resolveEventTimeout('StorageStateLoadedEvent', 30, init.event_timeout), + }); + this.path = init.path; + this.cookies_count = init.cookies_count; + this.origins_count = init.origins_count; + } +} +export class DownloadStartedEvent extends BrowserEvent { + guid; + url; + suggested_filename; + auto_download; + constructor(init) { + super('DownloadStartedEvent', { + ...init, + event_timeout: resolveEventTimeout('DownloadStartedEvent', 5, init.event_timeout), + }); + this.guid = init.guid; + this.url = init.url; + this.suggested_filename = init.suggested_filename; + this.auto_download = init.auto_download ?? false; + } +} +export class DownloadProgressEvent extends BrowserEvent { + guid; + received_bytes; + total_bytes; + state; + constructor(init) { + super('DownloadProgressEvent', { + ...init, + event_timeout: resolveEventTimeout('DownloadProgressEvent', 5, init.event_timeout), + }); + this.guid = init.guid; + this.received_bytes = init.received_bytes; + this.total_bytes = init.total_bytes; + this.state = init.state; + } +} +export class FileDownloadedEvent extends BrowserEvent { + guid; + url; + path; + file_name; + file_size; + file_type; + mime_type; + from_cache; + auto_download; + constructor(init) { + super('FileDownloadedEvent', { + ...init, + event_timeout: resolveEventTimeout('FileDownloadedEvent', 30, init.event_timeout), + }); + this.guid = init.guid ?? null; + this.url = init.url; + this.path = init.path; + this.file_name = init.file_name; + this.file_size = init.file_size; + this.file_type = init.file_type ?? null; + this.mime_type = init.mime_type ?? null; + this.from_cache = init.from_cache ?? false; + this.auto_download = init.auto_download ?? false; + } +} +export class AboutBlankDVDScreensaverShownEvent extends BrowserEvent { + target_id; + error; + constructor(init) { + super('AboutBlankDVDScreensaverShownEvent', init); + this.target_id = init.target_id; + this.error = init.error ?? null; + } +} +export class DialogOpenedEvent extends BrowserEvent { + dialog_type; + message; + url; + frame_id; + constructor(init) { + super('DialogOpenedEvent', init); + this.dialog_type = init.dialog_type; + this.message = init.message; + this.url = init.url; + this.frame_id = init.frame_id ?? null; + } +} +export const BROWSER_EVENT_CLASSES = [ + ElementSelectedEvent, + NavigateToUrlEvent, + ClickElementEvent, + ClickCoordinateEvent, + TypeTextEvent, + ScrollEvent, + SwitchTabEvent, + CloseTabEvent, + ScreenshotEvent, + BrowserStateRequestEvent, + GoBackEvent, + GoForwardEvent, + RefreshEvent, + WaitEvent, + SendKeysEvent, + UploadFileEvent, + GetDropdownOptionsEvent, + SelectDropdownOptionEvent, + ScrollToTextEvent, + BrowserStartEvent, + BrowserStopEvent, + BrowserLaunchEvent, + BrowserKillEvent, + BrowserConnectedEvent, + BrowserReconnectingEvent, + BrowserReconnectedEvent, + BrowserStoppedEvent, + TabCreatedEvent, + TabClosedEvent, + AgentFocusChangedEvent, + TargetCrashedEvent, + NavigationStartedEvent, + NavigationCompleteEvent, + BrowserErrorEvent, + SaveStorageStateEvent, + StorageStateSavedEvent, + LoadStorageStateEvent, + StorageStateLoadedEvent, + DownloadStartedEvent, + DownloadProgressEvent, + FileDownloadedEvent, + AboutBlankDVDScreensaverShownEvent, + DialogOpenedEvent, +]; +export const BROWSER_EVENT_NAMES = BROWSER_EVENT_CLASSES.map((eventClass) => eventClass.name); +const checkEventNamesDontOverlap = () => { + for (const nameA of BROWSER_EVENT_NAMES) { + if (!nameA.endsWith('Event')) { + throw new Error(`Event ${nameA} does not end with Event`); + } + for (const nameB of BROWSER_EVENT_NAMES) { + if (nameA === nameB) { + continue; + } + if (nameB.includes(nameA)) { + throw new Error(`Event ${nameA} is a substring of ${nameB}; event names must be non-overlapping`); + } + } + } +}; +checkEventNamesDontOverlap(); diff --git a/dist/browser/extensions.d.ts b/dist/browser/extensions.d.ts new file mode 100644 index 00000000..19575f8e --- /dev/null +++ b/dist/browser/extensions.d.ts @@ -0,0 +1,63 @@ +/** + * Browser Extension Management + * Handles Chrome extension download, installation, and runtime integration. + * Supports both Manifest V2 and V3 extensions. + */ +export interface BrowserExtensionDescriptor { + name: string; + webstore_id?: string; + id?: string; + webstore_url?: string; + crx_url?: string; + crx_path?: string; + unpacked_path?: string; + version?: string; + manifest?: any; + manifest_version?: string; + homepage_url?: string; + options_url?: string; + target?: any; + target_ctx?: any; + target_type?: string; + target_url?: string; + read_manifest?: () => any; + read_version?: () => string | null; + dispatch_eval?: (...args: any[]) => Promise; + dispatch_popup?: () => Promise; + dispatch_action?: (tab?: any) => Promise; + dispatch_message?: (message: any, options?: any) => Promise; + dispatch_command?: (command: string, tab?: any) => Promise; +} +/** + * Generate extension ID from unpacked path + * Chrome uses SHA256 hash of the directory path + */ +export declare function getExtensionId(unpackedPath: string): string | null; +/** + * Load or install extension + */ +export declare function loadOrInstallExtension(ext: BrowserExtensionDescriptor, extensionsDir?: string): Promise; +/** + * Check if a browser target is an extension + */ +export declare function isTargetExtension(target: any): Promise<{ + target_type: string | null; + target_ctx: any; + target_url: string | null; + target_is_bg: boolean; + target_is_extension: boolean; + extension_id: string | null; + manifest_version: string; +}>; +/** + * Load extension from browser target (runtime connection) + */ +export declare function loadExtensionFromTarget(extensions: BrowserExtensionDescriptor[], target: any): Promise; +/** + * Load all Chrome extensions from browser targets + */ +export declare function getChromeExtensionsFromBrowser(browser: any, extensions: BrowserExtensionDescriptor[]): Promise; +/** + * Install extensions from persona configuration + */ +export declare function installExtensionsFromConfig(extensionConfigs: BrowserExtensionDescriptor[], extensionsDir: string): Promise; diff --git a/dist/browser/extensions.js b/dist/browser/extensions.js new file mode 100644 index 00000000..9441024f --- /dev/null +++ b/dist/browser/extensions.js @@ -0,0 +1,359 @@ +/** + * Browser Extension Management + * Handles Chrome extension download, installation, and runtime integration. + * Supports both Manifest V2 and V3 extensions. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import { pipeline } from 'node:stream/promises'; +import { createWriteStream } from 'node:fs'; +import { Readable } from 'node:stream'; +// @ts-ignore - extract-zip types may not be available +import extract from 'extract-zip'; +import { createLogger } from '../logging-config.js'; +const logger = createLogger('browser_use.extensions'); +/** + * Generate extension ID from unpacked path + * Chrome uses SHA256 hash of the directory path + */ +export function getExtensionId(unpackedPath) { + const manifestPath = path.join(unpackedPath, 'manifest.json'); + if (!fs.existsSync(manifestPath)) { + return null; + } + // Chrome uses SHA256 hash and converts to letter format + const hash = createHash('sha256').update(unpackedPath).digest('hex'); + const extensionId = hash + .slice(0, 32) + .split('') + .map((char) => String.fromCharCode(parseInt(char, 16) + 'a'.charCodeAt(0))) + .join(''); + return extensionId; +} +/** + * Download CRX file from Chrome Web Store + */ +async function downloadCrx(crxUrl, crxPath) { + try { + logger.info(`[🛠️] Downloading extension from ${crxUrl}...`); + const response = await fetch(crxUrl); + if (!response.ok || !response.body) { + logger.warning(`[⚠️] Failed to download extension: ${response.statusText}`); + return false; + } + // Ensure directory exists + const dir = path.dirname(crxPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + // Download file + const fileStream = createWriteStream(crxPath); + await pipeline(Readable.fromWeb(response.body), fileStream); + logger.info(`[✅] Downloaded to ${crxPath}`); + return true; + } + catch (error) { + logger.error(`[❌] Download failed: ${error.message}`); + return false; + } +} +/** + * Extract CRX file to unpacked directory + */ +async function unpackCrx(crxPath, unpackedPath) { + try { + // Ensure unpacked directory exists + if (!fs.existsSync(unpackedPath)) { + fs.mkdirSync(unpackedPath, { recursive: true }); + } + // Extract zip file (CRX is essentially a ZIP with extra header) + await extract(crxPath, { dir: path.resolve(unpackedPath) }); + // Verify manifest exists + const manifestPath = path.join(unpackedPath, 'manifest.json'); + if (!fs.existsSync(manifestPath)) { + logger.error(`[❌] No manifest.json found in ${unpackedPath}`); + return false; + } + logger.info(`[✅] Extracted to ${unpackedPath}`); + return true; + } + catch (error) { + logger.error(`[❌] Extraction failed: ${error.message}`); + return false; + } +} +/** + * Install extension (download and unpack if needed) + */ +async function installExtension(extension) { + const manifestPath = path.join(extension.unpacked_path, 'manifest.json'); + const crxPath = extension.crx_path; + // Download CRX if neither manifest nor CRX exists + if (!fs.existsSync(manifestPath) && !fs.existsSync(crxPath)) { + logger.info(`[🛠️] Downloading missing extension ${extension.name} ${extension.webstore_id} -> ${crxPath}`); + const downloaded = await downloadCrx(extension.crx_url, crxPath); + if (!downloaded) { + return false; + } + } + // Unpack CRX if manifest doesn't exist + if (!fs.existsSync(manifestPath)) { + const unpacked = await unpackCrx(crxPath, extension.unpacked_path); + if (!unpacked) { + return false; + } + } + return true; +} +/** + * Load or install extension + */ +export async function loadOrInstallExtension(ext, extensionsDir = path.join(process.cwd(), '.browser-use', 'extensions')) { + if (!ext.webstore_id && !ext.unpacked_path) { + throw new Error('Extension must have either webstore_id or unpacked_path'); + } + // Set statically computable extension metadata + ext.webstore_id = ext.webstore_id || ext.id; + ext.name = ext.name || ext.webstore_id; + ext.webstore_url = + ext.webstore_url || + `https://chromewebstore.google.com/detail/${ext.webstore_id}`; + ext.crx_url = + ext.crx_url || + `https://clients2.google.com/service/update2/crx?response=redirect&prodversion=130.0&acceptformat=crx3&x=id%3D${ext.webstore_id}%26uc`; + ext.crx_path = + ext.crx_path || + path.join(extensionsDir, `${ext.webstore_id}__${ext.name}.crx`); + ext.unpacked_path = + ext.unpacked_path || + path.join(extensionsDir, `${ext.webstore_id}__${ext.name}`); + const manifestPath = path.join(ext.unpacked_path, 'manifest.json'); + // Helper functions + const readManifest = () => { + if (fs.existsSync(manifestPath)) { + return JSON.parse(fs.readFileSync(manifestPath, 'utf-8')); + } + return null; + }; + const readVersion = () => { + const manifest = readManifest(); + return manifest?.version || null; + }; + ext.read_manifest = readManifest; + ext.read_version = readVersion; + // Install extension if not already installed + if (!readVersion()) { + await installExtension(ext); + } + // Auto-detect ID and version + ext.id = getExtensionId(ext.unpacked_path) ?? undefined; + ext.version = readVersion() ?? undefined; + if (!ext.version) { + logger.warning(`[❌] Unable to detect ID and version of installed extension ${ext.unpacked_path}`); + } + else { + logger.info(`[➕] Installed extension ${ext.name} (${ext.version})... ${ext.unpacked_path}`); + } + return ext; +} +/** + * Check if a browser target is an extension + */ +export async function isTargetExtension(target) { + const targetInfo = await (async () => { + try { + const target_type = await target.type(); + const target_ctx = (await target.worker?.()) || (await target.page?.()) || null; + const target_url = (await target.url?.()) || (target_ctx ? await target_ctx.url() : null); + return { target_type, target_ctx, target_url }; + } + catch (error) { + if (error.message?.includes('No target with given id found')) { + // Target already closed + return { + target_type: 'closed', + target_ctx: null, + target_url: 'about:closed', + }; + } + throw error; + } + })(); + const { target_type, target_ctx, target_url } = targetInfo; + const target_is_bg = ['service_worker', 'background_page'].includes(target_type || ''); + const target_is_extension = target_url?.startsWith('chrome-extension://') || false; + const extension_id = target_is_extension + ? target_url.split('://')[1].split('/')[0] + : null; + const manifest_version = target_type === 'service_worker' ? '3' : '2'; + return { + target_type, + target_ctx, + target_url, + target_is_bg, + target_is_extension, + extension_id, + manifest_version, + }; +} +/** + * Load extension from browser target (runtime connection) + */ +export async function loadExtensionFromTarget(extensions, target) { + const extensionInfo = await isTargetExtension(target); + const { target_is_bg, target_is_extension, target_type, target_ctx, target_url, extension_id, manifest_version, } = extensionInfo; + if (!target_is_bg || !extension_id || !target_ctx) { + return null; + } + // Get manifest from extension + const manifest = await target_ctx.evaluate('() => chrome.runtime.getManifest()'); + const name = manifest.name; + const version = manifest.version; + const homepage_url = manifest.homepage_url; + const options_page = manifest.options_page; + const options_ui = manifest.options_ui || {}; + if (!version || !extension_id) { + return null; + } + // Get options URL + const options_url = await target_ctx.evaluate('(options_page) => chrome.runtime.getURL(options_page)', options_page || options_ui.page || 'options.html'); + // Get keyboard commands + const commands = await target_ctx.evaluate(` + async () => { + return await new Promise((resolve) => { + if (chrome.commands) { + chrome.commands.getAll(resolve); + } else { + resolve({}); + } + }); + } + `); + // Dispatch helpers + const dispatch_eval = async (...args) => { + return await target_ctx.evaluate(...args); + }; + const dispatch_popup = async () => { + return await target_ctx.evaluate("() => chrome.action?.openPopup() || chrome.tabs.create({url: chrome.runtime.getURL('popup.html')})"); + }; + let dispatch_action; + let dispatch_message; + let dispatch_command; + if (manifest_version === '3') { + // Manifest V3 APIs + dispatch_action = async (tab) => { + return await target_ctx.evaluate(` + async (tab) => { + tab = tab || (await new Promise((resolve) => + chrome.tabs.query({currentWindow: true, active: true}, ([tab]) => resolve(tab)) + )); + return await chrome.action.onClicked.dispatch(tab); + } + `, tab); + }; + dispatch_message = async (message, options) => { + return await target_ctx.evaluate(` + async (extension_id, message, options) => { + return await chrome.runtime.sendMessage(extension_id, message, options); + } + `, extension_id, message, options); + }; + dispatch_command = async (command, tab) => { + return await target_ctx.evaluate(` + async (command, tab) => { + return await chrome.commands.onCommand.dispatch(command, tab); + } + `, command, tab); + }; + } + else { + // Manifest V2 APIs + dispatch_action = async (tab) => { + return await target_ctx.evaluate(` + async (tab) => { + tab = tab || (await new Promise((resolve) => + chrome.tabs.query({currentWindow: true, active: true}, ([tab]) => resolve(tab)) + )); + return await chrome.browserAction.onClicked.dispatch(tab); + } + `, tab); + }; + dispatch_message = async (message, options) => { + return await target_ctx.evaluate(` + async (extension_id, message, options) => { + return await new Promise((resolve) => + chrome.runtime.sendMessage(extension_id, message, options, resolve) + ); + } + `, extension_id, message, options); + }; + dispatch_command = async (command, tab) => { + return await target_ctx.evaluate(` + async (command, tab) => { + return await new Promise((resolve) => + chrome.commands.onCommand.dispatch(command, tab, resolve) + ); + } + `, command, tab); + }; + } + // Find existing extension or create new one + const existing_extension = extensions.find((ext) => ext.id === extension_id) || + {}; + const new_extension = { + ...existing_extension, + id: extension_id, + name, + target, + target_ctx, + target_type: target_type, + target_url: target_url, + manifest_version, + manifest, + version, + homepage_url, + options_url, + dispatch_eval, + dispatch_popup, + dispatch_action, + dispatch_message, + dispatch_command, + }; + logger.info(`[➕] Loaded extension ${name.slice(0, 32)} (${version}) ${target_type}... ${target_url}`); + // Update existing extension in-place + Object.assign(existing_extension, new_extension); + return new_extension; +} +/** + * Load all Chrome extensions from browser targets + */ +export async function getChromeExtensionsFromBrowser(browser, extensions) { + logger.info(`[⚙️] Loading ${extensions.length} chrome extensions from browser...`); + // Find loaded extensions at runtime by checking all browser targets + const targets = await browser.targets(); + for (const target of targets) { + await loadExtensionFromTarget(extensions, target); + } + return extensions; +} +/** + * Install extensions from persona configuration + */ +export async function installExtensionsFromConfig(extensionConfigs, extensionsDir) { + logger.info('*************************************************************************'); + logger.info(`[⚙️] Installing ${extensionConfigs.length} chrome extensions...`); + const extensions = []; + try { + // Install each extension + for (const config of extensionConfigs) { + const extension = await loadOrInstallExtension(config, extensionsDir); + extensions.push(extension); + } + } + catch (error) { + logger.error(`[❌] Extension installation failed: ${error.message}`); + } + logger.info('*************************************************************************'); + return extensions; +} diff --git a/dist/browser/index.d.ts b/dist/browser/index.d.ts new file mode 100644 index 00000000..ba19fc92 --- /dev/null +++ b/dist/browser/index.d.ts @@ -0,0 +1,14 @@ +export * from './profile.js'; +export * from './views.js'; +export * from './utils.js'; +export * from './cloud/index.js'; +export * from './session.js'; +export * from './extensions.js'; +export * from './dvd-screensaver.js'; +export * from './playwright-manager.js'; +export * from './events.js'; +export * from './watchdogs/index.js'; +export * from './session-manager.js'; +export type { Browser, BrowserConfig, BrowserContext, BrowserContextConfig, } from './context.js'; +export type { Browser as PlaywrightBrowser, BrowserContext as PlaywrightBrowserContext, Page, Locator, FrameLocator, ElementHandle, Playwright, PlaywrightOrPatchright, ClientCertificate, Geolocation, HttpCredentials, ProxySettings, StorageState, ViewportSize, } from './types.js'; +export { async_playwright } from './types.js'; diff --git a/dist/browser/index.js b/dist/browser/index.js new file mode 100644 index 00000000..77fb678c --- /dev/null +++ b/dist/browser/index.js @@ -0,0 +1,13 @@ +export * from './profile.js'; +export * from './views.js'; +export * from './utils.js'; +export * from './cloud/index.js'; +export * from './session.js'; +export * from './extensions.js'; +export * from './dvd-screensaver.js'; +export * from './playwright-manager.js'; +export * from './events.js'; +export * from './watchdogs/index.js'; +export * from './session-manager.js'; +// Re-export the async playwright loader +export { async_playwright } from './types.js'; diff --git a/dist/browser/playwright-manager.d.ts b/dist/browser/playwright-manager.d.ts new file mode 100644 index 00000000..e9c6f588 --- /dev/null +++ b/dist/browser/playwright-manager.d.ts @@ -0,0 +1,47 @@ +/** + * Playwright Global Singleton Manager + * + * Manages Playwright instances at the event loop level to prevent + * duplicate instantiation and ensure proper resource cleanup. + * + * This is important because: + * 1. Playwright instances are heavy and should be reused + * 2. Multiple instances can cause port conflicts + * 3. Proper cleanup prevents resource leaks + */ +/** + * Get or create a Playwright instance for the current event loop + * Uses singleton pattern to prevent duplicate instances + */ +export declare function getPlaywrightInstance(options?: { + browserType?: 'chromium' | 'firefox' | 'webkit'; + forceNew?: boolean; +}): Promise; +/** + * Release a Playwright instance reference + * Decrements reference count and cleans up if no more references + */ +export declare function releasePlaywrightInstance(browserType?: 'chromium' | 'firefox' | 'webkit'): Promise; +/** + * Force cleanup of all Playwright instances + * Should be called on process exit + */ +export declare function cleanupAllPlaywrightInstances(): Promise; +/** + * Register a cleanup handler to be called on shutdown + */ +export declare function registerCleanupHandler(handler: () => Promise): void; +/** + * Unregister a cleanup handler + */ +export declare function unregisterCleanupHandler(handler: () => Promise): void; +/** + * Get statistics about Playwright instances + */ +export declare function getPlaywrightStats(): { + totalInstances: number; + instances: Array<{ + key: string; + refs: number; + }>; +}; diff --git a/dist/browser/playwright-manager.js b/dist/browser/playwright-manager.js new file mode 100644 index 00000000..e363cfc6 --- /dev/null +++ b/dist/browser/playwright-manager.js @@ -0,0 +1,146 @@ +/** + * Playwright Global Singleton Manager + * + * Manages Playwright instances at the event loop level to prevent + * duplicate instantiation and ensure proper resource cleanup. + * + * This is important because: + * 1. Playwright instances are heavy and should be reused + * 2. Multiple instances can cause port conflicts + * 3. Proper cleanup prevents resource leaks + */ +import { createLogger } from '../logging-config.js'; +const logger = createLogger('browser_use.playwright_manager'); +// Global registry of Playwright instances keyed by event loop/process +const playwrightInstances = new Map(); +const instanceRefCounts = new Map(); +// Track cleanup handlers +const cleanupHandlers = new Set(); +/** + * Get or create a Playwright instance for the current event loop + * Uses singleton pattern to prevent duplicate instances + */ +export async function getPlaywrightInstance(options = {}) { + const { browserType = 'chromium', forceNew = false } = options; + // Use process ID as key for singleton (Node.js is single event loop per process) + const instanceKey = `${process.pid}-${browserType}`; + // Return existing instance if available and not forcing new + if (!forceNew && playwrightInstances.has(instanceKey)) { + const instance = playwrightInstances.get(instanceKey); + // Increment reference count + instanceRefCounts.set(instanceKey, (instanceRefCounts.get(instanceKey) || 0) + 1); + logger.debug(`Reusing Playwright ${browserType} instance (refs: ${instanceRefCounts.get(instanceKey)})`); + return instance; + } + // Create new instance + logger.info(`Creating new Playwright ${browserType} instance`); + try { + const playwright = await import('playwright'); + const instance = playwright[browserType]; + // Store instance + playwrightInstances.set(instanceKey, instance); + instanceRefCounts.set(instanceKey, 1); + logger.debug(`Playwright ${browserType} instance created successfully`); + return instance; + } + catch (error) { + logger.error(`Failed to create Playwright instance: ${error.message}`); + throw error; + } +} +/** + * Release a Playwright instance reference + * Decrements reference count and cleans up if no more references + */ +export async function releasePlaywrightInstance(browserType = 'chromium') { + const instanceKey = `${process.pid}-${browserType}`; + if (!playwrightInstances.has(instanceKey)) { + logger.warning(`Attempted to release non-existent Playwright instance: ${instanceKey}`); + return; + } + // Decrement reference count + const currentRefs = instanceRefCounts.get(instanceKey) || 0; + const newRefs = Math.max(0, currentRefs - 1); + instanceRefCounts.set(instanceKey, newRefs); + logger.debug(`Released Playwright ${browserType} instance reference (refs: ${newRefs})`); + // If no more references, we could clean up, but Playwright itself doesn't need cleanup + // The actual browser instances are cleaned up separately + if (newRefs === 0) { + logger.debug(`No more references to Playwright ${browserType} instance, keeping for reuse`); + // We keep the instance for potential reuse rather than deleting it + } +} +/** + * Force cleanup of all Playwright instances + * Should be called on process exit + */ +export async function cleanupAllPlaywrightInstances() { + logger.info(`Cleaning up ${playwrightInstances.size} Playwright instances`); + // Execute all registered cleanup handlers + const cleanupPromises = Array.from(cleanupHandlers).map((handler) => handler().catch((error) => { + logger.error(`Cleanup handler failed: ${error.message}`); + })); + await Promise.all(cleanupPromises); + // Clear registries + playwrightInstances.clear(); + instanceRefCounts.clear(); + cleanupHandlers.clear(); + logger.info('All Playwright instances cleaned up'); +} +/** + * Register a cleanup handler to be called on shutdown + */ +export function registerCleanupHandler(handler) { + cleanupHandlers.add(handler); +} +/** + * Unregister a cleanup handler + */ +export function unregisterCleanupHandler(handler) { + cleanupHandlers.delete(handler); +} +/** + * Get statistics about Playwright instances + */ +export function getPlaywrightStats() { + return { + totalInstances: playwrightInstances.size, + instances: Array.from(playwrightInstances.keys()).map((key) => ({ + key, + refs: instanceRefCounts.get(key) || 0, + })), + }; +} +// Setup process exit handlers +let exitHandlersRegistered = false; +function registerExitHandlers() { + if (exitHandlersRegistered) { + return; + } + exitHandlersRegistered = true; + const exitHandler = async (signal) => { + logger.debug(`Received ${signal}, cleaning up Playwright instances...`); + await cleanupAllPlaywrightInstances(); + process.exit(0); + }; + // Register handlers for various exit signals + process.on('SIGINT', () => exitHandler('SIGINT')); + process.on('SIGTERM', () => exitHandler('SIGTERM')); + process.on('exit', () => { + // Synchronous cleanup on exit + logger.debug('Process exiting, Playwright cleanup complete'); + }); + // Handle uncaught exceptions + process.on('uncaughtException', async (error) => { + logger.error(`Uncaught exception: ${error.message}`); + await cleanupAllPlaywrightInstances(); + process.exit(1); + }); + process.on('unhandledRejection', async (reason) => { + logger.error(`Unhandled rejection: ${reason}`); + await cleanupAllPlaywrightInstances(); + process.exit(1); + }); +} +// Auto-register exit handlers on module load +registerExitHandlers(); diff --git a/dist/browser/profile.d.ts b/dist/browser/profile.d.ts new file mode 100644 index 00000000..715eb559 --- /dev/null +++ b/dist/browser/profile.d.ts @@ -0,0 +1,203 @@ +import type { ClientCertificate, Geolocation, HttpCredentials, ProxySettings, ViewportSize, StorageState } from './types.js'; +export declare const CHROME_DEBUG_PORT = 9242; +export declare const DOMAIN_OPTIMIZATION_THRESHOLD = 100; +export declare const CHROME_DISABLED_COMPONENTS: string[]; +export declare const CHROME_HEADLESS_ARGS: string[]; +export declare const CHROME_DOCKER_ARGS: string[]; +export declare const CHROME_DISABLE_SECURITY_ARGS: string[]; +export declare const CHROME_DETERMINISTIC_RENDERING_ARGS: string[]; +export declare const CHROME_DEFAULT_ARGS: string[]; +export declare const get_display_size: () => ViewportSize | null; +export declare const get_window_adjustments: () => [number, number]; +export declare enum ColorScheme { + LIGHT = "light", + DARK = "dark", + NO_PREFERENCE = "no-preference", + NULL = "null" +} +export declare enum Contrast { + NO_PREFERENCE = "no-preference", + MORE = "more", + NULL = "null" +} +export declare enum ReducedMotion { + REDUCE = "reduce", + NO_PREFERENCE = "no-preference", + NULL = "null" +} +export declare enum ForcedColors { + ACTIVE = "active", + NONE = "none", + NULL = "null" +} +export declare enum ServiceWorkers { + ALLOW = "allow", + BLOCK = "block" +} +export declare enum RecordHarContent { + OMIT = "omit", + EMBED = "embed", + ATTACH = "attach" +} +export declare enum RecordHarMode { + FULL = "full", + MINIMAL = "minimal" +} +export declare enum BrowserChannel { + CHROMIUM = "chromium", + CHROME = "chrome", + CHROME_BETA = "chrome-beta", + CHROME_DEV = "chrome-dev", + CHROME_CANARY = "chrome-canary", + MSEDGE = "msedge", + MSEDGE_BETA = "msedge-beta", + MSEDGE_DEV = "msedge-dev", + MSEDGE_CANARY = "msedge-canary" +} +export declare const BROWSERUSE_DEFAULT_CHANNEL = BrowserChannel.CHROMIUM; +type Nullable = T | null; +type WindowRect = { + width: number; + height: number; +}; +export interface BrowserContextArgs { + accept_downloads: boolean; + offline: boolean; + strict_selectors: boolean; + proxy: Nullable; + permissions: string[]; + bypass_csp: boolean; + client_certificates: ClientCertificate[]; + extra_http_headers: Record; + http_credentials: Nullable; + ignore_https_errors: boolean; + java_script_enabled: boolean; + base_url: Nullable; + service_workers: ServiceWorkers; + user_agent: Nullable; + screen: Nullable; + viewport: Nullable; + no_viewport: Nullable; + device_scale_factor: Nullable; + is_mobile: boolean; + has_touch: boolean; + locale: Nullable; + geolocation: Nullable; + timezone_id: Nullable; + color_scheme: ColorScheme; + contrast: Contrast; + reduced_motion: ReducedMotion; + forced_colors: ForcedColors; + record_har_content: RecordHarContent; + record_har_mode: RecordHarMode; + record_har_omit_content: boolean; + record_har_path: Nullable; + record_har_url_filter: Nullable; + record_video_dir: Nullable; + record_video_size: Nullable; +} +export interface BrowserConnectArgs { + headers: Nullable>; + slow_mo: number; + timeout: number; +} +export interface BrowserLaunchArgs { + env: Nullable>; + executable_path: Nullable; + headless: Nullable; + args: string[]; + ignore_default_args: string[] | true; + channel: Nullable; + chromium_sandbox: boolean; + devtools: boolean; + slow_mo: number; + timeout: number; + proxy: Nullable; + downloads_path: Nullable; + traces_dir: Nullable; + handle_sighup: boolean; + handle_sigint: boolean; + handle_sigterm: boolean; +} +export type BrowserNewContextArgs = BrowserContextArgs & { + storage_state: Nullable>; +}; +export type BrowserLaunchPersistentContextArgs = BrowserContextArgs & BrowserLaunchArgs & { + user_data_dir: Nullable; +}; +export interface BrowserProfileSpecificOptions { + id: string; + user_data_dir: Nullable; + storage_state: Nullable>; + stealth: boolean; + disable_security: boolean; + deterministic_rendering: boolean; + allowed_domains: Nullable>; + prohibited_domains: Nullable>; + block_ip_addresses: boolean; + keep_alive: Nullable; + enable_default_extensions: boolean; + captcha_solver: boolean; + window_size: Nullable; + window_height: Nullable; + window_width: Nullable; + window_position: Nullable; + default_navigation_timeout: Nullable; + default_timeout: Nullable; + minimum_wait_page_load_time: number; + wait_for_network_idle_page_load_time: number; + maximum_wait_page_load_time: number; + wait_between_actions: number; + include_dynamic_attributes: boolean; + highlight_elements: boolean; + viewport_expansion: number; + profile_directory: string; + cookies_file: Nullable; +} +export type BrowserProfileOptions = BrowserContextArgs & BrowserLaunchArgs & BrowserConnectArgs & BrowserProfileSpecificOptions; +export declare class BrowserProfile { + private options; + constructor(init?: Partial); + toString(): string; + describe(): string; + get config(): BrowserProfileOptions; + get allowed_domains(): Nullable>; + get prohibited_domains(): Nullable>; + get block_ip_addresses(): boolean; + get cookies_file(): Nullable; + get default_navigation_timeout(): Nullable; + get downloads_path(): Nullable; + get highlight_elements(): boolean; + get keep_alive(): boolean | null; + set keep_alive(value: boolean | null); + get maximum_wait_page_load_time(): number; + get traces_dir(): Nullable; + get user_data_dir(): Nullable; + get viewport_expansion(): number; + get viewport(): Nullable; + get wait_for_network_idle_page_load_time(): number; + get window_size(): Nullable; + private applyLegacyWindowSize; + private warnStorageStateUserDataDirConflict; + private warnUserDataDirNonDefault; + private warnDeterministicRenderingWeirdness; + private ensureDefaultDownloadsPath; + private getDefaultArgsList; + private getWindowSizeArgs; + private getWindowPositionArgs; + private getExtensionArgs; + private ensureDefaultExtensionsDownloaded; + private downloadExtension; + private extractExtension; + private stripCrxHeader; + getArgs(): Promise; + detect_display_configuration(): Promise; + private cloneContextArgs; + private cloneLaunchArgs; + kwargs_for_new_context(): BrowserNewContextArgs; + kwargs_for_connect(): BrowserConnectArgs; + kwargs_for_launch(): Promise; + kwargs_for_launch_persistent_context(): Promise; +} +export declare const DEFAULT_BROWSER_PROFILE: BrowserProfile; +export {}; diff --git a/dist/browser/profile.js b/dist/browser/profile.js new file mode 100644 index 00000000..b68ce703 --- /dev/null +++ b/dist/browser/profile.js @@ -0,0 +1,899 @@ +import fs from 'node:fs'; +import { promises as fsp } from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import https from 'node:https'; +import { randomUUID } from 'node:crypto'; +import AdmZip from 'adm-zip'; +import { CONFIG } from '../config.js'; +import { observe_debug } from '../observability.js'; +import { createLogger } from '../logging-config.js'; +import { log_pretty_path, uuid7str } from '../utils.js'; +const logger = createLogger('browser_use.browser.profile'); +export const CHROME_DEBUG_PORT = 9242; +export const DOMAIN_OPTIMIZATION_THRESHOLD = 100; +export const CHROME_DISABLED_COMPONENTS = [ + 'AcceptCHFrame', + 'AutoExpandDetailsElement', + 'AvoidUnnecessaryBeforeUnloadCheckSync', + 'CertificateTransparencyComponentUpdater', + 'DestroyProfileOnBrowserClose', + 'DialMediaRouteProvider', + 'ExtensionManifestV2Disabled', + 'GlobalMediaControls', + 'HttpsUpgrades', + 'ImprovedCookieControls', + 'LazyFrameLoading', + 'LensOverlay', + 'MediaRouter', + 'PaintHolding', + 'ThirdPartyStoragePartitioning', + 'Translate', + 'AutomationControlled', + 'BackForwardCache', + 'OptimizationHints', + 'ProcessPerSiteUpToMainFrameThreshold', + 'InterestFeedContentSuggestions', + 'CalculateNativeWinOcclusion', + 'HeavyAdPrivacyMitigations', + 'PrivacySandboxSettings4', + 'AutofillServerCommunication', + 'CrashReporting', + 'OverscrollHistoryNavigation', + 'InfiniteSessionRestore', + 'ExtensionDisableUnsupportedDeveloper', + 'ExtensionManifestV2Unsupported', +]; +export const CHROME_HEADLESS_ARGS = ['--headless=new']; +export const CHROME_DOCKER_ARGS = [ + '--no-sandbox', + '--disable-gpu-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--no-xshm', + '--no-zygote', + '--disable-site-isolation-trials', +]; +export const CHROME_DISABLE_SECURITY_ARGS = [ + '--disable-site-isolation-trials', + '--disable-web-security', + '--disable-features=IsolateOrigins,site-per-process', + '--allow-running-insecure-content', + '--ignore-certificate-errors', + '--ignore-ssl-errors', + '--ignore-certificate-errors-spki-list', +]; +export const CHROME_DETERMINISTIC_RENDERING_ARGS = [ + '--deterministic-mode', + '--js-flags=--random-seed=1157259159', + '--force-device-scale-factor=2', + '--enable-webgl', + '--font-render-hinting=none', + '--force-color-profile=srgb', +]; +export const CHROME_DEFAULT_ARGS = [ + '--disable-field-trial-config', + '--disable-background-networking', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-back-forward-cache', + '--disable-breakpad', + '--disable-client-side-phishing-detection', + '--disable-component-extensions-with-background-pages', + '--disable-component-update', + '--no-default-browser-check', + '--disable-dev-shm-usage', + '--disable-hang-monitor', + '--disable-ipc-flooding-protection', + '--disable-popup-blocking', + '--disable-prompt-on-repost', + '--disable-renderer-backgrounding', + '--metrics-recording-only', + '--no-first-run', + '--no-service-autorun', + '--export-tagged-pdf', + '--disable-search-engine-choice-screen', + '--unsafely-disable-devtools-self-xss-warnings', + '--enable-features=NetworkService,NetworkServiceInProcess', + '--enable-network-information-downlink-max', + '--test-type=gpu', + '--disable-sync', + '--allow-legacy-extension-manifests', + '--allow-pre-commit-input', + '--disable-blink-features=AutomationControlled', + '--install-autogenerated-theme=0,0,0', + '--log-level=2', + '--disable-focus-on-load', + '--disable-window-activation', + '--generate-pdf-document-outline', + '--no-pings', + '--ash-no-nudges', + '--disable-infobars', + '--simulate-outdated-no-au="Tue, 31 Dec 2099 23:59:59 GMT"', + '--hide-crash-restore-bubble', + '--suppress-message-center-popups', + '--disable-domain-reliability', + '--disable-datasaver-prompt', + '--disable-speech-synthesis-api', + '--disable-speech-api', + '--disable-print-preview', + '--safebrowsing-disable-auto-update', + '--disable-external-intent-requests', + '--disable-desktop-notifications', + '--noerrdialogs', + '--silent-debugger-extension-api', + '--disable-extensions-http-throttling', + '--extensions-on-chrome-urls', + '--disable-default-apps', + `--disable-features=${CHROME_DISABLED_COMPONENTS.join(',')}`, +]; +const DEFAULT_EXTENSIONS = [ + { + name: 'uBlock Origin', + id: 'cjpalhdlnbpafiamejdnhcphjbkeiagm', + url: 'https://clients2.google.com/service/update2/crx?response=redirect&prodversion=133&acceptformat=crx3&x=id%3Dcjpalhdlnbpafiamejdnhcphjbkeiagm%26uc', + }, + { + name: "I still don't care about cookies", + id: 'edibdbjcniadpccecjdfdjjppcpchdlm', + url: 'https://clients2.google.com/service/update2/crx?response=redirect&prodversion=133&acceptformat=crx3&x=id%3Dedibdbjcniadpccecjdfdjjppcpchdlm%26uc', + }, + { + name: 'ClearURLs', + id: 'lckanjgmijmafbedllaakclkaicjfmnk', + url: 'https://clients2.google.com/service/update2/crx?response=redirect&prodversion=133&acceptformat=crx3&x=id%3Dlckanjgmijmafbedllaakclkaicjfmnk%26uc', + }, + { + name: 'Force Background Tab', + id: 'gidlfommnbibbmegmgajdbikelkdcmcl', + url: 'https://clients2.google.com/service/update2/crx?response=redirect&prodversion=133&acceptformat=crx3&x=id%3Dgidlfommnbibbmegmgajdbikelkdcmcl%26uc', + }, +]; +const getEnableDefaultExtensionsDefault = () => { + const envValue = process.env.BROWSER_USE_DISABLE_EXTENSIONS; + if (typeof envValue === 'undefined') { + return true; + } + const normalized = envValue.trim().toLowerCase(); + if (normalized === '' || + normalized === '0' || + normalized === 'false' || + normalized === 'no' || + normalized === 'off') { + return true; + } + return false; +}; +const parseDisplayEnv = () => { + const width = process.env.BROWSER_USE_SCREEN_WIDTH; + const height = process.env.BROWSER_USE_SCREEN_HEIGHT; + if (width && height) { + const parsedWidth = Number(width); + const parsedHeight = Number(height); + if (!Number.isNaN(parsedWidth) && !Number.isNaN(parsedHeight)) { + return { width: parsedWidth, height: parsedHeight }; + } + } + return null; +}; +export const get_display_size = () => { + // Node.js lacks a portable cross-platform API for monitor size detection. + // Support manual overrides via env vars and fall back to null otherwise. + return parseDisplayEnv(); +}; +export const get_window_adjustments = () => { + if (process.platform === 'darwin') { + return [-4, 24]; + } + if (process.platform === 'win32') { + return [-8, 0]; + } + return [0, 0]; +}; +const validate_url = (url, schemes = []) => { + try { + const parsed = new URL(url); + if (schemes.length > 0 && parsed.protocol) { + const lowered = parsed.protocol.replace(':', '').toLowerCase(); + if (!schemes.map((s) => s.toLowerCase()).includes(lowered)) { + throw new Error(); + } + } + return url; + } + catch { + throw new Error(`Invalid URL format: ${url}`); + } +}; +const validate_float_range = (value, min, max) => { + if (value < min || value > max) { + throw new Error(`Value ${value} outside of range ${min}-${max}`); + } + return value; +}; +const validate_cli_arg = (arg) => { + if (!arg.startsWith('--')) { + throw new Error(`Invalid CLI argument: ${arg} (should start with --)`); + } + return arg; +}; +export var ColorScheme; +(function (ColorScheme) { + ColorScheme["LIGHT"] = "light"; + ColorScheme["DARK"] = "dark"; + ColorScheme["NO_PREFERENCE"] = "no-preference"; + ColorScheme["NULL"] = "null"; +})(ColorScheme || (ColorScheme = {})); +export var Contrast; +(function (Contrast) { + Contrast["NO_PREFERENCE"] = "no-preference"; + Contrast["MORE"] = "more"; + Contrast["NULL"] = "null"; +})(Contrast || (Contrast = {})); +export var ReducedMotion; +(function (ReducedMotion) { + ReducedMotion["REDUCE"] = "reduce"; + ReducedMotion["NO_PREFERENCE"] = "no-preference"; + ReducedMotion["NULL"] = "null"; +})(ReducedMotion || (ReducedMotion = {})); +export var ForcedColors; +(function (ForcedColors) { + ForcedColors["ACTIVE"] = "active"; + ForcedColors["NONE"] = "none"; + ForcedColors["NULL"] = "null"; +})(ForcedColors || (ForcedColors = {})); +export var ServiceWorkers; +(function (ServiceWorkers) { + ServiceWorkers["ALLOW"] = "allow"; + ServiceWorkers["BLOCK"] = "block"; +})(ServiceWorkers || (ServiceWorkers = {})); +export var RecordHarContent; +(function (RecordHarContent) { + RecordHarContent["OMIT"] = "omit"; + RecordHarContent["EMBED"] = "embed"; + RecordHarContent["ATTACH"] = "attach"; +})(RecordHarContent || (RecordHarContent = {})); +export var RecordHarMode; +(function (RecordHarMode) { + RecordHarMode["FULL"] = "full"; + RecordHarMode["MINIMAL"] = "minimal"; +})(RecordHarMode || (RecordHarMode = {})); +export var BrowserChannel; +(function (BrowserChannel) { + BrowserChannel["CHROMIUM"] = "chromium"; + BrowserChannel["CHROME"] = "chrome"; + BrowserChannel["CHROME_BETA"] = "chrome-beta"; + BrowserChannel["CHROME_DEV"] = "chrome-dev"; + BrowserChannel["CHROME_CANARY"] = "chrome-canary"; + BrowserChannel["MSEDGE"] = "msedge"; + BrowserChannel["MSEDGE_BETA"] = "msedge-beta"; + BrowserChannel["MSEDGE_DEV"] = "msedge-dev"; + BrowserChannel["MSEDGE_CANARY"] = "msedge-canary"; +})(BrowserChannel || (BrowserChannel = {})); +export const BROWSERUSE_DEFAULT_CHANNEL = BrowserChannel.CHROMIUM; +const DEFAULT_PERMISSIONS = [ + 'clipboard-read', + 'clipboard-write', + 'notifications', +]; +const DEFAULT_IGNORE_ARGS = [ + '--enable-automation', + '--disable-extensions', + '--hide-scrollbars', + '--disable-features=AcceptCHFrame,AutoExpandDetailsElement,AvoidUnnecessaryBeforeUnloadCheckSync,CertificateTransparencyComponentUpdater,DeferRendererTasksAfterInput,DestroyProfileOnBrowserClose,DialMediaRouteProvider,ExtensionManifestV2Disabled,GlobalMediaControls,HttpsUpgrades,ImprovedCookieControls,LazyFrameLoading,LensOverlay,MediaRouter,PaintHolding,ThirdPartyStoragePartitioning,Translate', +]; +const DEFAULT_BROWSER_PROFILE_OPTIONS = { + accept_downloads: true, + offline: false, + strict_selectors: false, + proxy: null, + permissions: DEFAULT_PERMISSIONS.slice(), + bypass_csp: false, + client_certificates: [], + extra_http_headers: {}, + http_credentials: null, + ignore_https_errors: false, + java_script_enabled: true, + base_url: null, + service_workers: ServiceWorkers.ALLOW, + user_agent: null, + screen: null, + viewport: null, + no_viewport: null, + device_scale_factor: null, + is_mobile: false, + has_touch: false, + locale: null, + geolocation: null, + timezone_id: null, + color_scheme: ColorScheme.LIGHT, + contrast: Contrast.NO_PREFERENCE, + reduced_motion: ReducedMotion.NO_PREFERENCE, + forced_colors: ForcedColors.NONE, + record_har_content: RecordHarContent.EMBED, + record_har_mode: RecordHarMode.FULL, + record_har_omit_content: false, + record_har_path: null, + record_har_url_filter: null, + record_video_dir: null, + record_video_size: null, + headers: null, + slow_mo: 0, + timeout: 30000, + env: null, + executable_path: null, + headless: null, + args: [], + ignore_default_args: DEFAULT_IGNORE_ARGS.slice(), + channel: null, + chromium_sandbox: !CONFIG.IN_DOCKER, + devtools: false, + downloads_path: null, + traces_dir: null, + handle_sighup: true, + handle_sigint: false, + handle_sigterm: false, + id: uuid7str(), + user_data_dir: CONFIG.BROWSER_USE_DEFAULT_USER_DATA_DIR, + storage_state: null, + stealth: false, + disable_security: false, + deterministic_rendering: false, + allowed_domains: null, + prohibited_domains: null, + block_ip_addresses: false, + keep_alive: null, + enable_default_extensions: getEnableDefaultExtensionsDefault(), + captcha_solver: true, + window_size: null, + window_height: null, + window_width: null, + window_position: { width: 0, height: 0 }, + default_navigation_timeout: null, + default_timeout: null, + minimum_wait_page_load_time: 0.25, + wait_for_network_idle_page_load_time: 0.5, + maximum_wait_page_load_time: 5.0, + wait_between_actions: 0.1, + include_dynamic_attributes: true, + highlight_elements: true, + viewport_expansion: 500, + profile_directory: 'Default', + cookies_file: null, +}; +const splitArgOnce = (arg) => { + const separatorIndex = arg.indexOf('='); + if (separatorIndex === -1) { + return [arg, '']; + } + return [arg.slice(0, separatorIndex), arg.slice(separatorIndex + 1)]; +}; +const argsAsDict = (args) => { + const result = {}; + for (const arg of args) { + const [keyPart, valuePart = ''] = splitArgOnce(arg); + const key = keyPart.trim().replace(/^-+/, ''); + result[key] = valuePart.trim(); + } + return result; +}; +const argsAsList = (args) => Object.entries(args).map(([key, value]) => value + ? `--${key.replace(/^-+/, '')}=${value}` + : `--${key.replace(/^-+/, '')}`); +const cloneDefaultOptions = () => JSON.parse(JSON.stringify(DEFAULT_BROWSER_PROFILE_OPTIONS)); +const normalizeDomainEntry = (entry) => String(entry ?? '') + .trim() + .toLowerCase(); +const isExactHostDomainEntry = (entry) => { + if (!entry) { + return false; + } + if (entry.includes('*') || entry.includes('://') || entry.includes('/')) { + return false; + } + // Keep set optimization for plain hostnames only. Entries with ports/pattern-like + // delimiters must stay as arrays to preserve wildcard/scheme matching semantics. + return !entry.includes(':'); +}; +const optimizeDomainList = (value) => { + const cleaned = value.map(normalizeDomainEntry).filter(Boolean); + const canOptimizeToSet = cleaned.length >= DOMAIN_OPTIMIZATION_THRESHOLD && + cleaned.every(isExactHostDomainEntry); + if (canOptimizeToSet) { + logger.warning(`Optimizing domain list with ${cleaned.length} entries to a Set for O(1) matching`); + return new Set(cleaned); + } + return cleaned; +}; +export class BrowserProfile { + options; + constructor(init = {}) { + const defaults = cloneDefaultOptions(); + this.options = { + ...defaults, + ...init, + permissions: init.permissions + ? [...init.permissions] + : defaults.permissions, + client_certificates: init.client_certificates + ? [...init.client_certificates] + : [], + extra_http_headers: init.extra_http_headers + ? { ...init.extra_http_headers } + : {}, + args: init.args ? init.args.map(validate_cli_arg) : [], + ignore_default_args: Array.isArray(init.ignore_default_args) + ? init.ignore_default_args.map(validate_cli_arg) + : (init.ignore_default_args ?? defaults.ignore_default_args), + allowed_domains: Array.isArray(init.allowed_domains) + ? optimizeDomainList(init.allowed_domains) + : init.allowed_domains instanceof Set + ? optimizeDomainList(Array.from(init.allowed_domains)) + : defaults.allowed_domains, + prohibited_domains: Array.isArray(init.prohibited_domains) + ? optimizeDomainList(init.prohibited_domains) + : init.prohibited_domains instanceof Set + ? optimizeDomainList(Array.from(init.prohibited_domains)) + : defaults.prohibited_domains, + window_position: init.window_position ?? defaults.window_position, + }; + this.options.id = init.id ?? uuid7str(); + this.ensureDefaultDownloadsPath(); + this.applyLegacyWindowSize(); + this.warnStorageStateUserDataDirConflict(); + this.warnUserDataDirNonDefault(); + this.warnDeterministicRenderingWeirdness(); + } + toString() { + return `BrowserProfile#${this.options.id.slice(-4)}`; + } + describe() { + const shortDir = this.options.user_data_dir + ? log_pretty_path(this.options.user_data_dir) + : ''; + return `BrowserProfile#${this.options.id.slice(-4)}(user_data_dir=${shortDir}, headless=${this.options.headless})`; + } + get config() { + return this.options; + } + // Public getters for commonly accessed properties + get allowed_domains() { + return this.options.allowed_domains; + } + get prohibited_domains() { + return this.options.prohibited_domains; + } + get block_ip_addresses() { + return this.options.block_ip_addresses; + } + get cookies_file() { + return this.options.cookies_file; + } + get default_navigation_timeout() { + return this.options.default_navigation_timeout; + } + get downloads_path() { + return this.options.downloads_path; + } + get highlight_elements() { + return this.options.highlight_elements; + } + get keep_alive() { + return this.options.keep_alive; + } + set keep_alive(value) { + this.options.keep_alive = value; + } + get maximum_wait_page_load_time() { + return this.options.maximum_wait_page_load_time; + } + get traces_dir() { + return this.options.traces_dir; + } + get user_data_dir() { + return this.options.user_data_dir; + } + get viewport_expansion() { + return this.options.viewport_expansion; + } + get viewport() { + return this.options.viewport; + } + get wait_for_network_idle_page_load_time() { + return this.options.wait_for_network_idle_page_load_time; + } + get window_size() { + return this.options.window_size; + } + applyLegacyWindowSize() { + const { window_width, window_height } = this.options; + if (window_width || window_height) { + logger.warning('⚠️ BrowserProfile(window_width=..., window_height=...) are deprecated, use BrowserProfile(window_size={"width": 1920, "height": 1080}) instead.'); + const newSize = { + ...(this.options.window_size ?? { width: 0, height: 0 }), + }; + newSize.width = newSize.width || window_width || 1920; + newSize.height = newSize.height || window_height || 1080; + this.options.window_size = newSize; + } + } + warnStorageStateUserDataDirConflict() { + const hasStorageState = this.options.storage_state !== null; + const hasUserDataDir = Boolean(this.options.user_data_dir && + !this.options.user_data_dir.toLowerCase().includes('tmp')); + const hasCookiesFile = this.options.cookies_file !== null; + const staticSource = hasCookiesFile + ? 'cookies_file' + : hasStorageState + ? 'storage_state' + : null; + if (staticSource && hasUserDataDir) { + logger.warning(`⚠️ BrowserSession(...) was passed both ${staticSource} AND user_data_dir. ${staticSource}=${this.options.storage_state ?? this.options.cookies_file} will forcibly overwrite cookies/localStorage/sessionStorage in user_data_dir=${this.options.user_data_dir}. For multiple browsers in parallel, use only storage_state with user_data_dir=None, or use a separate user_data_dir for each browser and set storage_state=None.`); + } + } + warnUserDataDirNonDefault() { + const isNotDefault = Boolean(this.options.executable_path || + (this.options.channel && + this.options.channel !== BROWSERUSE_DEFAULT_CHANNEL)); + if (this.options.user_data_dir === CONFIG.BROWSER_USE_DEFAULT_USER_DATA_DIR && + isNotDefault) { + const alternateName = this.options.executable_path + ? path + .basename(this.options.executable_path) + .toLowerCase() + .replace(/ /g, '-') + : this.options.channel + ? this.options.channel.toLowerCase() + : 'none'; + logger.warning(`⚠️ ${this} Changing user_data_dir=${log_pretty_path(this.options.user_data_dir)} ➡️ .../default-${alternateName} to avoid ${alternateName.toUpperCase()} corruping default profile created by ${BROWSERUSE_DEFAULT_CHANNEL}`); + const dir = path.dirname(CONFIG.BROWSER_USE_DEFAULT_USER_DATA_DIR); + this.options.user_data_dir = path.join(dir, `default-${alternateName}`); + } + } + warnDeterministicRenderingWeirdness() { + if (this.options.deterministic_rendering) { + logger.warning('⚠️ BrowserSession(deterministic_rendering=True) is NOT RECOMMENDED. It breaks many sites and increases chances of getting blocked by anti-bot systems. It hardcodes the JS random seed and forces browsers across Linux/Mac/Windows to use the same font rendering engine so that identical screenshots can be generated.'); + } + } + ensureDefaultDownloadsPath() { + if (this.options.downloads_path) { + return; + } + let downloadsPath = path.join(os.tmpdir(), `browser-use-downloads-${randomUUID().slice(0, 8)}`); + while (fs.existsSync(downloadsPath)) { + downloadsPath = path.join(os.tmpdir(), `browser-use-downloads-${randomUUID().slice(0, 8)}`); + } + fs.mkdirSync(downloadsPath, { recursive: true }); + this.options.downloads_path = downloadsPath; + } + getDefaultArgsList() { + const ignore = this.options.ignore_default_args; + if (Array.isArray(ignore)) { + const ignoreSet = new Set(ignore); + return CHROME_DEFAULT_ARGS.filter((arg) => !ignoreSet.has(arg)); + } + if (ignore === true) { + return []; + } + return [...CHROME_DEFAULT_ARGS]; + } + getWindowSizeArgs() { + const args = []; + const size = this.options.window_size; + if (size && size.width && size.height) { + args.push(`--window-size=${size.width},${size.height}`); + } + else if (!this.options.headless) { + args.push('--start-maximized'); + } + return args; + } + getWindowPositionArgs() { + const args = []; + const position = this.options.window_position; + if (position && + typeof position.width === 'number' && + typeof position.height === 'number') { + args.push(`--window-position=${position.width},${position.height}`); + } + return args; + } + async getExtensionArgs() { + const extensionPaths = await this.ensureDefaultExtensionsDownloaded(); + const args = [ + '--enable-extensions', + '--disable-extensions-file-access-check', + '--disable-extensions-http-throttling', + '--enable-extension-activity-logging', + ]; + if (extensionPaths.length) { + args.push(`--load-extension=${extensionPaths.join(',')}`); + } + return args; + } + async ensureDefaultExtensionsDownloaded() { + const cacheDir = CONFIG.BROWSER_USE_EXTENSIONS_DIR; + await fsp.mkdir(cacheDir, { recursive: true }); + const extensionPaths = []; + const loadedNames = []; + for (const ext of DEFAULT_EXTENSIONS) { + const extDir = path.join(cacheDir, ext.id); + const manifestPath = path.join(extDir, 'manifest.json'); + if (fs.existsSync(extDir) && fs.existsSync(manifestPath)) { + extensionPaths.push(extDir); + loadedNames.push(ext.name); + continue; + } + const crxFile = path.join(cacheDir, `${ext.id}.crx`); + try { + if (!fs.existsSync(crxFile)) { + logger.info(`📦 Downloading ${ext.name} extension...`); + await this.downloadExtension(ext.url, crxFile); + } + if (fs.existsSync(crxFile)) { + logger.info(`📂 Extracting ${ext.name} extension...`); + await this.extractExtension(crxFile, extDir); + extensionPaths.push(extDir); + loadedNames.push(ext.name); + } + } + catch (error) { + logger.warning(`⚠️ Failed to setup ${ext.name} extension: ${error.message}`); + } + } + if (extensionPaths.length) { + logger.info(`✅ Extensions ready: ${extensionPaths.length} extensions loaded (${loadedNames.join(', ')})`); + } + else { + logger.warning('⚠️ No default extensions could be loaded'); + } + return extensionPaths; + } + downloadExtension(url, outputPath, redirectCount = 0) { + const maxRedirects = 5; + return new Promise((resolve, reject) => { + https + .get(url, (response) => { + const { statusCode, headers } = response; + if (statusCode && + statusCode >= 300 && + statusCode < 400 && + headers.location && + redirectCount < maxRedirects) { + response.resume(); + this.downloadExtension(headers.location, outputPath, redirectCount + 1) + .then(resolve) + .catch(reject); + return; + } + if (!statusCode || statusCode >= 400) { + reject(new Error(`Failed to download extension (status ${statusCode ?? 'unknown'})`)); + return; + } + const chunks = []; + response.on('data', (chunk) => chunks.push(chunk)); + response.on('end', async () => { + try { + await fsp.writeFile(outputPath, Buffer.concat(chunks)); + resolve(); + } + catch (error) { + reject(error); + } + }); + }) + .on('error', reject); + }); + } + async extractExtension(crxPath, extractDir) { + if (fs.existsSync(extractDir)) { + await fsp.rm(extractDir, { recursive: true, force: true }); + } + await fsp.mkdir(extractDir, { recursive: true }); + const buffer = await fsp.readFile(crxPath); + try { + const zip = new AdmZip(buffer); + zip.extractAllTo(extractDir, true); + } + catch { + const zipBuffer = this.stripCrxHeader(buffer); + const zip = new AdmZip(zipBuffer); + zip.extractAllTo(extractDir, true); + } + if (!fs.existsSync(path.join(extractDir, 'manifest.json'))) { + throw new Error('No manifest.json found in extension'); + } + } + stripCrxHeader(buffer) { + const magic = buffer.subarray(0, 4).toString(); + if (magic !== 'Cr24') { + throw new Error('Invalid CRX file format'); + } + const version = buffer.readUInt32LE(4); + if (version === 2) { + const pubkeyLen = buffer.readUInt32LE(8); + const sigLen = buffer.readUInt32LE(12); + const offset = 16 + pubkeyLen + sigLen; + return buffer.subarray(offset); + } + if (version === 3) { + const headerLen = buffer.readUInt32LE(8); + const offset = 12 + headerLen; + return buffer.subarray(offset); + } + throw new Error(`Unsupported CRX version: ${version}`); + } + async getArgs() { + const defaultArgs = this.getDefaultArgsList(); + const preConversionArgs = [ + ...defaultArgs, + ...this.options.args, + `--profile-directory=${this.options.profile_directory}`, + ...(CONFIG.IN_DOCKER || !this.options.chromium_sandbox + ? CHROME_DOCKER_ARGS + : []), + ...(this.options.headless ? CHROME_HEADLESS_ARGS : []), + ...(this.options.disable_security ? CHROME_DISABLE_SECURITY_ARGS : []), + ...(this.options.deterministic_rendering + ? CHROME_DETERMINISTIC_RENDERING_ARGS + : []), + ...this.getWindowSizeArgs(), + ...this.getWindowPositionArgs(), + ...(this.options.enable_default_extensions + ? await this.getExtensionArgs() + : []), + ]; + const finalArgs = argsAsList(argsAsDict(preConversionArgs)); + return finalArgs; + } + async detect_display_configuration() { + const runner = observe_debug({ + name: 'detect_display_configuration', + ignore_input: true, + ignore_output: true, + })(async () => { + const displaySize = get_display_size(); + const hasScreen = Boolean(displaySize); + this.options.screen = this.options.screen || + displaySize || { width: 1920, height: 1080 }; + if (this.options.headless === null) { + this.options.headless = !hasScreen; + } + if (this.options.headless) { + this.options.viewport = + this.options.viewport || + this.options.window_size || + this.options.screen; + this.options.window_position = null; + this.options.window_size = null; + this.options.no_viewport = false; + } + else { + this.options.window_size = + this.options.window_size || this.options.screen; + this.options.no_viewport = this.options.no_viewport ?? true; + this.options.viewport = this.options.no_viewport + ? null + : this.options.viewport; + } + let useViewport = Boolean(this.options.headless) || + Boolean(this.options.viewport) || + Boolean(this.options.device_scale_factor); + if (this.options.no_viewport === null) { + this.options.no_viewport = !useViewport; + } + useViewport = !this.options.no_viewport; + if (useViewport) { + this.options.viewport = this.options.viewport || + this.options.screen || { width: 1920, height: 1080 }; + this.options.device_scale_factor = + this.options.device_scale_factor || 1.0; + } + else { + this.options.viewport = null; + this.options.device_scale_factor = null; + this.options.screen = null; + } + if (this.options.headless && this.options.no_viewport) { + throw new Error('headless=True and no_viewport=True cannot both be set at the same time'); + } + }); + return runner.call(this); + } + cloneContextArgs() { + const o = this.options; + return { + accept_downloads: o.accept_downloads, + offline: o.offline, + strict_selectors: o.strict_selectors, + proxy: o.proxy ? { ...o.proxy } : null, + permissions: [...o.permissions], + bypass_csp: o.bypass_csp, + client_certificates: o.client_certificates + ? [...o.client_certificates] + : [], + extra_http_headers: { ...o.extra_http_headers }, + http_credentials: o.http_credentials ? { ...o.http_credentials } : null, + ignore_https_errors: o.ignore_https_errors, + java_script_enabled: o.java_script_enabled, + base_url: o.base_url, + service_workers: o.service_workers, + user_agent: o.user_agent, + screen: o.screen ? { ...o.screen } : null, + viewport: o.viewport ? { ...o.viewport } : null, + no_viewport: o.no_viewport, + device_scale_factor: o.device_scale_factor, + is_mobile: o.is_mobile, + has_touch: o.has_touch, + locale: o.locale, + geolocation: o.geolocation ? { ...o.geolocation } : null, + timezone_id: o.timezone_id, + color_scheme: o.color_scheme, + contrast: o.contrast, + reduced_motion: o.reduced_motion, + forced_colors: o.forced_colors, + record_har_content: o.record_har_content, + record_har_mode: o.record_har_mode, + record_har_omit_content: o.record_har_omit_content, + record_har_path: o.record_har_path, + record_har_url_filter: o.record_har_url_filter, + record_video_dir: o.record_video_dir, + record_video_size: o.record_video_size + ? { ...o.record_video_size } + : null, + }; + } + cloneLaunchArgs(args) { + const o = this.options; + return { + env: o.env ? { ...o.env } : null, + executable_path: o.executable_path, + headless: o.headless, + args, + ignore_default_args: Array.isArray(o.ignore_default_args) + ? [...o.ignore_default_args] + : o.ignore_default_args, + channel: o.channel, + chromium_sandbox: o.chromium_sandbox, + devtools: o.devtools, + slow_mo: o.slow_mo, + timeout: o.timeout, + proxy: o.proxy ? { ...o.proxy } : null, + downloads_path: o.downloads_path, + traces_dir: o.traces_dir, + handle_sighup: o.handle_sighup, + handle_sigint: o.handle_sigint, + handle_sigterm: o.handle_sigterm, + }; + } + kwargs_for_new_context() { + return { + ...this.cloneContextArgs(), + storage_state: this.options.storage_state + ? typeof this.options.storage_state === 'string' + ? this.options.storage_state + : { ...this.options.storage_state } + : null, + }; + } + kwargs_for_connect() { + return { + headers: this.options.headers ? { ...this.options.headers } : null, + slow_mo: this.options.slow_mo, + timeout: this.options.timeout, + }; + } + async kwargs_for_launch() { + const args = await this.getArgs(); + return this.cloneLaunchArgs(args); + } + async kwargs_for_launch_persistent_context() { + const args = await this.getArgs(); + return { + ...this.cloneContextArgs(), + ...this.cloneLaunchArgs(args), + user_data_dir: this.options.user_data_dir, + }; + } +} +export const DEFAULT_BROWSER_PROFILE = new BrowserProfile(); diff --git a/dist/browser/session-manager.d.ts b/dist/browser/session-manager.d.ts new file mode 100644 index 00000000..40c28874 --- /dev/null +++ b/dist/browser/session-manager.d.ts @@ -0,0 +1,85 @@ +import type { TabInfo } from './views.js'; +export type SessionManagerTargetSource = 'tab' | 'cdp' | 'unknown'; +export interface SessionManagerTarget { + target_id: string; + target_type: string; + url: string; + title: string; + attached: boolean; + source: SessionManagerTargetSource; + first_seen_at: string; + last_seen_at: string; +} +export interface SessionManagerChannel { + session_id: string; + target_id: string; + attached_at: string; + last_seen_at: string; +} +export interface TargetAttachedPayload { + target_id: string; + session_id?: string | null; + target_type?: string; + url?: string; + title?: string; +} +export interface TargetDetachedPayload { + target_id: string; + session_id?: string | null; +} +export interface TargetInfoChangedPayload { + target_id: string; + target_type?: string; + url?: string; + title?: string; +} +export declare class SessionManager { + private _targets; + private _sessions; + private _target_sessions; + private _session_to_target; + private _page_targets; + private _tab_target_ids; + private _focused_target_id; + sync_tabs(tabs: TabInfo[], current_tab_index: number, target_id_factory: (page_id: number) => string): void; + handle_target_attached(payload: TargetAttachedPayload): void; + handle_target_detached(payload: TargetDetachedPayload): void; + handle_target_info_changed(payload: TargetInfoChangedPayload): void; + upsert_target(init: { + target_id: string; + target_type?: string; + url?: string; + title?: string; + attached?: boolean; + source?: SessionManagerTargetSource; + }): SessionManagerTarget; + remove_target(target_id: string): void; + upsert_session(session_id: string, target_id: string): void; + remove_session(session_id: string): void; + bind_page_to_target(page_id: number, target_id: string): void; + unbind_page(page_id: number): void; + set_focused_target(target_id: string | null): void; + get_focused_target_id(): string | null; + get_target(target_id: string): SessionManagerTarget | null; + get_session(session_id: string): SessionManagerChannel | null; + get_target_id_for_session(session_id: string): string | null; + get_target_id_for_page(page_id: number): string | null; + get_sessions_for_target(target_id: string): SessionManagerChannel[]; + get_all_targets(): { + target_id: string; + target_type: string; + url: string; + title: string; + attached: boolean; + source: SessionManagerTargetSource; + first_seen_at: string; + last_seen_at: string; + }[]; + get_all_sessions(): { + session_id: string; + target_id: string; + attached_at: string; + last_seen_at: string; + }[]; + clear(): void; +} diff --git a/dist/browser/session-manager.js b/dist/browser/session-manager.js new file mode 100644 index 00000000..02a8122e --- /dev/null +++ b/dist/browser/session-manager.js @@ -0,0 +1,208 @@ +export class SessionManager { + _targets = new Map(); + _sessions = new Map(); + _target_sessions = new Map(); + _session_to_target = new Map(); + _page_targets = new Map(); + _tab_target_ids = new Set(); + _focused_target_id = null; + sync_tabs(tabs, current_tab_index, target_id_factory) { + const nextTabTargetIds = new Set(); + const seenPageIds = new Set(); + for (const tab of tabs) { + const target_id = tab.target_id ?? target_id_factory(tab.page_id); + tab.target_id = target_id; + nextTabTargetIds.add(target_id); + seenPageIds.add(tab.page_id); + this.upsert_target({ + target_id, + target_type: 'page', + url: tab.url, + title: tab.title, + attached: true, + source: 'tab', + }); + this.bind_page_to_target(tab.page_id, target_id); + } + for (const target_id of [...this._tab_target_ids]) { + if (!nextTabTargetIds.has(target_id)) { + this.remove_target(target_id); + } + } + this._tab_target_ids = nextTabTargetIds; + for (const page_id of [...this._page_targets.keys()]) { + if (!seenPageIds.has(page_id)) { + this._page_targets.delete(page_id); + } + } + const currentTab = tabs[current_tab_index] ?? null; + this._focused_target_id = currentTab?.target_id ?? null; + } + handle_target_attached(payload) { + this.upsert_target({ + target_id: payload.target_id, + target_type: payload.target_type ?? 'page', + url: payload.url ?? '', + title: payload.title ?? '', + attached: true, + source: 'cdp', + }); + if (payload.session_id) { + this.upsert_session(payload.session_id, payload.target_id); + } + } + handle_target_detached(payload) { + if (payload.session_id) { + this.remove_session(payload.session_id); + } + const targetSessions = this._target_sessions.get(payload.target_id); + if (targetSessions && targetSessions.size > 0) { + return; + } + const target = this._targets.get(payload.target_id); + if (!target) { + return; + } + if (target.source === 'tab' && + this._tab_target_ids.has(payload.target_id)) { + target.attached = false; + target.last_seen_at = new Date().toISOString(); + this._targets.set(payload.target_id, target); + return; + } + this.remove_target(payload.target_id); + } + handle_target_info_changed(payload) { + const current = this._targets.get(payload.target_id); + this.upsert_target({ + target_id: payload.target_id, + target_type: payload.target_type ?? current?.target_type ?? 'page', + url: payload.url ?? current?.url ?? '', + title: payload.title ?? current?.title ?? '', + attached: current?.attached ?? true, + source: current?.source ?? 'cdp', + }); + } + upsert_target(init) { + const now = new Date().toISOString(); + const existing = this._targets.get(init.target_id); + const nextTarget = { + target_id: init.target_id, + target_type: init.target_type ?? existing?.target_type ?? 'page', + url: init.url ?? existing?.url ?? '', + title: init.title ?? existing?.title ?? '', + attached: init.attached ?? existing?.attached ?? true, + source: init.source ?? existing?.source ?? 'unknown', + first_seen_at: existing?.first_seen_at ?? now, + last_seen_at: now, + }; + this._targets.set(init.target_id, nextTarget); + return nextTarget; + } + remove_target(target_id) { + this._targets.delete(target_id); + this._tab_target_ids.delete(target_id); + const sessions = this._target_sessions.get(target_id); + if (sessions) { + for (const session_id of sessions) { + this._sessions.delete(session_id); + this._session_to_target.delete(session_id); + } + } + this._target_sessions.delete(target_id); + for (const [page_id, mapped_target_id] of this._page_targets.entries()) { + if (mapped_target_id === target_id) { + this._page_targets.delete(page_id); + } + } + if (this._focused_target_id === target_id) { + this._focused_target_id = null; + } + } + upsert_session(session_id, target_id) { + const now = new Date().toISOString(); + const existing = this._sessions.get(session_id); + this._sessions.set(session_id, { + session_id, + target_id, + attached_at: existing?.attached_at ?? now, + last_seen_at: now, + }); + const previous_target_id = this._session_to_target.get(session_id); + if (previous_target_id && previous_target_id !== target_id) { + const previousSessions = this._target_sessions.get(previous_target_id); + previousSessions?.delete(session_id); + if (previousSessions && previousSessions.size === 0) { + this._target_sessions.delete(previous_target_id); + } + } + this._session_to_target.set(session_id, target_id); + const sessions = this._target_sessions.get(target_id) ?? new Set(); + sessions.add(session_id); + this._target_sessions.set(target_id, sessions); + } + remove_session(session_id) { + const target_id = this._session_to_target.get(session_id); + this._session_to_target.delete(session_id); + this._sessions.delete(session_id); + if (!target_id) { + return; + } + const sessions = this._target_sessions.get(target_id); + if (!sessions) { + return; + } + sessions.delete(session_id); + if (sessions.size === 0) { + this._target_sessions.delete(target_id); + } + } + bind_page_to_target(page_id, target_id) { + this._page_targets.set(page_id, target_id); + } + unbind_page(page_id) { + this._page_targets.delete(page_id); + } + set_focused_target(target_id) { + this._focused_target_id = target_id; + } + get_focused_target_id() { + return this._focused_target_id; + } + get_target(target_id) { + return this._targets.get(target_id) ?? null; + } + get_session(session_id) { + return this._sessions.get(session_id) ?? null; + } + get_target_id_for_session(session_id) { + return this._session_to_target.get(session_id) ?? null; + } + get_target_id_for_page(page_id) { + return this._page_targets.get(page_id) ?? null; + } + get_sessions_for_target(target_id) { + const session_ids = this._target_sessions.get(target_id); + if (!session_ids) { + return []; + } + return [...session_ids] + .map((session_id) => this._sessions.get(session_id)) + .filter((session) => session != null); + } + get_all_targets() { + return [...this._targets.values()].map((target) => ({ ...target })); + } + get_all_sessions() { + return [...this._sessions.values()].map((session) => ({ ...session })); + } + clear() { + this._targets.clear(); + this._sessions.clear(); + this._target_sessions.clear(); + this._session_to_target.clear(); + this._page_targets.clear(); + this._tab_target_ids.clear(); + this._focused_target_id = null; + } +} diff --git a/dist/browser/session.d.ts b/dist/browser/session.d.ts new file mode 100644 index 00000000..9f99574d --- /dev/null +++ b/dist/browser/session.d.ts @@ -0,0 +1,658 @@ +import { EventBus, type EventDispatchOptions, type EventPayload } from '../event-bus.js'; +import { type Browser, type BrowserContext, type Page, type Locator } from './types.js'; +import { BrowserProfile, type BrowserProfileOptions, DEFAULT_BROWSER_PROFILE } from './profile.js'; +import { BrowserStateSummary, type TabInfo } from './views.js'; +import { type WaitUntilState } from './events.js'; +import { DOMElementNode, type SelectorMap } from '../dom/views.js'; +import { SessionManager } from './session-manager.js'; +import { type CaptchaWaitResult } from './watchdogs/captcha-watchdog.js'; +import type { BaseWatchdog } from './watchdogs/base.js'; +export interface BrowserSessionInit { + id?: string; + browser_profile?: BrowserProfile; + profile?: Partial; + browser?: Browser | null; + browser_context?: BrowserContext | null; + page?: Page | null; + title?: string | null; + url?: string | null; + wss_url?: string | null; + cdp_url?: string | null; + browser_pid?: number | null; + playwright?: unknown; + downloaded_files?: string[]; + closed_popup_messages?: string[]; +} +export interface ChromeProfileInfo { + directory: string; + name: string; + email?: string; +} +export declare const systemChrome: { + findExecutable(): string | null; + getUserDataDir(executablePath?: string | null): string | null; + listProfiles(userDataDir?: string | null): ChromeProfileInfo[]; +}; +export interface BrowserSessionFromSystemChromeInit extends Omit { + browser_profile?: BrowserProfile; + profile?: Partial; + profile_directory?: string | null; +} +export interface BrowserStateOptions { + cache_clickable_elements_hashes?: boolean; + include_screenshot?: boolean; + include_recent_events?: boolean; + signal?: AbortSignal | null; +} +export interface BrowserActionOptions { + signal?: AbortSignal | null; + clear?: boolean; +} +export interface BrowserNavigationOptions extends BrowserActionOptions { + wait_until?: WaitUntilState; + timeout_ms?: number | null; +} +export declare class BrowserSession { + readonly id: string; + readonly browser_profile: BrowserProfile; + readonly event_bus: EventBus; + readonly session_manager: SessionManager; + browser: Browser | null; + browser_context: BrowserContext | null; + agent_current_page: Page | null; + human_current_page: Page | null; + initialized: boolean; + wss_url: string | null; + cdp_url: string | null; + browser_pid: number | null; + playwright: unknown; + private cachedBrowserState; + private _cachedClickableElementHashes; + private currentUrl; + private currentTitle; + private _logger; + private _tabCounter; + private _tabs; + private currentTabIndex; + private historyStack; + downloaded_files: string[]; + llm_screenshot_size: [number, number] | null; + _original_viewport_size: [number, number] | null; + private ownsBrowserResources; + private _autoDownloadPdfs; + private tabPages; + private currentPageLoadingStatus; + private _subprocess; + private _childProcesses; + private attachedAgentId; + private attachedSharedAgentIds; + private _stoppingPromise; + private _closedPopupMessages; + private _dialogHandlersAttached; + private readonly _maxClosedPopupMessages; + private _recentEvents; + private readonly _maxRecentEvents; + private _watchdogs; + private _defaultWatchdogsAttached; + private _captchaWatchdog; + readonly RECONNECT_WAIT_TIMEOUT = 54; + private _reconnecting; + private _reconnectTask; + private _reconnectWaitPromise; + private _resolveReconnectWait; + private _intentionalStop; + private _disconnectAwareBrowser; + private _browserDisconnectHandler; + constructor(init?: BrowserSessionInit); + static from_system_chrome(init?: BrowserSessionFromSystemChromeInit): BrowserSession; + static list_chrome_profiles(): ChromeProfileInfo[]; + attach_watchdog(watchdog: BaseWatchdog): void; + attach_watchdogs(watchdogs: BaseWatchdog[]): void; + detach_watchdog(watchdog: BaseWatchdog): void; + detach_all_watchdogs(): void; + get_watchdogs(): BaseWatchdog[]; + dispatch_browser_event(event: TEvent, options?: Omit): Promise>; + launch(): Promise<{ + cdp_url: string; + }>; + attach_default_watchdogs(): void; + wait_if_captcha_solving(timeoutSeconds?: number): Promise; + private _formatTabId; + private _createTabInfo; + private _buildSyntheticTargetId; + private _syncSessionManagerFromTabs; + get_or_create_cdp_session(page?: Page | null): Promise; + private _waitForStableNetwork; + private _setActivePage; + private _syncCurrentTabFromPage; + private _syncTabsWithBrowserPages; + private _captureClosedPopupMessage; + private _getClosedPopupMessagesSnapshot; + private _recordRecentEvent; + private _getRecentEventsSummary; + private _attachDialogHandler; + private _getPendingNetworkRequests; + get tabs(): TabInfo[]; + get active_tab_index(): number; + get active_tab(): TabInfo; + describe(): string; + get _owns_browser_resources(): boolean; + get is_stopping(): boolean; + get is_reconnecting(): boolean; + get should_gate_watchdog_events(): boolean; + get is_cdp_connected(): boolean; + wait_for_reconnect(timeoutSeconds?: number): Promise; + claim_agent(agentId: string, mode?: 'exclusive' | 'shared'): boolean; + claimAgent(agentId: string, mode?: 'exclusive' | 'shared'): boolean; + release_agent(agentId?: string): boolean; + releaseAgent(agentId?: string): boolean; + get_attached_agent_id(): string | null; + getAttachedAgentId(): string | null; + get_attached_agent_ids(): string[]; + getAttachedAgentIds(): string[]; + private _determineOwnership; + private _createAbortError; + private _isAbortError; + private _throwIfAborted; + private _waitWithAbort; + private _withAbort; + private _toPlaywrightOptions; + set_extra_headers(headers: Record): Promise; + private _applyConfiguredExtraHttpHeaders; + private _usesRemoteBrowserConnection; + private _connectToConfiguredBrowser; + private _ensureBrowserContextFromBrowser; + private _beginReconnectWait; + private _endReconnectWait; + private _detachRemoteDisconnectHandler; + private _attachRemoteDisconnectHandler; + private _handleUnexpectedRemoteDisconnect; + private _restorePagesAfterReconnect; + reconnect(options?: { + preferred_url?: string | null; + preferred_tab_index?: number; + }): Promise; + private _auto_reconnect; + private _isSandboxLaunchError; + private _createNoSandboxLaunchOptions; + private _launchChromiumWithSandboxFallback; + private _connectionDescriptor; + toString(): string; + get logger(): import("../logging-config.js").Logger; + start(): Promise; + /** + * Setup browser session by connecting to an existing browser process via PID + * Useful for debugging or connecting to manually launched browsers + * @param browserPid - Process ID of the browser to connect to + * @param cdpUrl - Optional CDP URL (will be discovered if not provided) + */ + setupBrowserViaBrowserPid(browserPid: number, cdpUrl?: string): Promise; + /** + * Discover CDP URL from browser PID + * Tries common ports and checks for debugging endpoints + */ + private _discoverCdpUrl; + private _shutdown_browser_session; + close(): Promise; + get_browser_state_with_recovery(options?: BrowserStateOptions): Promise; + get_current_page(): Promise; + update_current_page(page: Page | null, title?: string | null, url?: string | null): void; + private _buildTabs; + navigate_to(url: string, options?: BrowserNavigationOptions): Promise; + create_new_tab(url: string, options?: BrowserNavigationOptions): Promise; + private _resolveTabIndex; + switch_to_tab(identifier: number | string, options?: BrowserActionOptions): Promise; + close_tab(identifier: number | string): Promise; + wait(seconds: number, options?: BrowserActionOptions): Promise; + send_keys(keys: string, options?: BrowserActionOptions): Promise; + click_coordinates(coordinate_x: number, coordinate_y: number, options?: BrowserActionOptions & { + button?: 'left' | 'right' | 'middle'; + }): Promise; + scroll(direction: 'up' | 'down' | 'left' | 'right', amount: number, options?: BrowserActionOptions & { + node?: DOMElementNode | null; + }): Promise; + scroll_to_text(text: string, options?: BrowserActionOptions & { + direction?: 'up' | 'down'; + }): Promise; + get_dropdown_options(element_node: DOMElementNode, options?: BrowserActionOptions): Promise<{ + type: string; + options: string; + formatted_options: any; + message: any; + short_term_memory: any; + long_term_memory: string; + }>; + select_dropdown_option(element_node: DOMElementNode, text: string, options?: BrowserActionOptions): Promise<{ + message: string; + short_term_memory: string; + long_term_memory: string; + matched_text: string; + matched_value: string; + } | { + message: string; + short_term_memory: string; + long_term_memory: string; + matched_text: string; + matched_value?: undefined; + }>; + upload_file(element_node: DOMElementNode, file_path: string, options?: BrowserActionOptions): Promise; + go_back(options?: BrowserActionOptions): Promise; + get_dom_element_by_index(_index: number, options?: BrowserActionOptions): Promise; + set_downloaded_files(files: string[]): void; + add_downloaded_file(filePath: string): void; + get_downloaded_files(): string[]; + set_auto_download_pdfs(enabled: boolean): void; + auto_download_pdfs(): boolean; + static get_unique_filename(directory: string, filename: string): Promise; + get_selector_map(options?: BrowserActionOptions): Promise; + static is_file_input(node: DOMElementNode | null): boolean; + is_file_input(node: DOMElementNode | null): boolean; + find_file_upload_element_by_index(index: number, maxHeight?: number, maxDescendantDepth?: number, options?: BrowserActionOptions): Promise; + get_locate_element(node: DOMElementNode): Promise; + _input_text_element_node(node: DOMElementNode, text: string, options?: BrowserActionOptions): Promise; + _click_element_node(node: DOMElementNode, options?: BrowserActionOptions): Promise; + private _waitForLoad; + /** + * Get all cookies from the current browser context + */ + get_cookies(): Promise>>; + /** + * Save cookies to a file (deprecated, use save_storage_state instead) + * @deprecated Use save_storage_state() instead + */ + save_cookies(...args: any[]): Promise; + /** + * Load cookies from a file (deprecated, use load_storage_state instead) + * @deprecated Use load_storage_state() instead + */ + load_cookies_from_file(...args: any[]): Promise; + /** + * Save the current storage state (cookies, localStorage, sessionStorage) to a file + */ + save_storage_state(filePath?: string): Promise; + /** + * Load storage state (cookies, localStorage, sessionStorage) from a file + */ + load_storage_state(filePath?: string): Promise; + /** + * Execute JavaScript in the current page context + */ + execute_javascript(script: string): Promise; + /** + * Get comprehensive page information (size, scroll position, etc.) + */ + get_page_info(page?: Page): Promise; + /** + * Get the HTML content of the current page + */ + get_page_html(): Promise; + /** + * Get a debug view of the page structure including iframes + */ + get_page_structure(): Promise; + /** + * Navigate forward in browser history + */ + go_forward(): Promise; + /** + * Refresh the current page + */ + refresh(): Promise; + /** + * Wait for an element to appear on the page + */ + wait_for_element(selector: string, timeout?: number): Promise; + /** + * Take a screenshot of the current page. + * @param full_page Whether to capture the full scrollable page + * @param clip Optional clip region for partial screenshots + * @returns Base64 encoded PNG screenshot + */ + take_screenshot(full_page?: boolean, clip?: { + x: number; + y: number; + width: number; + height: number; + } | null): Promise; + /** + * Add a request event listener to the current page + */ + on_request(callback: (request: any) => void | Promise): Promise; + /** + * Add a response event listener to the current page + */ + on_response(callback: (response: any) => void | Promise): Promise; + /** + * Remove a request event listener from the current page + */ + off_request(callback: (request: any) => void | Promise): Promise; + /** + * Remove a response event listener from the current page + */ + off_response(callback: (response: any) => void | Promise): Promise; + /** + * Get information about all open tabs + * @returns Array of tab information including page_id, tab_id, url, and title + */ + get_tabs_info(): Promise>; + /** + * Check if a page is responsive by trying to evaluate simple JavaScript + * @param page - The page to check + * @param timeout - Timeout in seconds (default: 5) + * @returns True if page is responsive, false otherwise + */ + _is_page_responsive(page: any, timeout?: number): Promise; + /** + * Get scroll information for the current page + * @returns Object with scroll position and page dimensions + */ + get_scroll_info(): Promise<{ + scroll_x: number; + scroll_y: number; + page_width: number; + page_height: number; + viewport_width: number; + viewport_height: number; + }>; + /** + * Get a summary of the current browser state + * @param cache_clickable_elements_hashes - Cache clickable element hashes to detect new elements + * @param include_screenshot - Include screenshot in state summary + * @returns BrowserStateSummary with current page state + */ + get_state_summary(cache_clickable_elements_hashes?: boolean, include_screenshot?: boolean, include_recent_events?: boolean): Promise; + /** + * Get minimal state summary without DOM processing, but with screenshot + * Used when page is in error state or unresponsive + */ + get_minimal_state_summary(include_recent_events?: boolean): Promise; + /** + * Internal method to get updated browser state with DOM processing + * @param focus_element - Element index to focus on (default: -1) + * @param include_screenshot - Whether to include screenshot + */ + private _get_updated_state; + /** + * Check if a URL is a new tab page + */ + private _is_new_tab_page; + private _is_ip_address_host; + private _get_domain_variants; + private _setEntryMatchesUrl; + /** + * Check if page is displaying a PDF + */ + private _is_pdf_viewer; + /** + * Auto-download PDF if detected and auto-download is enabled + */ + private _auto_download_pdf_if_needed; + /** + * Check if an element is visible on the page + */ + private _is_visible; + /** + * Locate an element by XPath + */ + get_locate_element_by_xpath(xpath: string): Promise; + /** + * Locate an element by CSS selector + */ + get_locate_element_by_css_selector(css_selector: string): Promise; + /** + * Locate an element by text content + * @param text - Text to search for + * @param nth - Which matching element to return (0-based index) + * @param element_type - Optional tag name to filter by (e.g., 'button', 'span') + */ + get_locate_element_by_text(text: string, nth?: number, element_type?: string | null): Promise; + /** + * Check if browser session is connected and has valid browser/context objects + * @param restart - If true, attempt to create a new tab if no pages exist + */ + is_connected(restart?: boolean): Promise; + /** + * Check if a URL is allowed based on allowed_domains configuration + * @param url - URL to check + */ + private _get_url_access_denial_reason; + private _is_url_allowed; + private _formatDomainCollection; + private _assert_url_allowed; + /** + * Navigate helper with URL validation + */ + navigate(url: string): Promise; + /** + * Kill the browser session (force close even if keep_alive=true) + */ + kill(): Promise; + /** + * Alias for close() to match Python API + */ + stop(): Promise; + /** + * Perform a click action with download and navigation handling + * @param element_node - DOM element to click + */ + perform_click(element_node: DOMElementNode): Promise; + /** + * Remove all highlights from the current page + */ + remove_highlights(): Promise; + /** + * Start tracing on browser context if traces_dir is configured + * Note: Currently optional as it may cause performance issues in some cases + */ + start_trace_recording(): Promise; + /** + * Save browser trace recording if active + */ + save_trace_recording(): Promise; + /** + * Start tracing on browser context if traces_dir is configured + * Note: Currently optional as it may cause performance issues in some cases + */ + private _startContextTracing; + /** + * Save browser trace recording + */ + private _saveTraceRecording; + /** + * Scroll using CDP Input.synthesizeScrollGesture for universal compatibility + * @param page - The page to scroll + * @param pixels - Number of pixels to scroll (positive = up, negative = down) + * @returns true if successful, false if failed + */ + private _scrollWithCdpGesture; + /** + * Scroll the current page container + * @param pixels - Number of pixels to scroll (positive = up, negative = down) + */ + private _scrollContainer; + /** + * Compute hashes for all clickable elements in the selector map + * @param selectorMap - Selector map from DOM state + * @returns Set of element hashes + */ + private _computeElementHashes; + /** + * Mark elements in the selector map as new if they weren't in the cached hashes + * @param selectorMap - Selector map to update + * @param cachedHashes - Previously cached element hashes + */ + private _markNewElements; + /** + * Helper to get a safe method name from the calling context + * Used for recovery error messages + */ + private _getCurrentMethodName; + /** + * Get current page with fallback logic + * Alias for compatibility with Python API + */ + getCurrentPage(): Promise; + /** + * Log warning about unsafe glob patterns + * @param pattern - The glob pattern being used + */ + private _logGlobWarning; + /** + * Create a shallow copy of the browser session + * Note: This doesn't copy the actual browser instance, just the session metadata + * @returns A new BrowserSession instance with copied state + */ + modelCopy(): BrowserSession; + model_copy(): BrowserSession; + private _inRecovery; + /** + * Check if a page is responsive by trying to evaluate simple JavaScript + * @param page - The page to check + * @param timeout - Timeout in seconds (default: 5.0) + * @returns true if page is responsive, false otherwise + */ + private _isPageResponsive; + /** + * Force close a crashed page using CDP from a clean temporary page + * @param pageUrl - The URL of the page to force close + * @returns true if successful, false otherwise + */ + private _forceClosePageViaCdp; + /** + * Try to reopen a URL in a new page and check if it's responsive + * @param url - The URL to reopen + * @param timeoutMs - Navigation timeout in milliseconds + * @returns true if successful and responsive, false otherwise + */ + private _tryReopenUrl; + /** + * Create a new blank page as a fallback when recovery fails + * @param url - The original URL that failed + */ + private _createBlankFallbackPage; + /** + * Recover from an unresponsive page by closing and reopening it + * @param callingMethod - The name of the method that detected the unresponsive page + * @param timeoutMs - Navigation timeout in milliseconds + */ + private _recoverUnresponsivePage; + /** + * Generate enhanced CSS selector for an element + * Handles special characters and provides fallback strategies + * @param xpath - XPath of the element + * @param element - Optional element node for additional context + * @returns Enhanced CSS selector string + */ + private _enhancedCssSelectorForElement; + /** + * Convert XPath to CSS selector + * Handles simple XPath expressions + */ + private _xpathToCss; + /** + * Escape special characters in CSS selectors + * Handles characters that need escaping in CSS + */ + private _escapeSelector; + /** + * Prepare user data directory for browser profile + * Handles singleton lock conflicts and creates temp profiles if needed + */ + prepareUserDataDir(userDataDir?: string): Promise; + /** + * Check if user data directory has a singleton lock + * This happens when another Chrome instance is using the profile + */ + private _checkForSingletonLockConflict; + /** + * Fallback to a temporary profile when the primary one is locked + */ + private _fallbackToTempProfile; + /** + * Create a temporary user data directory + */ + private _createTempUserDataDir; + /** + * Setup listeners for page visibility changes + * Tracks when user switches tabs to update human_current_page + */ + private _setupCurrentPageChangeListeners; + /** + * Callback when tab visibility changes + * Updates human_current_page to reflect which tab the user is viewing + */ + private _onTabVisibilityChange; + /** + * Normalize pid values before issuing process operations. + */ + private _normalizePid; + /** + * Kill all child processes spawned by this browser session + */ + private _killChildProcesses; + /** + * Terminate the browser process and all its children + */ + private _terminateBrowserProcess; + /** + * Get child processes of a given PID + * Cross-platform implementation using ps on Unix-like systems and WMIC on Windows + */ + private _getChildProcesses; + /** + * Track a child process + */ + private _trackChildProcess; + /** + * Untrack a child process + */ + private _untrackChildProcess; + /** + * Show DVD screensaver loading animation + * Returns a function to stop the animation + * + * @param message - Message to display (default: 'Loading...') + * @param fps - Frames per second (default: 10) + * @returns Function to stop the animation + * + * @example + * const stopAnimation = this._showDvdScreensaverLoadingAnimation('Loading page...'); + * await someLongOperation(); + * stopAnimation(); + */ + _showDvdScreensaverLoadingAnimation(message?: string, fps?: number): () => void; + /** + * Show simple spinner loading animation + * Returns a function to stop the animation + * + * @param message - Message to display (default: 'Loading...') + * @param fps - Frames per second (default: 10) + * @returns Function to stop the animation + * + * @example + * const stopSpinner = this._showSpinnerLoadingAnimation('Processing...'); + * await someLongOperation(); + * stopSpinner(); + */ + _showSpinnerLoadingAnimation(message?: string, fps?: number): () => void; + /** + * Execute an async operation with DVD screensaver animation + * + * @param operation - Async operation to execute + * @param message - Message to display during operation + * @returns Result of the operation + * + * @example + * const page = await this._withDvdScreensaver( + * async () => await this.browser_context!.newPage(), + * 'Opening new page...' + * ); + */ + _withDvdScreensaver(operation: () => Promise, message?: string): Promise; +} +export { DEFAULT_BROWSER_PROFILE }; diff --git a/dist/browser/session.js b/dist/browser/session.js new file mode 100644 index 00000000..d85968fb --- /dev/null +++ b/dist/browser/session.js @@ -0,0 +1,5292 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { isIP } from 'node:net'; +import { execFile, execFileSync, } from 'node:child_process'; +import { promisify } from 'node:util'; +import { createLogger } from '../logging-config.js'; +import { match_url_with_domain_pattern, uuid7str } from '../utils.js'; +import { EventBus, } from '../event-bus.js'; +import { async_playwright, } from './types.js'; +import { BrowserProfile, CHROME_DOCKER_ARGS, DEFAULT_BROWSER_PROFILE, } from './profile.js'; +import { BrowserStateSummary, BrowserError, URLNotAllowedError, } from './views.js'; +import { AgentFocusChangedEvent, BrowserConnectedEvent, BrowserErrorEvent, BrowserLaunchEvent, BrowserReconnectedEvent, BrowserReconnectingEvent, BrowserStartEvent, BrowserStoppedEvent, BrowserStopEvent, DialogOpenedEvent, DownloadProgressEvent, DownloadStartedEvent, FileDownloadedEvent, TabClosedEvent, TabCreatedEvent, } from './events.js'; +import { DOMElementNode, DOMState } from '../dom/views.js'; +import { normalize_url } from './utils.js'; +import { DomService } from '../dom/service.js'; +import { showDVDScreensaver, showSpinner, withDVDScreensaver, } from './dvd-screensaver.js'; +import { SessionManager } from './session-manager.js'; +import { AboutBlankWatchdog } from './watchdogs/aboutblank-watchdog.js'; +import { CaptchaWatchdog, } from './watchdogs/captcha-watchdog.js'; +import { CDPSessionWatchdog } from './watchdogs/cdp-session-watchdog.js'; +import { CrashWatchdog } from './watchdogs/crash-watchdog.js'; +import { DefaultActionWatchdog } from './watchdogs/default-action-watchdog.js'; +import { DOMWatchdog } from './watchdogs/dom-watchdog.js'; +import { DownloadsWatchdog } from './watchdogs/downloads-watchdog.js'; +import { HarRecordingWatchdog } from './watchdogs/har-recording-watchdog.js'; +import { LocalBrowserWatchdog } from './watchdogs/local-browser-watchdog.js'; +import { PermissionsWatchdog } from './watchdogs/permissions-watchdog.js'; +import { PopupsWatchdog } from './watchdogs/popups-watchdog.js'; +import { RecordingWatchdog } from './watchdogs/recording-watchdog.js'; +import { ScreenshotWatchdog } from './watchdogs/screenshot-watchdog.js'; +import { SecurityWatchdog } from './watchdogs/security-watchdog.js'; +import { StorageStateWatchdog } from './watchdogs/storage-state-watchdog.js'; +const execFileAsync = promisify(execFile); +const PLAYWRIGHT_OPTION_KEY_OVERRIDES = { + extra_http_headers: 'extraHTTPHeaders', +}; +const EMPTY_DOM_RETRY_DELAY_MS = 250; +const REMOTE_RECONNECT_DELAYS_MS = [1000, 2000, 4000]; +const REMOTE_RECONNECT_ATTEMPT_TIMEOUT_MS = 15_000; +const cloneBrowserProfileConfig = (profile) => typeof structuredClone === 'function' + ? structuredClone(profile.config) + : JSON.parse(JSON.stringify(profile.config)); +const detectSystemChromeVariant = (executablePath) => { + const normalizedPath = String(executablePath ?? '') + .trim() + .toLowerCase(); + if (!normalizedPath) { + return 'chrome'; + } + if (normalizedPath.includes('chromium')) { + return 'chromium'; + } + if (normalizedPath.includes('chrome canary') || + normalizedPath.includes('chrome sxs')) { + return 'chrome-canary'; + } + if (normalizedPath.includes('google-chrome-beta')) { + return 'chrome-beta'; + } + if (normalizedPath.includes('google-chrome-unstable')) { + return 'chrome-unstable'; + } + return 'chrome'; +}; +export const systemChrome = { + findExecutable() { + if (process.platform === 'darwin') { + const candidates = [ + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + '/Applications/Chromium.app/Contents/MacOS/Chromium', + '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary', + ]; + return candidates.find((candidate) => fs.existsSync(candidate)) ?? null; + } + if (process.platform === 'linux') { + const commands = [ + 'google-chrome', + 'google-chrome-stable', + 'google-chrome-beta', + 'google-chrome-unstable', + 'chromium', + 'chromium-browser', + ]; + for (const command of commands) { + try { + const resolved = execFileSync('which', [command], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + if (resolved) { + return resolved; + } + } + catch { + // Ignore missing commands and try the next candidate. + } + } + return null; + } + if (process.platform === 'win32') { + const candidates = [ + path.join(process.env.ProgramFiles ?? 'C:\\Program Files', 'Google', 'Chrome', 'Application', 'chrome.exe'), + path.join(process.env['ProgramFiles(x86)'] ?? 'C:\\Program Files (x86)', 'Google', 'Chrome', 'Application', 'chrome.exe'), + path.join(process.env.LOCALAPPDATA ?? '', 'Google', 'Chrome', 'Application', 'chrome.exe'), + path.join(process.env.LOCALAPPDATA ?? '', 'Google', 'Chrome SxS', 'Application', 'chrome.exe'), + path.join(process.env.LOCALAPPDATA ?? '', 'Chromium', 'Application', 'chrome.exe'), + path.join(process.env.ProgramFiles ?? 'C:\\Program Files', 'Chromium', 'Application', 'chrome.exe'), + path.join(process.env['ProgramFiles(x86)'] ?? 'C:\\Program Files (x86)', 'Chromium', 'Application', 'chrome.exe'), + ]; + return candidates.find((candidate) => fs.existsSync(candidate)) ?? null; + } + return null; + }, + getUserDataDir(executablePath = systemChrome.findExecutable()) { + const variant = detectSystemChromeVariant(executablePath); + if (process.platform === 'darwin') { + const applicationSupportDir = path.join(os.homedir(), 'Library', 'Application Support'); + if (variant === 'chromium') { + return path.join(applicationSupportDir, 'Chromium'); + } + if (variant === 'chrome-canary') { + return path.join(applicationSupportDir, 'Google', 'Chrome Canary'); + } + return path.join(applicationSupportDir, 'Google', 'Chrome'); + } + if (process.platform === 'linux') { + if (variant === 'chromium') { + return path.join(os.homedir(), '.config', 'chromium'); + } + if (variant === 'chrome-beta') { + return path.join(os.homedir(), '.config', 'google-chrome-beta'); + } + if (variant === 'chrome-unstable') { + return path.join(os.homedir(), '.config', 'google-chrome-unstable'); + } + return path.join(os.homedir(), '.config', 'google-chrome'); + } + if (process.platform === 'win32') { + const localAppData = process.env.LOCALAPPDATA ?? path.join(os.homedir(), 'AppData', 'Local'); + if (variant === 'chromium') { + return path.join(localAppData, 'Chromium', 'User Data'); + } + if (variant === 'chrome-canary') { + return path.join(localAppData, 'Google', 'Chrome SxS', 'User Data'); + } + return path.join(localAppData, 'Google', 'Chrome', 'User Data'); + } + return null; + }, + listProfiles(userDataDir = systemChrome.getUserDataDir()) { + if (!userDataDir) { + return []; + } + const localStatePath = path.join(userDataDir, 'Local State'); + if (!fs.existsSync(localStatePath)) { + return []; + } + try { + const raw = fs.readFileSync(localStatePath, 'utf8'); + const localState = JSON.parse(raw); + const infoCache = localState.profile?.info_cache ?? {}; + return Object.entries(infoCache) + .map(([directory, info]) => ({ + directory, + name: info?.name || directory, + email: info?.user_name || '', + })) + .sort((left, right) => left.directory.localeCompare(right.directory)); + } + catch { + return []; + } + }, +}; +const createEmptyDomState = () => { + const root = new DOMElementNode(true, null, 'html', '/html[1]', {}, []); + return new DOMState(root, {}); +}; +export class BrowserSession { + id; + browser_profile; + event_bus; + session_manager; + browser; + browser_context; + agent_current_page; + human_current_page; + initialized = false; + wss_url; + cdp_url; + browser_pid; + playwright; + cachedBrowserState = null; + _cachedClickableElementHashes = null; + currentUrl; + currentTitle; + _logger = null; + _tabCounter = 0; + _tabs = []; + currentTabIndex = 0; + historyStack = []; + downloaded_files = []; + llm_screenshot_size = null; + _original_viewport_size = null; + ownsBrowserResources = true; + _autoDownloadPdfs = true; + tabPages = new Map(); + currentPageLoadingStatus = null; + _subprocess = null; + _childProcesses = new Set(); + attachedAgentId = null; + attachedSharedAgentIds = new Set(); + _stoppingPromise = null; + _closedPopupMessages = []; + _dialogHandlersAttached = new WeakSet(); + _maxClosedPopupMessages = 20; + _recentEvents = []; + _maxRecentEvents = 100; + _watchdogs = new Set(); + _defaultWatchdogsAttached = false; + _captchaWatchdog = null; + RECONNECT_WAIT_TIMEOUT = 54; + _reconnecting = false; + _reconnectTask = null; + _reconnectWaitPromise = Promise.resolve(); + _resolveReconnectWait = null; + _intentionalStop = false; + _disconnectAwareBrowser = null; + _browserDisconnectHandler = null; + constructor(init = {}) { + const sourceProfileConfig = init.browser_profile + ? cloneBrowserProfileConfig(init.browser_profile) + : (init.profile ?? {}); + this.browser_profile = new BrowserProfile(sourceProfileConfig); + this.id = init.id ?? uuid7str(); + this.event_bus = new EventBus(`BrowserSession_${this.id.slice(-4)}`); + this.session_manager = new SessionManager(); + this.browser = init.browser ?? null; + this.browser_context = init.browser_context ?? null; + this.agent_current_page = init.page ?? null; + this.human_current_page = init.page ?? null; + this.currentUrl = normalize_url(init.url ?? 'about:blank'); + this.currentTitle = init.title ?? ''; + this.wss_url = init.wss_url ?? null; + this.cdp_url = init.cdp_url ?? null; + this.browser_pid = init.browser_pid ?? null; + this.playwright = init.playwright ?? null; + this.downloaded_files = Array.isArray(init.downloaded_files) + ? [...init.downloaded_files] + : []; + this._closedPopupMessages = Array.isArray(init.closed_popup_messages) + ? [...init.closed_popup_messages] + : []; + if (typeof init?.auto_download_pdfs === 'boolean') { + this._autoDownloadPdfs = Boolean(init.auto_download_pdfs); + } + const initialPageId = this._tabCounter++; + this._tabs = [ + this._createTabInfo({ + page_id: initialPageId, + url: this.currentUrl, + title: this.currentTitle || this.currentUrl, + }), + ]; + this.historyStack.push(this.currentUrl); + this.ownsBrowserResources = this._determineOwnership(); + this.tabPages.set(this._tabs[0].page_id, this.agent_current_page ?? null); + this._syncSessionManagerFromTabs(); + this._attachDialogHandler(this.agent_current_page); + this._recordRecentEvent('session_initialized', { url: this.currentUrl }); + } + static from_system_chrome(init = {}) { + const executablePath = systemChrome.findExecutable(); + if (!executablePath) { + throw new Error('Chrome not found. Please install Chrome or use BrowserSession with an explicit executable_path.\n' + + 'Expected locations:\n' + + ' macOS: /Applications/Google Chrome.app/Contents/MacOS/Google Chrome\n' + + ' Linux: /usr/bin/google-chrome or /usr/bin/chromium\n' + + ' Windows: C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe'); + } + const userDataDir = systemChrome.getUserDataDir(executablePath); + if (!userDataDir) { + throw new Error('Could not detect Chrome profile directory for your platform.\n' + + 'Expected locations:\n' + + ' macOS: ~/Library/Application Support/Google/Chrome\n' + + ' Linux: ~/.config/google-chrome\n' + + ' Windows: %LocalAppData%\\Google\\Chrome\\User Data'); + } + const availableProfiles = systemChrome.listProfiles(userDataDir); + const selectedProfileDirectory = init.profile_directory ?? availableProfiles[0]?.directory ?? 'Default'; + if (typeof init.profile_directory === 'undefined' && availableProfiles[0]) { + createLogger('browser_use.browser.session').info(`Auto-selected Chrome profile: ${availableProfiles[0].name} (${availableProfiles[0].directory})`); + } + const sourceProfileConfig = init.browser_profile + ? cloneBrowserProfileConfig(init.browser_profile) + : (init.profile ?? {}); + const { browser_profile: _browserProfile, profile: _profile, profile_directory: _profileDirectory, ...sessionInit } = init; + return new BrowserSession({ + ...sessionInit, + browser_profile: new BrowserProfile({ + ...sourceProfileConfig, + executable_path: executablePath, + user_data_dir: userDataDir, + profile_directory: selectedProfileDirectory, + }), + }); + } + static list_chrome_profiles() { + const executablePath = systemChrome.findExecutable(); + const userDataDir = systemChrome.getUserDataDir(executablePath); + return systemChrome.listProfiles(userDataDir); + } + attach_watchdog(watchdog) { + if (this._watchdogs.has(watchdog)) { + return; + } + watchdog.attach_to_session(); + this._watchdogs.add(watchdog); + if (watchdog instanceof CaptchaWatchdog) { + this._captchaWatchdog = watchdog; + } + } + attach_watchdogs(watchdogs) { + for (const watchdog of watchdogs) { + this.attach_watchdog(watchdog); + } + } + detach_watchdog(watchdog) { + if (!this._watchdogs.has(watchdog)) { + return; + } + watchdog.detach_from_session(); + this._watchdogs.delete(watchdog); + if (watchdog === this._captchaWatchdog) { + this._captchaWatchdog = null; + } + } + detach_all_watchdogs() { + for (const watchdog of [...this._watchdogs]) { + this.detach_watchdog(watchdog); + } + this._defaultWatchdogsAttached = false; + this._captchaWatchdog = null; + } + get_watchdogs() { + return [...this._watchdogs]; + } + async dispatch_browser_event(event, options = {}) { + return this.event_bus.dispatch_or_throw(event, options); + } + async launch() { + this.attach_default_watchdogs(); + const dispatchResult = await this.dispatch_browser_event(new BrowserLaunchEvent()); + const eventResult = dispatchResult.event.event_result; + return { + cdp_url: eventResult?.cdp_url ?? this.cdp_url ?? this.wss_url ?? 'playwright', + }; + } + attach_default_watchdogs() { + if (this._defaultWatchdogsAttached) { + return; + } + const watchdogs = [ + new LocalBrowserWatchdog({ browser_session: this }), + new CDPSessionWatchdog({ browser_session: this }), + new CrashWatchdog({ browser_session: this }), + new AboutBlankWatchdog({ browser_session: this }), + new PermissionsWatchdog({ browser_session: this }), + new PopupsWatchdog({ browser_session: this }), + new SecurityWatchdog({ browser_session: this }), + new DOMWatchdog({ browser_session: this }), + new ScreenshotWatchdog({ browser_session: this }), + new RecordingWatchdog({ browser_session: this }), + new DownloadsWatchdog({ browser_session: this }), + new StorageStateWatchdog({ browser_session: this }), + new DefaultActionWatchdog({ browser_session: this }), + ]; + if (this.browser_profile.config.captcha_solver) { + this._captchaWatchdog = new CaptchaWatchdog({ browser_session: this }); + watchdogs.push(this._captchaWatchdog); + } + const configuredHarPath = this.browser_profile.config.record_har_path; + if (typeof configuredHarPath === 'string' && + configuredHarPath.trim().length > 0) { + watchdogs.push(new HarRecordingWatchdog({ browser_session: this })); + } + this.attach_watchdogs(watchdogs); + this._defaultWatchdogsAttached = true; + } + async wait_if_captcha_solving(timeoutSeconds) { + return (this._captchaWatchdog?.wait_if_captcha_solving(timeoutSeconds) ?? null); + } + _formatTabId(pageId) { + const normalized = Number.isFinite(pageId) && pageId >= 0 ? Math.floor(pageId) : 0; + return String(normalized).padStart(4, '0').slice(-4); + } + _createTabInfo({ page_id, url, title, parent_page_id = null, }) { + return { + page_id, + tab_id: this._formatTabId(page_id), + target_id: this._buildSyntheticTargetId(page_id), + url, + title, + parent_page_id, + }; + } + _buildSyntheticTargetId(pageId) { + return `tab_${this.id.slice(-4)}_${this._formatTabId(pageId)}`; + } + _syncSessionManagerFromTabs() { + this.session_manager.sync_tabs(this._tabs, this.currentTabIndex, (page_id) => this._buildSyntheticTargetId(page_id)); + } + async get_or_create_cdp_session(page = null) { + if (!this.browser_context?.newCDPSession) { + throw new Error('CDP sessions are not available for this browser context'); + } + const targetPage = page ?? (await this.get_current_page()); + if (!targetPage) { + throw new Error('No active page available to create CDP session'); + } + return this.browser_context.newCDPSession(targetPage); + } + async _waitForStableNetwork(page, signal = null) { + const pendingRequests = new Set(); + let lastActivity = Date.now() / 1000; + // Relevant resource types that indicate page loading progress + const relevantResourceTypes = new Set([ + 'document', + 'stylesheet', + 'image', + 'font', + 'script', + 'iframe', + ]); + const ignoredResourceTypes = new Set([ + 'websocket', + 'media', + 'eventsource', + 'manifest', + 'other', + ]); + // Expanded URL pattern filters - more comprehensive blocking + const ignoredUrlPatterns = [ + 'analytics', + 'tracking', + 'telemetry', + 'beacon', + 'metrics', + 'doubleclick', + 'adsystem', + 'adserver', + 'advertising', + 'facebook.com/plugins', + 'platform.twitter', + 'linkedin.com/embed', + 'livechat', + 'zendesk', + 'intercom', + 'crisp.chat', + 'hotjar', + 'push-notifications', + 'onesignal', + 'pushwoosh', + 'heartbeat', + 'ping', + 'alive', + 'webrtc', + 'rtmp://', + 'wss://', + 'cloudfront.net/assets', + 'fastly.net', + ]; + // Content types that should be filtered + const relevantContentTypes = new Set([ + 'text/html', + 'text/css', + 'application/javascript', + 'application/x-javascript', + 'text/javascript', + 'image/png', + 'image/jpeg', + 'image/gif', + 'image/webp', + 'image/svg+xml', + 'font/woff', + 'font/woff2', + 'application/font-woff', + 'application/font-woff2', + ]); + // Streaming media content types to ignore + const streamingContentTypes = new Set([ + 'video/', + 'audio/', + 'application/octet-stream', + 'application/x-mpegurl', + 'application/vnd.apple.mpegurl', + ]); + // Max response size to track (5MB) + const maxResponseSize = 5 * 1024 * 1024; + const onRequest = (request) => { + const resourceType = request.resourceType?.() ?? request.resourceType; + if (!resourceType || !relevantResourceTypes.has(resourceType)) { + return; + } + if (ignoredResourceTypes.has(resourceType)) { + return; + } + const url = request.url?.().toLowerCase?.() ?? request.url?.toLowerCase?.() ?? ''; + // Filter data URLs and blob URLs + if (url.startsWith('data:') || url.startsWith('blob:')) { + return; + } + // Filter by URL patterns + if (ignoredUrlPatterns.some((pattern) => url.includes(pattern.toLowerCase()))) { + return; + } + // Filter prefetch requests + const headers = request.headers?.() ?? request.headers ?? {}; + const purpose = headers['purpose'] || headers['sec-fetch-dest']; + if (purpose === 'prefetch' || headers['x-moz'] === 'prefetch') { + return; + } + pendingRequests.add(request); + lastActivity = Date.now() / 1000; + }; + const onResponse = async (response) => { + const request = response.request?.() ?? response.request; + if (!pendingRequests.has(request)) { + return; + } + try { + // Check Content-Type header + const headers = response.headers?.() ?? response.headers ?? {}; + const contentType = headers['content-type'] || headers['Content-Type'] || ''; + // Filter streaming media + if (streamingContentTypes.has(contentType.split(';')[0].trim())) { + pendingRequests.delete(request); + return; + } + // Check if content type is relevant + const baseContentType = contentType.split(';')[0].trim(); + const isRelevant = Array.from(relevantContentTypes).some((ct) => baseContentType.startsWith(ct) || ct.startsWith(baseContentType)); + if (contentType && !isRelevant) { + // Unknown content type, still track but log it + this.logger.debug(`Tracking unknown content type: ${baseContentType}`); + } + // Check response size (if available) + const contentLength = headers['content-length'] || headers['Content-Length']; + if (contentLength && parseInt(contentLength, 10) > maxResponseSize) { + this.logger.debug(`Skipping large response (${contentLength} bytes): ${request.url?.().substring?.(0, 50) ?? ''}`); + pendingRequests.delete(request); + return; + } + } + catch (error) { + // If header inspection fails, still process the response + this.logger.debug(`Error inspecting response headers: ${error.message}`); + } + pendingRequests.delete(request); + lastActivity = Date.now() / 1000; + }; + const waitForIdle = async () => { + const startTime = Date.now() / 1000; + while (true) { + this._throwIfAborted(signal); + await this._waitWithAbort(100, signal); + this._throwIfAborted(signal); + const now = Date.now() / 1000; + if (pendingRequests.size === 0 && + now - lastActivity >= + (this.browser_profile.wait_for_network_idle_page_load_time ?? 0.5)) { + this.currentPageLoadingStatus = null; + break; + } + if (now - startTime > + (this.browser_profile.maximum_wait_page_load_time ?? 5)) { + this.currentPageLoadingStatus = `Page loading was aborted after ${this.browser_profile.maximum_wait_page_load_time ?? 5}s with ${pendingRequests.size} pending network requests. You may want to use the wait action to allow more time for the page to fully load.`; + break; + } + } + }; + if (typeof page?.on === 'function' && typeof page?.off === 'function') { + page.on('request', onRequest); + page.on('response', onResponse); + try { + await waitForIdle(); + } + finally { + page.off('request', onRequest); + page.off('response', onResponse); + } + } + else { + this.currentPageLoadingStatus = null; + } + } + _setActivePage(page) { + const currentTab = this._tabs[this.currentTabIndex]; + if (currentTab) { + this.tabPages.set(currentTab.page_id, page ?? null); + this.session_manager.set_focused_target(currentTab.target_id ?? null); + } + this._attachDialogHandler(page); + this.agent_current_page = page ?? null; + } + async _syncCurrentTabFromPage(page) { + if (!page) { + return; + } + let resolvedUrl = null; + try { + const rawUrl = page.url(); + if (typeof rawUrl === 'string' && rawUrl.trim()) { + resolvedUrl = normalize_url(rawUrl); + this.currentUrl = resolvedUrl; + } + } + catch { + // Ignore transient URL read failures. + } + let resolvedTitle = null; + if (typeof page.title === 'function') { + try { + const title = await page.title(); + if (typeof title === 'string' && title.trim()) { + resolvedTitle = title; + } + } + catch { + // Ignore transient title read failures. + } + } + if (!resolvedTitle) { + resolvedTitle = resolvedUrl ?? this.currentTitle ?? this.currentUrl; + } + this.currentTitle = resolvedTitle; + const currentTab = this._tabs[this.currentTabIndex]; + if (currentTab) { + if (resolvedUrl) { + currentTab.url = resolvedUrl; + } + currentTab.title = resolvedTitle; + this._syncSessionManagerFromTabs(); + } + } + _syncTabsWithBrowserPages() { + const pages = this.browser_context?.pages?.() ?? []; + if (!pages.length) { + return; + } + const nextTabs = []; + const nextTabPages = new Map(); + const usedPageIds = new Set(); + const knownPageMappings = Array.from(this.tabPages.entries()); + for (const page of pages) { + this._attachDialogHandler(page ?? null); + let pageId = null; + for (const [candidateId, candidatePage] of knownPageMappings) { + if (candidatePage === page && !usedPageIds.has(candidateId)) { + pageId = candidateId; + break; + } + } + if (pageId === null) { + pageId = this._tabCounter++; + } + usedPageIds.add(pageId); + const existingTab = this._tabs.find((tab) => tab.page_id === pageId); + const tab = existingTab + ? { ...existingTab } + : this._createTabInfo({ + page_id: pageId, + url: 'about:blank', + title: 'about:blank', + }); + try { + const rawUrl = page.url(); + if (typeof rawUrl === 'string' && rawUrl.trim()) { + tab.url = normalize_url(rawUrl); + } + } + catch { + // Keep existing tab url when page url is not readable. + } + if (!existingTab || !tab.title || tab.title === 'about:blank') { + tab.title = tab.url; + } + nextTabs.push(tab); + nextTabPages.set(pageId, page); + } + if (!nextTabs.length) { + return; + } + this._tabs = nextTabs; + this.tabPages = nextTabPages; + const activePage = this.agent_current_page && pages.includes(this.agent_current_page) + ? this.agent_current_page + : (pages[0] ?? null); + if (activePage) { + const activeIndex = this._tabs.findIndex((tab) => this.tabPages.get(tab.page_id) === activePage); + if (activeIndex !== -1) { + this.currentTabIndex = activeIndex; + } + } + if (this.currentTabIndex < 0 || this.currentTabIndex >= this._tabs.length) { + this.currentTabIndex = Math.max(0, this._tabs.length - 1); + } + const activeTab = this._tabs[this.currentTabIndex] ?? null; + if (activeTab) { + this.currentUrl = activeTab.url; + this.currentTitle = activeTab.title || activeTab.url; + this.agent_current_page = this.tabPages.get(activeTab.page_id) ?? null; + this.human_current_page = + this.human_current_page ?? this.agent_current_page; + } + this._syncSessionManagerFromTabs(); + } + _captureClosedPopupMessage(dialogType, message) { + const normalizedType = String(dialogType || 'alert').trim() || 'alert'; + const normalizedMessage = String(message || '').trim(); + if (!normalizedMessage) { + return; + } + const formatted = `[${normalizedType}] ${normalizedMessage}`; + this._closedPopupMessages.push(formatted); + if (this._closedPopupMessages.length > this._maxClosedPopupMessages) { + this._closedPopupMessages.splice(0, this._closedPopupMessages.length - this._maxClosedPopupMessages); + } + } + _getClosedPopupMessagesSnapshot() { + return [...this._closedPopupMessages]; + } + _recordRecentEvent(event_type, details = {}) { + const event = { + event_type: String(event_type || 'unknown').trim() || 'unknown', + timestamp: new Date().toISOString(), + }; + if (typeof details.url === 'string' && details.url.trim()) { + event.url = details.url.trim(); + } + if (typeof details.error_message === 'string' && + details.error_message.trim()) { + event.error_message = details.error_message.trim(); + } + if (typeof details.page_id === 'number' && + Number.isFinite(details.page_id)) { + event.page_id = details.page_id; + } + if (typeof details.tab_id === 'string' && details.tab_id.trim()) { + event.tab_id = details.tab_id.trim(); + } + this._recentEvents.push(event); + if (this._recentEvents.length > this._maxRecentEvents) { + this._recentEvents.splice(0, this._recentEvents.length - this._maxRecentEvents); + } + } + _getRecentEventsSummary(limit = 10) { + if (!this._recentEvents.length || limit <= 0) { + return null; + } + const events = this._recentEvents.slice(-limit); + return JSON.stringify(events); + } + _attachDialogHandler(page) { + if (!page || this._dialogHandlersAttached.has(page)) { + return; + } + const pageWithEvents = page; + if (typeof pageWithEvents.on !== 'function') { + return; + } + const handler = async (dialog) => { + try { + const dialogType = typeof dialog?.type === 'function' ? dialog.type() : 'alert'; + const message = typeof dialog?.message === 'function' ? dialog.message() : ''; + try { + await this.event_bus.dispatch(new DialogOpenedEvent({ + dialog_type: dialogType, + message: String(message ?? ''), + url: this.currentUrl ?? 'about:blank', + frame_id: null, + })); + } + catch (error) { + this.logger.debug(`Failed to dispatch DialogOpenedEvent: ${error.message}`); + } + this._captureClosedPopupMessage(dialogType, message); + this._recordRecentEvent('javascript_dialog_closed', { + url: this.currentUrl, + error_message: message + ? `[${dialogType}] ${String(message).trim()}` + : `[${dialogType}]`, + }); + const shouldAccept = dialogType === 'alert' || + dialogType === 'confirm' || + dialogType === 'beforeunload'; + if (shouldAccept && typeof dialog?.accept === 'function') { + await dialog.accept(); + } + else if (typeof dialog?.dismiss === 'function') { + await dialog.dismiss(); + } + } + catch (error) { + this.logger.debug(`Failed to auto-handle JavaScript dialog: ${error.message}`); + } + }; + pageWithEvents.on('dialog', handler); + this._dialogHandlersAttached.add(page); + } + async _getPendingNetworkRequests(page) { + if (!page || typeof page.evaluate !== 'function') { + return []; + } + try { + const pending = await page.evaluate(() => { + const perf = window.performance; + if (!perf?.getEntriesByType) { + return []; + } + const entries = perf.getEntriesByType('resource'); + const now = perf.now?.() ?? Date.now(); + const blockedPatterns = [ + 'doubleclick', + 'analytics', + 'tracking', + 'metrics', + 'telemetry', + 'facebook.net', + 'hotjar', + 'clarity', + 'mixpanel', + 'segment', + '/beacon/', + '/collector/', + '/telemetry/', + ]; + const pendingRequests = []; + for (const entry of entries) { + const responseEnd = typeof entry.responseEnd === 'number' + ? entry.responseEnd + : 0; + if (responseEnd !== 0) { + continue; + } + const url = String(entry.name ?? ''); + if (!url || url.startsWith('data:') || url.length > 500) { + continue; + } + const lower = url.toLowerCase(); + if (blockedPatterns.some((pattern) => lower.includes(pattern))) { + continue; + } + const startTime = typeof entry.startTime === 'number' + ? entry.startTime + : now; + const loadingDuration = Math.max(0, now - startTime); + if (loadingDuration > 10000) { + continue; + } + const resourceType = String(entry.initiatorType ?? '').toLowerCase(); + if ((resourceType === 'img' || + resourceType === 'image' || + resourceType === 'font') && + loadingDuration > 3000) { + continue; + } + pendingRequests.push({ + url, + method: 'GET', + loading_duration_ms: Math.round(loadingDuration), + resource_type: resourceType || null, + }); + if (pendingRequests.length >= 20) { + break; + } + } + return pendingRequests; + }); + return Array.isArray(pending) + ? pending.map((entry) => ({ + url: String(entry.url ?? ''), + method: typeof entry.method === 'string' + ? entry.method + : 'GET', + loading_duration_ms: typeof entry.loading_duration_ms === 'number' + ? entry.loading_duration_ms + : 0, + resource_type: typeof entry.resource_type === 'string' + ? entry.resource_type + : null, + })) + : []; + } + catch (error) { + this.logger.debug(`Failed to gather pending network requests: ${error.message}`); + return []; + } + } + get tabs() { + return this._tabs.slice(); + } + get active_tab_index() { + return this.currentTabIndex; + } + get active_tab() { + return this._tabs[this.currentTabIndex] ?? null; + } + describe() { + return this.toString(); + } + get _owns_browser_resources() { + return this.ownsBrowserResources; + } + get is_stopping() { + return this._stoppingPromise !== null; + } + get is_reconnecting() { + return this._reconnecting; + } + get should_gate_watchdog_events() { + return Boolean(this.initialized || + this.browser || + this.browser_context || + this.cdp_url || + this.wss_url || + this._reconnecting); + } + get is_cdp_connected() { + try { + if (this.browser) { + const browser = this.browser; + if (typeof browser.isConnected === 'function' && + !browser.isConnected()) { + return false; + } + } + if (this.browser_context) { + const contextBrowser = this.browser_context.browser?.(); + if (contextBrowser && + typeof contextBrowser.isConnected === 'function' && + !contextBrowser.isConnected()) { + return false; + } + return true; + } + return Boolean(this.browser); + } + catch { + return false; + } + } + async wait_for_reconnect(timeoutSeconds = this.RECONNECT_WAIT_TIMEOUT) { + if (!this._reconnecting) { + return; + } + const timeoutMs = Number.isFinite(timeoutSeconds) && timeoutSeconds > 0 + ? timeoutSeconds * 1000 + : this.RECONNECT_WAIT_TIMEOUT * 1000; + let timeoutHandle = null; + try { + await Promise.race([ + this._reconnectWaitPromise, + new Promise((_, reject) => { + timeoutHandle = setTimeout(() => { + reject(new Error(`Reconnection wait timed out after ${Math.round(timeoutMs / 1000)}s`)); + }, timeoutMs); + }), + ]); + } + finally { + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + } + } + claim_agent(agentId, mode = 'exclusive') { + if (!agentId) { + return false; + } + if (mode === 'shared') { + if (this.attachedAgentId && + this.attachedAgentId !== agentId && + this.attachedSharedAgentIds.size === 0) { + return false; + } + if (this.attachedSharedAgentIds.size === 0 && this.attachedAgentId) { + this.attachedSharedAgentIds.add(this.attachedAgentId); + } + this.attachedSharedAgentIds.add(agentId); + this.attachedAgentId = this.attachedAgentId ?? agentId; + return true; + } + if (this.attachedSharedAgentIds.size > 0) { + if (this.attachedSharedAgentIds.size === 1 && + this.attachedSharedAgentIds.has(agentId)) { + this.attachedSharedAgentIds.clear(); + this.attachedAgentId = agentId; + return true; + } + return false; + } + if (this.attachedAgentId && this.attachedAgentId !== agentId) { + return false; + } + this.attachedAgentId = agentId; + return true; + } + claimAgent(agentId, mode = 'exclusive') { + return this.claim_agent(agentId, mode); + } + release_agent(agentId) { + if (this.attachedSharedAgentIds.size > 0) { + if (!agentId) { + this.attachedSharedAgentIds.clear(); + this.attachedAgentId = null; + return true; + } + if (!this.attachedSharedAgentIds.has(agentId)) { + return false; + } + this.attachedSharedAgentIds.delete(agentId); + if (this.attachedSharedAgentIds.size === 0) { + this.attachedAgentId = null; + } + else if (this.attachedAgentId === agentId) { + const [nextOwner] = this.attachedSharedAgentIds; + this.attachedAgentId = nextOwner ?? null; + } + return true; + } + if (!this.attachedAgentId) { + return true; + } + if (agentId && this.attachedAgentId !== agentId) { + return false; + } + this.attachedAgentId = null; + return true; + } + releaseAgent(agentId) { + return this.release_agent(agentId); + } + get_attached_agent_id() { + return this.attachedAgentId; + } + getAttachedAgentId() { + return this.get_attached_agent_id(); + } + get_attached_agent_ids() { + if (this.attachedSharedAgentIds.size > 0) { + return Array.from(this.attachedSharedAgentIds); + } + return this.attachedAgentId ? [this.attachedAgentId] : []; + } + getAttachedAgentIds() { + return this.get_attached_agent_ids(); + } + _determineOwnership() { + if (this.cdp_url || + this.wss_url || + this.browser || + this.browser_context || + this.browser_pid) { + return false; + } + return true; + } + _createAbortError(reason) { + if (reason instanceof Error) { + return reason; + } + const error = new Error('Operation aborted'); + error.name = 'AbortError'; + return error; + } + _isAbortError(error) { + if (!(error instanceof Error)) { + return false; + } + return (error.name === 'AbortError' || + /abort|aborted|interrupted/i.test(error.message)); + } + _throwIfAborted(signal = null) { + if (signal?.aborted) { + throw this._createAbortError(signal.reason); + } + } + async _waitWithAbort(timeoutMs, signal = null) { + if (timeoutMs <= 0) { + this._throwIfAborted(signal); + return; + } + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + resolve(); + }, timeoutMs); + const onAbort = () => { + clearTimeout(timeout); + cleanup(); + reject(this._createAbortError(signal?.reason)); + }; + const cleanup = () => { + signal?.removeEventListener('abort', onAbort); + }; + if (signal) { + if (signal.aborted) { + onAbort(); + return; + } + signal.addEventListener('abort', onAbort, { once: true }); + } + }); + } + async _withAbort(promise, signal = null) { + if (!signal) { + return promise; + } + this._throwIfAborted(signal); + return await new Promise((resolve, reject) => { + const onAbort = () => { + cleanup(); + reject(this._createAbortError(signal.reason)); + }; + const cleanup = () => { + signal.removeEventListener('abort', onAbort); + }; + signal.addEventListener('abort', onAbort, { once: true }); + promise + .then((result) => { + cleanup(); + resolve(result); + }) + .catch((error) => { + cleanup(); + reject(error); + }); + }); + } + _toPlaywrightOptions(value) { + if (value === null || value === undefined) { + return undefined; + } + if (Array.isArray(value)) { + const converted = value + .map((item) => this._toPlaywrightOptions(item)) + .filter((item) => item !== undefined); + return converted; + } + if (typeof value !== 'object' || + value instanceof Date || + Buffer.isBuffer(value)) { + return value; + } + const result = {}; + for (const [rawKey, rawVal] of Object.entries(value)) { + const convertedValue = this._toPlaywrightOptions(rawVal); + if (convertedValue === undefined) { + continue; + } + const normalizedKey = PLAYWRIGHT_OPTION_KEY_OVERRIDES[rawKey] ?? + rawKey.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()); + result[normalizedKey] = convertedValue; + } + return result; + } + async set_extra_headers(headers) { + const normalizedHeaders = Object.fromEntries(Object.entries(headers) + .map(([key, value]) => [String(key).trim(), String(value)]) + .filter(([key]) => key.length > 0)); + if (!this.browser_context || + Object.keys(normalizedHeaders).length === 0 || + typeof this.browser_context.setExtraHTTPHeaders !== 'function') { + return; + } + await this.browser_context.setExtraHTTPHeaders(normalizedHeaders); + } + async _applyConfiguredExtraHttpHeaders() { + const configuredHeaders = this.browser_profile.config.extra_http_headers; + if (!configuredHeaders || Object.keys(configuredHeaders).length === 0) { + return; + } + await this.set_extra_headers(configuredHeaders); + } + _usesRemoteBrowserConnection() { + return Boolean(this.cdp_url || this.wss_url); + } + async _connectToConfiguredBrowser(playwright) { + const connectOptions = this._toPlaywrightOptions(this.browser_profile.kwargs_for_connect()); + if (this.cdp_url) { + return await playwright.chromium.connectOverCDP(this.cdp_url, connectOptions ?? {}); + } + if (this.wss_url) { + return await playwright.chromium.connect(this.wss_url, connectOptions ?? {}); + } + throw new Error('Cannot connect to a remote browser without cdp_url or wss_url'); + } + async _ensureBrowserContextFromBrowser(browser) { + const existingContexts = (typeof browser?.contexts === 'function' ? browser.contexts() : []) ?? []; + if (existingContexts.length > 0) { + return existingContexts[0] ?? null; + } + if (typeof browser?.newContext === 'function') { + const contextOptions = this._toPlaywrightOptions(this.browser_profile.kwargs_for_new_context()); + return await browser.newContext(contextOptions ?? {}); + } + return null; + } + _beginReconnectWait() { + this._reconnectWaitPromise = new Promise((resolve) => { + this._resolveReconnectWait = resolve; + }); + } + _endReconnectWait() { + this._resolveReconnectWait?.(); + this._resolveReconnectWait = null; + this._reconnectWaitPromise = Promise.resolve(); + } + _detachRemoteDisconnectHandler() { + if (!this._disconnectAwareBrowser || !this._browserDisconnectHandler) { + this._disconnectAwareBrowser = null; + this._browserDisconnectHandler = null; + return; + } + if (typeof this._disconnectAwareBrowser.off === 'function') { + this._disconnectAwareBrowser.off('disconnected', this._browserDisconnectHandler); + } + else if (typeof this._disconnectAwareBrowser.removeListener === 'function') { + this._disconnectAwareBrowser.removeListener('disconnected', this._browserDisconnectHandler); + } + this._disconnectAwareBrowser = null; + this._browserDisconnectHandler = null; + } + _attachRemoteDisconnectHandler(browser) { + this._detachRemoteDisconnectHandler(); + if (!this._usesRemoteBrowserConnection()) { + return; + } + const browserWithEvents = browser; + if (!browserWithEvents || typeof browserWithEvents.on !== 'function') { + return; + } + const onDisconnected = () => { + this._handleUnexpectedRemoteDisconnect(); + }; + browserWithEvents.on('disconnected', onDisconnected); + this._disconnectAwareBrowser = browserWithEvents; + this._browserDisconnectHandler = onDisconnected; + } + _handleUnexpectedRemoteDisconnect() { + if (this._intentionalStop || + this._reconnecting || + !this._usesRemoteBrowserConnection()) { + return; + } + this.logger.warning('Remote browser connection closed unexpectedly; attempting to reconnect'); + this._recordRecentEvent('browser_disconnected', { + url: this.currentUrl, + }); + const reconnectTask = this._auto_reconnect(); + this._reconnectTask = reconnectTask; + void reconnectTask.finally(() => { + if (this._reconnectTask === reconnectTask) { + this._reconnectTask = null; + } + }); + } + async _restorePagesAfterReconnect(preferredUrl, preferredTabIndex) { + if (!this.browser_context) { + this.agent_current_page = null; + this.human_current_page = null; + return; + } + let pages = this.browser_context.pages?.() ?? []; + if (!pages.length && typeof this.browser_context.newPage === 'function') { + const createdPage = await this.browser_context.newPage(); + if (createdPage) { + pages = this.browser_context.pages?.() ?? [createdPage]; + } + } + this.tabPages = new Map(); + this.agent_current_page = null; + this.human_current_page = null; + this._syncTabsWithBrowserPages(); + if (!pages.length) { + this.currentTabIndex = 0; + this.currentUrl = normalize_url(preferredUrl ?? 'about:blank'); + this.currentTitle = this.currentUrl; + if (!this._tabs.length) { + this._tabs = [ + this._createTabInfo({ + page_id: this._tabCounter++, + url: this.currentUrl, + title: this.currentTitle, + }), + ]; + } + this._syncSessionManagerFromTabs(); + return; + } + const normalizedPreferredUrl = typeof preferredUrl === 'string' && preferredUrl.trim().length > 0 + ? normalize_url(preferredUrl) + : null; + const pageByUrl = normalizedPreferredUrl == null + ? null + : (pages.find((page) => { + try { + return normalize_url(page.url()) === normalizedPreferredUrl; + } + catch { + return false; + } + }) ?? null); + const clampedIndex = preferredTabIndex >= 0 && preferredTabIndex < pages.length + ? preferredTabIndex + : 0; + const nextPage = pageByUrl ?? pages[clampedIndex] ?? pages[0] ?? null; + const nextTabIndex = nextPage + ? this._tabs.findIndex((tab) => this.tabPages.get(tab.page_id) === nextPage) + : -1; + if (nextTabIndex >= 0) { + this.currentTabIndex = nextTabIndex; + } + this._setActivePage(nextPage); + this.human_current_page = nextPage; + await this._syncCurrentTabFromPage(nextPage); + } + async reconnect(options = {}) { + if (!this._usesRemoteBrowserConnection()) { + throw new Error('Cannot reconnect without a remote browser connection'); + } + const preferredUrl = typeof options.preferred_url === 'string' + ? options.preferred_url + : this.currentUrl; + const preferredTabIndex = typeof options.preferred_tab_index === 'number' + ? options.preferred_tab_index + : this.currentTabIndex; + this._detachRemoteDisconnectHandler(); + this.cachedBrowserState = null; + this.currentPageLoadingStatus = null; + this.browser = null; + this.browser_context = null; + this.agent_current_page = null; + this.human_current_page = null; + this._dialogHandlersAttached = new WeakSet(); + this.session_manager.clear(); + const playwright = this.playwright ?? (await async_playwright()); + this.playwright = playwright; + this.browser = await this._connectToConfiguredBrowser(playwright); + this.ownsBrowserResources = false; + this.browser_context = await this._ensureBrowserContextFromBrowser(this.browser); + await this._applyConfiguredExtraHttpHeaders(); + await this._restorePagesAfterReconnect(preferredUrl, preferredTabIndex); + this._attachRemoteDisconnectHandler(this.browser); + this.initialized = true; + this._recordRecentEvent('browser_reconnected', { + url: this.currentUrl, + }); + } + async _auto_reconnect(maxAttempts = 3) { + if (this._reconnecting || !this._usesRemoteBrowserConnection()) { + return; + } + this._reconnecting = true; + this._beginReconnectWait(); + const startTime = Date.now(); + const preferredUrl = this.currentUrl; + const preferredTabIndex = this.currentTabIndex; + try { + await this.event_bus.dispatch(new BrowserStoppedEvent({ + reason: 'connection_lost', + })); + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + if (this._intentionalStop) { + return; + } + await this.event_bus.dispatch(new BrowserReconnectingEvent({ + cdp_url: this.cdp_url ?? this.wss_url ?? 'remote', + attempt, + max_attempts: maxAttempts, + })); + try { + await Promise.race([ + this.reconnect({ + preferred_url: preferredUrl, + preferred_tab_index: preferredTabIndex, + }), + new Promise((_, reject) => { + setTimeout(() => { + reject(new Error(`Reconnect attempt timed out after ${Math.round(REMOTE_RECONNECT_ATTEMPT_TIMEOUT_MS / 1000)}s`)); + }, REMOTE_RECONNECT_ATTEMPT_TIMEOUT_MS); + }), + ]); + if (this._intentionalStop) { + return; + } + await this.event_bus.dispatch(new BrowserConnectedEvent({ + cdp_url: this.cdp_url ?? this.wss_url ?? 'remote', + })); + await this.event_bus.dispatch(new BrowserReconnectedEvent({ + cdp_url: this.cdp_url ?? this.wss_url ?? 'remote', + attempt, + downtime_seconds: (Date.now() - startTime) / 1000, + })); + return; + } + catch (error) { + this.logger.warning(`Reconnect attempt ${attempt}/${maxAttempts} failed: ${error.message}`); + if (attempt >= maxAttempts) { + break; + } + const delayMs = REMOTE_RECONNECT_DELAYS_MS[attempt - 1] ?? + REMOTE_RECONNECT_DELAYS_MS[REMOTE_RECONNECT_DELAYS_MS.length - 1]; + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'ReconnectionFailed', + message: `Failed to reconnect after ${maxAttempts} attempts (${((Date.now() - startTime) / 1000).toFixed(1)}s)`, + details: { + cdp_url: this.cdp_url ?? this.wss_url ?? 'remote', + max_attempts: maxAttempts, + }, + })); + } + finally { + this._reconnecting = false; + this._endReconnectWait(); + } + } + _isSandboxLaunchError(error) { + const message = error instanceof Error ? error.message : String(error); + return (/no usable sandbox/i.test(message) || + /chromium sandboxing failed/i.test(message) || + /zygote_host_impl_linux\.cc/i.test(message)); + } + _createNoSandboxLaunchOptions(launchOptions) { + const rawArgs = Array.isArray(launchOptions.args) + ? launchOptions.args.filter((arg) => typeof arg === 'string') + : []; + const mergedArgs = [...rawArgs]; + for (const arg of CHROME_DOCKER_ARGS) { + if (!mergedArgs.includes(arg)) { + mergedArgs.push(arg); + } + } + return { + ...launchOptions, + chromiumSandbox: false, + args: mergedArgs, + }; + } + async _launchChromiumWithSandboxFallback(playwright, launchOptions) { + try { + return await playwright.chromium.launch(launchOptions); + } + catch (error) { + const sandboxEnabled = this.browser_profile.config.chromium_sandbox; + if (!sandboxEnabled || !this._isSandboxLaunchError(error)) { + throw error; + } + this.logger.warning('Chromium sandbox is unavailable in this environment. Retrying launch with chromium_sandbox=false (--no-sandbox).'); + const fallbackOptions = this._createNoSandboxLaunchOptions(launchOptions); + return await playwright.chromium.launch(fallbackOptions); + } + } + _connectionDescriptor() { + const source = this.cdp_url || + this.wss_url || + (this.browser_pid ? String(this.browser_pid) : 'playwright'); + const tail = source.split('/').pop() ?? source; + const port = tail.includes(':') ? tail.split(':').pop() : tail; + return `${this.id.slice(-4)}:${port}`; + } + toString() { + const ownershipFlag = this.ownsBrowserResources ? '#' : '©'; + return `BrowserSession🆂 ${this._connectionDescriptor()} ${ownershipFlag}${String(this.id).slice(-2)}`; + } + get logger() { + if (!this._logger) { + this._logger = createLogger(`browser_use.browser.session.${this.id.slice(-4)}`); + } + return this._logger; + } + async start() { + this.attach_default_watchdogs(); + this._intentionalStop = false; + if (this.initialized) { + return this; + } + await this.event_bus.dispatch(new BrowserStartEvent({ + cdp_url: this.cdp_url, + })); + const ensurePage = async () => { + const current = this.agent_current_page; + if (current && !current.isClosed?.()) { + this._setActivePage(current); + return; + } + const existingPages = (typeof this.browser_context?.pages === 'function' + ? this.browser_context.pages() + : []) ?? []; + const firstOpenPage = existingPages.find((page) => !page.isClosed?.()) ?? null; + if (firstOpenPage) { + this._setActivePage(firstOpenPage); + return; + } + if (typeof this.browser_context?.newPage === 'function') { + const created = await this.browser_context.newPage(); + this._setActivePage(created ?? null); + return; + } + this._setActivePage(null); + }; + if (!this.browser_context) { + if (!this.browser) { + const playwright = this.playwright ?? (await async_playwright()); + this.playwright = playwright; + if (this.cdp_url) { + this.browser = await this._connectToConfiguredBrowser(playwright); + this.ownsBrowserResources = false; + } + else if (this.wss_url) { + this.browser = await this._connectToConfiguredBrowser(playwright); + this.ownsBrowserResources = false; + } + else { + const launchOptions = this._toPlaywrightOptions(await this.browser_profile.kwargs_for_launch()); + this.browser = await this._launchChromiumWithSandboxFallback(playwright, launchOptions ?? {}); + this.ownsBrowserResources = true; + const processGetter = this.browser?.process; + if (typeof processGetter === 'function') { + const processRef = processGetter.call(this.browser); + if (typeof processRef?.pid === 'number') { + this.browser_pid = processRef.pid; + } + } + } + } + const existingContexts = (typeof this.browser?.contexts === 'function' + ? this.browser.contexts() + : []) ?? []; + if (existingContexts.length > 0) { + this.browser_context = existingContexts[0] ?? null; + } + else { + this.browser_context = await this._ensureBrowserContextFromBrowser(this.browser); + } + } + await this._applyConfiguredExtraHttpHeaders(); + await ensurePage(); + if (!this.human_current_page || + this.human_current_page.isClosed?.()) { + this.human_current_page = this.agent_current_page; + } + const activePage = await this.get_current_page(); + if (activePage) { + try { + this.currentUrl = normalize_url(activePage.url()); + } + catch { + // Ignore url read errors from transient pages. + } + if (typeof activePage.title === 'function') { + try { + this.currentTitle = await activePage.title(); + } + catch { + // Ignore title read errors from transient pages. + } + } + } + this.initialized = true; + this._recordRecentEvent('browser_started', { url: this.currentUrl }); + this.logger.debug(`Started ${this.describe()} with profile ${this.browser_profile.toString()}`); + this._attachRemoteDisconnectHandler(this.browser); + await this.event_bus.dispatch(new BrowserConnectedEvent({ + cdp_url: this.cdp_url ?? this.wss_url ?? 'playwright', + })); + return this; + } + /** + * Setup browser session by connecting to an existing browser process via PID + * Useful for debugging or connecting to manually launched browsers + * @param browserPid - Process ID of the browser to connect to + * @param cdpUrl - Optional CDP URL (will be discovered if not provided) + */ + async setupBrowserViaBrowserPid(browserPid, cdpUrl) { + this.logger.info(`Connecting to existing browser with PID ${browserPid}`); + this.browser_pid = browserPid; + // If CDP URL not provided, try to discover it + if (!cdpUrl) { + cdpUrl = (await this._discoverCdpUrl(browserPid)) ?? undefined; + } + if (!cdpUrl) { + throw new Error(`Could not discover CDP URL for browser PID ${browserPid}`); + } + this.cdp_url = cdpUrl; + this.logger.info(`Discovered CDP URL: ${cdpUrl}`); + // Connect to browser via CDP + try { + const playwright = await import('playwright'); + const browser = await this._connectToConfiguredBrowser(playwright); + this.browser = browser; + this.playwright = playwright; + // Get or create context + const contexts = browser.contexts(); + if (contexts.length > 0) { + this.browser_context = contexts[0]; + } + else { + this.browser_context = (await browser.newContext()); + } + await this._applyConfiguredExtraHttpHeaders(); + // Get or create page + if (!this.browser_context) { + throw new Error('Browser context not available'); + } + const pages = this.browser_context.pages(); + if (pages.length > 0) { + this.agent_current_page = pages[0]; + this.human_current_page = pages[0]; + } + else { + const page = await this.browser_context.newPage(); + this.agent_current_page = page; + this.human_current_page = page; + } + // We don't own this browser since we're connecting to existing one + this.ownsBrowserResources = false; + this._attachRemoteDisconnectHandler(this.browser); + this.initialized = true; + this.logger.info(`Successfully connected to browser PID ${browserPid}`); + } + catch (error) { + throw new Error(`Failed to connect to browser PID ${browserPid}: ${error.message}`, { cause: error }); + } + } + /** + * Discover CDP URL from browser PID + * Tries common ports and checks for debugging endpoints + */ + async _discoverCdpUrl(browserPid) { + const commonPorts = [9222, 9223, 9224, 9225]; + for (const port of commonPorts) { + try { + const response = await fetch(`http://localhost:${port}/json/version`); + if (response.ok) { + const data = await response.json(); + if (data.webSocketDebuggerUrl) { + this.logger.debug(`Found CDP endpoint on port ${port}`); + return data.webSocketDebuggerUrl; + } + } + } + catch { + // Port not accessible, try next + continue; + } + } + this.logger.warning(`Could not discover CDP URL for PID ${browserPid} on common ports`); + return null; + } + async _shutdown_browser_session() { + this.initialized = false; + this._intentionalStop = true; + this._reconnecting = false; + this._endReconnectWait(); + this._reconnectTask = null; + this._detachRemoteDisconnectHandler(); + this.attachedAgentId = null; + this.attachedSharedAgentIds.clear(); + const closeWithTimeout = async (label, operation, timeoutMs = 3000) => { + let timeoutHandle = null; + const timeoutPromise = new Promise((_, reject) => { + timeoutHandle = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs); + }); + try { + await Promise.race([operation, timeoutPromise]); + } + finally { + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + } + }; + if (this.ownsBrowserResources) { + if (typeof this.browser_context?.close === 'function') { + try { + await closeWithTimeout('Closing browser context', this.browser_context.close()); + } + catch (error) { + this.logger.debug(`Failed to close browser context: ${error.message}`); + } + } + if (typeof this.browser?.close === 'function') { + try { + await closeWithTimeout('Closing browser instance', this.browser.close()); + } + catch (error) { + this.logger.debug(`Failed to close browser instance: ${error.message}`); + } + } + } + // Kill child processes first + await this._killChildProcesses(); + // If we own the browser resources, terminate the browser process + if (this.ownsBrowserResources && this.browser_pid) { + await this._terminateBrowserProcess(); + } + this.browser = null; + this.browser_context = null; + this.agent_current_page = null; + this.human_current_page = null; + this.browser_pid = null; + this.cdp_url = null; + this.wss_url = null; + this.playwright = null; + this.cachedBrowserState = null; + this._tabs = []; + this.session_manager.clear(); + this.downloaded_files = []; + this._closedPopupMessages = []; + this._dialogHandlersAttached = new WeakSet(); + this._recentEvents = []; + } + async close() { + await this.stop(); + } + async get_browser_state_with_recovery(options = {}) { + const signal = options.signal ?? null; + const includeRecentEvents = options.include_recent_events ?? false; + this._throwIfAborted(signal); + if (!this.initialized) { + await this._withAbort(this.start(), signal); + } + const page = await this._withAbort(this.get_current_page(), signal); + this._throwIfAborted(signal); + this.cachedBrowserState = null; + let domState; + if (!page) { + domState = createEmptyDomState(); + } + else { + try { + const domService = new DomService(page, this.logger); + domState = await this._withAbort(domService.get_clickable_elements(this.browser_profile.highlight_elements, -1, this.browser_profile.viewport_expansion), signal); + } + catch (error) { + if (this._isAbortError(error)) { + throw error; + } + this.logger.debug(`Failed to build DOM tree: ${error.message}`); + domState = createEmptyDomState(); + } + const liveUrl = typeof page.url === 'function' + ? normalize_url(page.url()) + : this.currentUrl; + const shouldRetryEmptyDom = Object.keys(domState.selector_map).length === 0 && + !this._is_new_tab_page(liveUrl) && + !liveUrl.toLowerCase().endsWith('.pdf'); + if (shouldRetryEmptyDom) { + this.logger.debug(`Empty DOM detected for ${liveUrl}; retrying once`); + await this._waitWithAbort(EMPTY_DOM_RETRY_DELAY_MS, signal); + try { + const retryDomService = new DomService(page, this.logger); + const retriedDomState = await this._withAbort(retryDomService.get_clickable_elements(this.browser_profile.highlight_elements, -1, this.browser_profile.viewport_expansion), signal); + if (Object.keys(retriedDomState.selector_map).length > 0) { + domState = retriedDomState; + } + } + catch (error) { + if (this._isAbortError(error)) { + throw error; + } + this.logger.debug(`Retry after empty DOM failed: ${error.message}`); + } + } + } + let screenshot = null; + if (options.include_screenshot && page?.screenshot) { + try { + const image = await this._withAbort(page.screenshot({ + type: 'png', + fullPage: true, + }), signal); + screenshot = + typeof image === 'string' + ? image + : Buffer.from(image).toString('base64'); + } + catch (error) { + if (this._isAbortError(error)) { + throw error; + } + this.logger.debug(`Failed to capture screenshot: ${error.message}`); + } + } + let pageInfo = null; + let pixelsAbove = 0; + let pixelsBelow = 0; + if (page) { + try { + const metrics = await this._withAbort(page.evaluate(() => { + const doc = document.documentElement; + const body = document.body; + const width = Math.max(doc?.scrollWidth ?? 0, body?.scrollWidth ?? 0, doc?.clientWidth ?? 0); + const height = Math.max(doc?.scrollHeight ?? 0, body?.scrollHeight ?? 0, doc?.clientHeight ?? 0); + return { + viewportWidth: window.innerWidth, + viewportHeight: window.innerHeight, + scrollX: window.scrollX, + scrollY: window.scrollY, + pageWidth: width, + pageHeight: height, + }; + }), signal); + pixelsAbove = Math.max(metrics.scrollY ?? 0, 0); + const viewportHeight = metrics.viewportHeight ?? 0; + const viewportWidth = metrics.viewportWidth ?? 0; + pixelsBelow = Math.max((metrics.pageHeight ?? 0) - (metrics.scrollY + viewportHeight), 0); + const pixelsLeft = Math.max(metrics.scrollX ?? 0, 0); + const pixelsRight = Math.max((metrics.pageWidth ?? 0) - (metrics.scrollX + viewportWidth), 0); + pageInfo = { + viewport_width: viewportWidth, + viewport_height: viewportHeight, + page_width: metrics.pageWidth ?? viewportWidth, + page_height: metrics.pageHeight ?? viewportHeight, + scroll_x: metrics.scrollX ?? 0, + scroll_y: metrics.scrollY ?? 0, + pixels_above: pixelsAbove, + pixels_below: pixelsBelow, + pixels_left: pixelsLeft, + pixels_right: pixelsRight, + }; + } + catch (error) { + if (this._isAbortError(error)) { + throw error; + } + this.logger.debug(`Failed to compute page metrics: ${error.message}`); + } + } + const pendingNetworkRequests = await this._getPendingNetworkRequests(page); + if (page) { + await this._syncCurrentTabFromPage(page); + } + if (pageInfo && + Number.isFinite(pageInfo.viewport_width) && + Number.isFinite(pageInfo.viewport_height)) { + this._original_viewport_size = [ + Math.floor(pageInfo.viewport_width), + Math.floor(pageInfo.viewport_height), + ]; + } + const paginationButtons = DomService.detect_pagination_buttons(domState.selector_map); + const summary = new BrowserStateSummary(domState, { + url: this.currentUrl, + title: this.currentTitle || this.currentUrl, + tabs: this._buildTabs(), + screenshot, + page_info: pageInfo, + pixels_above: pixelsAbove, + pixels_below: pixelsBelow, + browser_errors: this.currentPageLoadingStatus + ? [this.currentPageLoadingStatus] + : [], + is_pdf_viewer: Boolean(this.currentUrl?.toLowerCase().endsWith('.pdf')), + loading_status: this.currentPageLoadingStatus, + recent_events: includeRecentEvents + ? this._getRecentEventsSummary() + : null, + pending_network_requests: pendingNetworkRequests, + pagination_buttons: paginationButtons, + closed_popup_messages: this._getClosedPopupMessagesSnapshot(), + }); + // Implement clickable element hash caching to detect new elements + if (options.cache_clickable_elements_hashes && page) { + const currentUrl = page.url(); + const currentHashes = this._computeElementHashes(domState.selector_map); + // Mark new elements if we have cached hashes for this URL + if (this._cachedClickableElementHashes && + this._cachedClickableElementHashes.url === currentUrl) { + this._markNewElements(domState.selector_map, this._cachedClickableElementHashes.hashes); + } + // Update cache with current hashes + this._cachedClickableElementHashes = { + url: currentUrl, + hashes: currentHashes, + }; + } + this._throwIfAborted(signal); + this.cachedBrowserState = summary; + return summary; + } + async get_current_page() { + this._syncTabsWithBrowserPages(); + if (this.agent_current_page) { + return this.agent_current_page; + } + const currentTab = this._tabs[this.currentTabIndex]; + if (currentTab) { + const tabPage = this.tabPages.get(currentTab.page_id) ?? null; + if (tabPage) { + this._setActivePage(tabPage); + return tabPage; + } + } + const fallback = this.browser_context?.pages()?.[0] ?? null; + this._setActivePage(fallback ?? null); + return fallback; + } + update_current_page(page, title, url) { + this._setActivePage(page); + this.human_current_page = this.human_current_page ?? page; + if (url) { + this.currentUrl = normalize_url(url); + } + if (title) { + this.currentTitle = title; + } + } + _buildTabs() { + if (!this._tabs.length) { + const pageId = this._tabCounter++; + this._tabs.push(this._createTabInfo({ + page_id: pageId, + url: this.currentUrl, + title: this.currentTitle || this.currentUrl, + })); + } + else { + const tab = this._tabs[this.currentTabIndex]; + if (tab && !tab.tab_id) { + tab.tab_id = this._formatTabId(tab.page_id); + } + tab.url = this.currentUrl; + tab.title = this.currentTitle || this.currentUrl; + } + this._syncSessionManagerFromTabs(); + return this._tabs.slice(); + } + async navigate_to(url, options = {}) { + const signal = options.signal ?? null; + this._throwIfAborted(signal); + this._assert_url_allowed(url); + const normalized = normalize_url(url); + let completedUrl = normalized; + const waitUntil = options.wait_until ?? 'domcontentloaded'; + const timeoutMs = typeof options.timeout_ms === 'number' && + Number.isFinite(options.timeout_ms) + ? Math.max(0, options.timeout_ms) + : null; + this._recordRecentEvent('navigation_started', { url: normalized }); + const page = await this._withAbort(this.get_current_page(), signal); + if (page?.goto) { + try { + this.currentPageLoadingStatus = null; + const gotoOptions = { + waitUntil, + }; + if (timeoutMs !== null) { + gotoOptions.timeout = timeoutMs; + } + await this._withAbort(page.goto(normalized, gotoOptions), signal); + const finalUrl = page.url(); + this._assert_url_allowed(finalUrl); + completedUrl = normalize_url(finalUrl); + await this._waitForStableNetwork(page, signal); + } + catch (error) { + if (this._isAbortError(error)) { + throw error; + } + const message = error.message ?? 'Navigation failed'; + this._recordRecentEvent('navigation_failed', { + url: normalized, + error_message: message, + }); + throw new BrowserError(message); + } + } + this._throwIfAborted(signal); + if (page) { + await this._syncCurrentTabFromPage(page); + completedUrl = this.currentUrl || completedUrl; + } + else { + this.currentUrl = normalized; + this.currentTitle = normalized; + if (this._tabs[this.currentTabIndex]) { + this._tabs[this.currentTabIndex].url = normalized; + this._tabs[this.currentTabIndex].title = normalized; + } + this._syncSessionManagerFromTabs(); + } + if (this.historyStack[this.historyStack.length - 1] !== completedUrl) { + this.historyStack.push(completedUrl); + } + this._setActivePage(page ?? null); + this._recordRecentEvent('navigation_completed', { url: completedUrl }); + this.cachedBrowserState = null; + return this.agent_current_page; + } + async create_new_tab(url, options = {}) { + const signal = options.signal ?? null; + this._throwIfAborted(signal); + this._assert_url_allowed(url); + const normalized = normalize_url(url); + let completedUrl = normalized; + const waitUntil = options.wait_until ?? 'domcontentloaded'; + const timeoutMs = typeof options.timeout_ms === 'number' && + Number.isFinite(options.timeout_ms) + ? Math.max(0, options.timeout_ms) + : null; + const previousTabIndex = this.currentTabIndex; + const previousTab = this._tabs[this.currentTabIndex] ?? null; + const newTab = this._createTabInfo({ + page_id: this._tabCounter++, + url: normalized, + title: normalized, + }); + this._tabs.push(newTab); + this.currentTabIndex = this._tabs.length - 1; + this.currentUrl = normalized; + this.currentTitle = normalized; + this.historyStack.push(normalized); + let page = null; + try { + page = + (await this._withAbort(this.browser_context?.newPage?.() ?? Promise.resolve(null), signal)) ?? null; + if (page) { + this.currentPageLoadingStatus = null; + const gotoOptions = { + waitUntil, + }; + if (timeoutMs !== null) { + gotoOptions.timeout = timeoutMs; + } + await this._withAbort(page.goto(normalized, gotoOptions), signal); + const finalUrl = page.url(); + this._assert_url_allowed(finalUrl); + completedUrl = normalize_url(finalUrl); + await this._waitForStableNetwork(page, signal); + } + } + catch (error) { + if (this._isAbortError(error)) { + throw error; + } + const message = error.message ?? 'Failed to open new tab'; + this._recordRecentEvent('tab_navigation_failed', { + url: normalized, + page_id: newTab.page_id, + tab_id: newTab.tab_id, + error_message: message, + }); + this.logger.debug(`Failed to open new tab via Playwright: ${message}`); + if (page?.close) { + try { + await page.close(); + } + catch { + // Ignore best-effort tab close failures during rollback. + } + } + this._tabs = this._tabs.filter((tab) => tab.page_id !== newTab.page_id); + this.tabPages.delete(newTab.page_id); + if (this.historyStack[this.historyStack.length - 1] === normalized) { + this.historyStack.pop(); + } + if (this._tabs.length > 0) { + let restoredIndex = previousTab + ? this._tabs.findIndex((tab) => tab.page_id === previousTab.page_id) + : -1; + if (restoredIndex === -1) { + restoredIndex = Math.min(previousTabIndex, this._tabs.length - 1); + } + this.currentTabIndex = Math.max(0, restoredIndex); + const restoredTab = this._tabs[this.currentTabIndex]; + this.currentUrl = restoredTab.url; + this.currentTitle = restoredTab.title; + const restoredPage = this.tabPages.get(restoredTab.page_id) ?? null; + this._setActivePage(restoredPage); + await this._syncCurrentTabFromPage(restoredPage); + } + else { + this.currentTabIndex = 0; + this.currentUrl = 'about:blank'; + this.currentTitle = 'about:blank'; + this._setActivePage(null); + } + this._syncSessionManagerFromTabs(); + this.cachedBrowserState = null; + throw new BrowserError(message); + } + this.tabPages.set(newTab.page_id, page); + this._syncSessionManagerFromTabs(); + this._setActivePage(page); + if (page) { + await this._syncCurrentTabFromPage(page); + completedUrl = this.currentUrl || completedUrl; + } + if (this.historyStack[this.historyStack.length - 1] === normalized) { + this.historyStack[this.historyStack.length - 1] = completedUrl; + } + else if (this.historyStack[this.historyStack.length - 1] !== completedUrl) { + this.historyStack.push(completedUrl); + } + this.currentPageLoadingStatus = null; + if (!this.human_current_page) { + this.human_current_page = page; + } + this._recordRecentEvent('tab_created', { + url: completedUrl, + page_id: newTab.page_id, + tab_id: newTab.tab_id, + }); + this._recordRecentEvent('tab_ready', { + url: completedUrl, + page_id: newTab.page_id, + tab_id: newTab.tab_id, + }); + await this.event_bus.dispatch(new TabCreatedEvent({ + target_id: newTab.target_id ?? newTab.tab_id ?? 'unknown_target', + url: completedUrl, + })); + this.cachedBrowserState = null; + return this.agent_current_page; + } + _resolveTabIndex(identifier) { + if (typeof identifier === 'string') { + const normalized = identifier.trim(); + if (!normalized) { + return -1; + } + if (normalized === '-1') { + return Math.max(0, this._tabs.length - 1); + } + const byTabId = this._tabs.findIndex((tab) => tab.tab_id === normalized); + if (byTabId !== -1) { + return byTabId; + } + const byTargetId = this._tabs.findIndex((tab) => tab.target_id === normalized); + if (byTargetId !== -1) { + return byTargetId; + } + const numeric = Number.parseInt(normalized, 10); + if (Number.isFinite(numeric)) { + return this._resolveTabIndex(numeric); + } + return -1; + } + if (identifier === -1) { + return Math.max(0, this._tabs.length - 1); + } + const byId = this._tabs.findIndex((tab) => tab.page_id === identifier); + if (byId !== -1) { + return byId; + } + if (identifier >= 0 && identifier < this._tabs.length) { + return identifier; + } + return -1; + } + async switch_to_tab(identifier, options = {}) { + const signal = options.signal ?? null; + this._throwIfAborted(signal); + this._syncTabsWithBrowserPages(); + const index = this._resolveTabIndex(identifier); + const tab = index >= 0 ? (this._tabs[index] ?? null) : null; + if (!tab) { + throw new Error(`Tab '${identifier}' does not exist`); + } + if (!tab.tab_id) { + tab.tab_id = this._formatTabId(tab.page_id); + } + this.currentTabIndex = index; + this.currentUrl = tab.url; + this.currentTitle = tab.title; + this._syncSessionManagerFromTabs(); + const page = this.tabPages.get(tab.page_id) ?? null; + this._setActivePage(page); + if (page?.bringToFront) { + try { + await this._withAbort(page.bringToFront(), signal); + } + catch (error) { + if (this._isAbortError(error)) { + throw error; + } + this.logger.debug(`Failed to focus tab: ${error.message}`); + } + } + await this._waitForLoad(page, 5000, signal); + this._recordRecentEvent('tab_switched', { + url: tab.url, + page_id: tab.page_id, + tab_id: tab.tab_id, + }); + await this.event_bus.dispatch(new AgentFocusChangedEvent({ + target_id: tab.target_id ?? tab.tab_id, + url: tab.url, + })); + this.cachedBrowserState = null; + return page; + } + async close_tab(identifier) { + this._syncTabsWithBrowserPages(); + const index = this._resolveTabIndex(identifier); + if (index < 0 || index >= this._tabs.length) { + throw new Error(`Tab '${identifier}' does not exist`); + } + const closingTab = this._tabs[index]; + if (!closingTab.tab_id) { + closingTab.tab_id = this._formatTabId(closingTab.page_id); + } + const closingPage = this.tabPages.get(closingTab.page_id) ?? null; + if (closingPage?.close) { + try { + await closingPage.close(); + } + catch (error) { + this.logger.debug(`Failed to close page: ${error.message}`); + } + } + this.tabPages.delete(closingTab.page_id); + this._recordRecentEvent('tab_closed', { + url: closingTab.url, + page_id: closingTab.page_id, + tab_id: closingTab.tab_id, + }); + this._tabs.splice(index, 1); + if (this.currentTabIndex >= this._tabs.length) { + this.currentTabIndex = Math.max(0, this._tabs.length - 1); + } + this._syncSessionManagerFromTabs(); + const tab = this._tabs[this.currentTabIndex] ?? null; + const current = tab ? (this.tabPages.get(tab.page_id) ?? null) : null; + this._setActivePage(current); + this.currentPageLoadingStatus = null; + this.cachedBrowserState = null; + if (this._tabs.length) { + const tab = this._tabs[this.currentTabIndex]; + this.currentUrl = tab.url; + this.currentTitle = tab.title; + await this.event_bus.dispatch(new AgentFocusChangedEvent({ + target_id: tab.target_id ?? tab.tab_id ?? 'unknown_target', + url: tab.url, + })); + } + else { + this.currentUrl = 'about:blank'; + this.currentTitle = 'about:blank'; + this._setActivePage(null); + } + await this.event_bus.dispatch(new TabClosedEvent({ + target_id: closingTab.target_id ?? closingTab.tab_id, + })); + } + async wait(seconds, options = {}) { + const signal = options.signal ?? null; + this._throwIfAborted(signal); + const boundedSeconds = Math.max(Number(seconds) || 0, 0); + const delayMs = boundedSeconds * 1000; + if (delayMs <= 0) { + return; + } + await this._waitWithAbort(delayMs, signal); + } + async send_keys(keys, options = {}) { + const signal = options.signal ?? null; + this._throwIfAborted(signal); + const page = await this._withAbort(this.get_current_page(), signal); + const keyboard = page?.keyboard; + if (!keyboard) { + throw new BrowserError('Keyboard input is not available on the current page.'); + } + try { + await this._withAbort(keyboard.press(keys), signal); + } + catch (error) { + if (error instanceof Error && error.message.includes('Unknown key')) { + for (const char of keys) { + await this._withAbort(keyboard.press(char), signal); + } + return; + } + throw error; + } + } + async click_coordinates(coordinate_x, coordinate_y, options = {}) { + const signal = options.signal ?? null; + this._throwIfAborted(signal); + const page = await this._withAbort(this.get_current_page(), signal); + if (!page?.mouse?.click) { + throw new BrowserError('Unable to perform coordinate click on the current page.'); + } + await this._withAbort(page.mouse.click(coordinate_x, coordinate_y, { + button: options.button ?? 'left', + }), signal); + } + async scroll(direction, amount, options = {}) { + const signal = options.signal ?? null; + this._throwIfAborted(signal); + const normalizedAmount = Math.max(Math.floor(Math.abs(amount)), 0); + if (normalizedAmount === 0) { + return; + } + const page = await this._withAbort(this.get_current_page(), signal); + if (!page?.evaluate) { + throw new BrowserError('Unable to access current page for scrolling.'); + } + const node = options.node ?? null; + if (node?.xpath) { + const scrolled = await this._withAbort(page.evaluate((payload) => { + const root = document.evaluate(payload.xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + if (!root) { + return false; + } + const topDelta = payload.direction === 'up' + ? -payload.amount + : payload.direction === 'down' + ? payload.amount + : 0; + const leftDelta = payload.direction === 'left' + ? -payload.amount + : payload.direction === 'right' + ? payload.amount + : 0; + root.scrollBy({ + top: topDelta, + left: leftDelta, + behavior: 'auto', + }); + return true; + }, { xpath: node.xpath, direction, amount: normalizedAmount }), signal); + if (scrolled) { + return; + } + } + if (direction === 'up' || direction === 'down') { + const pixels = direction === 'down' ? -normalizedAmount : normalizedAmount; + await this._withAbort(this._scrollContainer(pixels), signal); + return; + } + const horizontalDelta = direction === 'left' ? -normalizedAmount : normalizedAmount; + await this._withAbort(page.evaluate((x) => window.scrollBy(x, 0), horizontalDelta), signal); + } + async scroll_to_text(text, options = {}) { + const signal = options.signal ?? null; + this._throwIfAborted(signal); + const page = await this._withAbort(this.get_current_page(), signal); + if (!page?.evaluate) { + throw new BrowserError('Unable to access page for scrolling.'); + } + const success = await this._withAbort(page.evaluate((payload) => { + const query = payload.text.toLowerCase(); + const iterator = document.createNodeIterator(document.body, NodeFilter.SHOW_ELEMENT); + let node; + while ((node = iterator.nextNode())) { + const el = node; + if (!el || !el.textContent) { + continue; + } + if (el.textContent.toLowerCase().includes(query)) { + el.scrollIntoView({ + behavior: 'smooth', + block: payload.direction === 'up' ? 'start' : 'center', + }); + return true; + } + } + return false; + }, { text, direction: options.direction ?? 'down' }), signal); + if (!success) { + throw new BrowserError(`Text '${text}' not found on page`); + } + } + async get_dropdown_options(element_node, options = {}) { + const signal = options.signal ?? null; + this._throwIfAborted(signal); + const page = await this._withAbort(this.get_current_page(), signal); + if (!page?.evaluate) { + throw new BrowserError('Unable to evaluate dropdown options on current page.'); + } + if (!element_node?.xpath) { + throw new BrowserError('DOM element does not include an XPath selector.'); + } + const payload = await this._withAbort(page.evaluate(({ xpath }) => { + const element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + if (!element) + return null; + if (element.tagName?.toLowerCase() === 'select') { + const options = Array.from(element.options).map((opt, index) => ({ + text: opt.textContent?.trim() ?? '', + value: (opt.value ?? '').trim(), + index, + })); + return { type: 'select', options }; + } + const ariaRoles = new Set(['menu', 'listbox', 'combobox']); + const role = element.getAttribute('role'); + if (role && ariaRoles.has(role)) { + const nodes = element.querySelectorAll('[role="menuitem"],[role="option"]'); + const options = Array.from(nodes).map((node, index) => ({ + text: node.textContent?.trim() ?? '', + value: node.textContent?.trim() ?? '', + index, + })); + return { type: 'aria', options }; + } + return null; + }, { xpath: element_node.xpath }), signal); + if (!payload || !Array.isArray(payload.options)) { + throw new BrowserError('No options found for the specified dropdown.'); + } + const normalizedOptions = payload.options + .map((option, index) => ({ + index: typeof option?.index === 'number' && Number.isFinite(option.index) + ? option.index + : index, + text: String(option?.text ?? ''), + value: String(option?.value ?? ''), + })) + .filter((option) => option.text.length > 0 || option.value.length > 0); + if (normalizedOptions.length === 0) { + throw new BrowserError('No options found for the specified dropdown.'); + } + const formattedOptions = normalizedOptions.map((option) => `${option.index}: text=${JSON.stringify(option.text)}, value=${JSON.stringify(option.value)}`); + formattedOptions.push('Prefer exact text first; if needed select_dropdown_option also supports case-insensitive text/value matching.'); + const message = formattedOptions.join('\n'); + const indexForMemory = element_node.highlight_index ?? 'unknown'; + return { + type: String(payload.type ?? 'unknown'), + options: JSON.stringify(normalizedOptions), + formatted_options: formattedOptions.join('\n'), + message, + short_term_memory: message, + long_term_memory: `Found dropdown options for index ${indexForMemory}.`, + }; + } + async select_dropdown_option(element_node, text, options = {}) { + const signal = options.signal ?? null; + this._throwIfAborted(signal); + if (!element_node?.xpath) { + throw new BrowserError('DOM element does not include an XPath selector.'); + } + const page = await this._withAbort(this.get_current_page(), signal); + if (!page) { + throw new BrowserError('No active page for selection.'); + } + const formatAvailableOptions = (opts) => opts + .map((opt) => ` - [${opt.index}] text=${JSON.stringify(opt.text)} value=${JSON.stringify(opt.value)}`) + .join('\n'); + const pageFrames = (() => { + const framesAccessor = page.frames; + if (typeof framesAccessor === 'function') { + try { + const result = framesAccessor.call(page); + return Array.isArray(result) ? result : []; + } + catch { + return []; + } + } + return Array.isArray(framesAccessor) ? framesAccessor : []; + })(); + for (const frame of pageFrames) { + try { + const typeInfo = await this._withAbort(frame.evaluate((xpath) => { + const element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + if (!element) + return { found: false }; + const tagName = element.tagName?.toLowerCase(); + const role = element.getAttribute?.('role'); + if (tagName === 'select') + return { found: true, type: 'select' }; + if (role && ['menu', 'listbox', 'combobox'].includes(role)) + return { found: true, type: 'aria' }; + return { found: false }; + }, element_node.xpath), signal); + if (!typeInfo?.found) { + continue; + } + if (typeInfo.type === 'select') { + const selection = await this._withAbort(frame.evaluate(({ xpath, optionText, }) => { + const root = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + if (!root || root.tagName?.toLowerCase() !== 'select') { + return { found: false }; + } + const options = Array.from(root.options).map((opt, index) => ({ + index, + text: opt.textContent?.trim() ?? '', + value: (opt.value ?? '').trim(), + })); + const targetRaw = optionText.trim(); + const targetLower = optionText.trim().toLowerCase(); + let matchedIndex = options.findIndex((opt) => opt.text === targetRaw || opt.value === targetRaw); + if (matchedIndex < 0) { + matchedIndex = options.findIndex((opt) => opt.text.trim().toLowerCase() === targetLower || + opt.value.trim().toLowerCase() === targetLower); + } + if (matchedIndex < 0) { + return { found: true, success: false, options }; + } + const matched = options[matchedIndex]; + root.value = matched.value; + root.dispatchEvent(new Event('input', { bubbles: true })); + root.dispatchEvent(new Event('change', { bubbles: true })); + const selectedOption = root.selectedIndex >= 0 + ? root.options[root.selectedIndex] + : null; + const selectedText = selectedOption?.textContent?.trim() ?? ''; + const selectedValue = (root.value ?? '').trim(); + const selectedValueLower = selectedValue.trim().toLowerCase(); + const selectedTextLower = selectedText.trim().toLowerCase(); + const matchedValueLower = String(matched.value ?? '') + .trim() + .toLowerCase(); + const matchedTextLower = String(matched.text ?? '') + .trim() + .toLowerCase(); + const verified = selectedValueLower === matchedValueLower || + selectedTextLower === matchedTextLower; + return { + found: true, + success: verified, + options, + selectedText, + selectedValue, + matched, + }; + }, { xpath: element_node.xpath, optionText: text }), signal); + if (selection?.found && selection.success) { + const matchedText = selection.matched?.text ?? text; + const matchedValue = selection.matched?.value ?? ''; + const msg = `Selected option ${matchedText} (${matchedValue})`; + return { + message: msg, + short_term_memory: msg, + long_term_memory: msg, + matched_text: String(matchedText), + matched_value: String(matchedValue), + }; + } + if (selection?.found) { + const details = formatAvailableOptions(selection.options ?? []); + throw new BrowserError(`Could not select option '${text}' for index ${element_node.highlight_index ?? 'unknown'}.\nAvailable options:\n${details}`); + } + continue; + } + const clicked = await this._withAbort(frame.evaluate(({ xpath, optionText }) => { + const root = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + if (!root) + return false; + const nodes = root.querySelectorAll('[role="menuitem"],[role="option"]'); + const options = Array.from(nodes).map((node, index) => ({ + index, + text: node.textContent?.trim() ?? '', + value: node.textContent?.trim() ?? '', + })); + const targetRaw = optionText.trim(); + const targetLower = optionText.trim().toLowerCase(); + let matchedIndex = options.findIndex((opt) => opt.text === targetRaw || opt.value === targetRaw); + if (matchedIndex < 0) { + matchedIndex = options.findIndex((opt) => opt.text.trim().toLowerCase() === targetLower || + opt.value.trim().toLowerCase() === targetLower); + } + if (matchedIndex < 0) { + return { found: true, success: false, options }; + } + nodes[matchedIndex].click(); + return { + found: true, + success: true, + options, + matched: options[matchedIndex], + }; + }, { xpath: element_node.xpath, optionText: text }), signal); + if (clicked?.found && clicked.success) { + const matchedText = clicked.matched?.text ?? text; + const msg = `Selected menu item ${matchedText}`; + return { + message: msg, + short_term_memory: msg, + long_term_memory: msg, + matched_text: String(matchedText), + }; + } + if (clicked?.found) { + const details = formatAvailableOptions(clicked.options ?? []); + throw new BrowserError(`Could not select option '${text}' for index ${element_node.highlight_index ?? 'unknown'}.\nAvailable options:\n${details}`); + } + } + catch (error) { + if (error instanceof BrowserError) { + throw error; + } + continue; + } + } + throw new BrowserError(`Could not select option '${text}' for index ${element_node.highlight_index ?? 'unknown'}`); + } + async upload_file(element_node, file_path, options = {}) { + const signal = options.signal ?? null; + this._throwIfAborted(signal); + const locator = await this.get_locate_element(element_node); + if (!locator) { + throw new Error('Element not found'); + } + if (!fs.existsSync(file_path)) { + throw new Error(`File does not exist: ${file_path}`); + } + const locatorWithUpload = locator; + if (typeof locatorWithUpload.setInputFiles !== 'function') { + throw new Error('Element does not support file upload'); + } + await this._withAbort(locatorWithUpload.setInputFiles(file_path, { timeout: 5000 }), signal); + } + async go_back(options = {}) { + const signal = options.signal ?? null; + this._throwIfAborted(signal); + const page = await this._withAbort(this.get_current_page(), signal); + if (!page?.goBack) { + return; + } + const previousUrl = this.currentUrl; + try { + await this._withAbort(page.goBack(), signal); + } + catch (error) { + if (this._isAbortError(error)) { + throw error; + } + this.logger.debug(`Failed to navigate back: ${error.message}`); + } + this._throwIfAborted(signal); + await this._syncCurrentTabFromPage(page); + const currentUrl = this.currentUrl; + if (currentUrl && currentUrl !== previousUrl) { + const existingIndex = this.historyStack.lastIndexOf(currentUrl); + if (existingIndex !== -1) { + this.historyStack = this.historyStack.slice(0, existingIndex + 1); + } + else if (this.historyStack[this.historyStack.length - 1] !== currentUrl) { + this.historyStack.push(currentUrl); + } + } + this.cachedBrowserState = null; + this._recordRecentEvent('navigation_back', { url: currentUrl }); + } + async get_dom_element_by_index(_index, options = {}) { + const selectorMap = await this.get_selector_map(options); + return selectorMap?.[_index] ?? null; + } + set_downloaded_files(files) { + if (!Array.isArray(files)) { + return; + } + this.downloaded_files = [...files]; + } + add_downloaded_file(filePath) { + if (!filePath) { + return; + } + if (!this.downloaded_files.includes(filePath)) { + this.downloaded_files = [...this.downloaded_files, filePath]; + this.logger.info(`📁 Added download to session tracking (total: ${this.downloaded_files.length} files)`); + } + } + get_downloaded_files() { + this.logger.debug(`📁 Retrieved ${this.downloaded_files.length} downloaded files from session tracking`); + return [...this.downloaded_files]; + } + set_auto_download_pdfs(enabled) { + this._autoDownloadPdfs = Boolean(enabled); + this.logger.info(`📄 PDF auto-download ${this._autoDownloadPdfs ? 'enabled' : 'disabled'}`); + } + auto_download_pdfs() { + return this._autoDownloadPdfs; + } + static async get_unique_filename(directory, filename) { + const resolvedDir = path.resolve(directory); + const parsed = path.parse(filename); + let candidate = filename; + let counter = 1; + while (fs.existsSync(path.join(resolvedDir, candidate))) { + candidate = `${parsed.name} (${counter})${parsed.ext}`; + counter += 1; + } + return candidate; + } + async get_selector_map(options = {}) { + if (!this.cachedBrowserState) { + await this.get_browser_state_with_recovery({ + cache_clickable_elements_hashes: true, + include_screenshot: false, + signal: options.signal ?? null, + }); + } + return this.cachedBrowserState?.selector_map ?? {}; + } + static is_file_input(node) { + if (!node) { + return false; + } + return (node.tag_name?.toLowerCase() === 'input' && + (node.attributes?.type ?? '').toLowerCase() === 'file'); + } + is_file_input(node) { + return BrowserSession.is_file_input(node); + } + async find_file_upload_element_by_index(index, maxHeight = 3, maxDescendantDepth = 3, options = {}) { + const selectorMap = await this.get_selector_map(options); + const root = selectorMap[index]; + if (!root) { + return null; + } + const findInDescendants = (node, depth) => { + if (depth < 0) { + return null; + } + if (BrowserSession.is_file_input(node)) { + return node; + } + for (const child of node.children) { + if (child instanceof DOMElementNode) { + const found = findInDescendants(child, depth - 1); + if (found) { + return found; + } + } + } + return null; + }; + let current = root; + let remainingHeight = maxHeight; + while (current && remainingHeight >= 0) { + const direct = findInDescendants(current, maxDescendantDepth); + if (direct) { + return direct; + } + if (current.parent) { + for (const sibling of current.parent.children) { + if (sibling instanceof DOMElementNode && sibling !== current) { + const fromSibling = findInDescendants(sibling, maxDescendantDepth); + if (fromSibling) { + return fromSibling; + } + } + } + } + current = current.parent; + remainingHeight -= 1; + } + return null; + } + async get_locate_element(node) { + const page = await this.get_current_page(); + if (!page || !node?.xpath) { + return null; + } + try { + const locator = page.locator(`xpath=${node.xpath}`); + const count = await locator.count(); + if (count === 0) { + return null; + } + return locator; + } + catch (error) { + this.logger.debug(`Failed to locate element via xpath ${node.xpath}: ${error.message}`); + return null; + } + } + async _input_text_element_node(node, text, options = {}) { + const signal = options.signal ?? null; + const clear = options.clear ?? true; + this._throwIfAborted(signal); + const locator = await this.get_locate_element(node); + if (!locator) { + throw new Error('Element not found'); + } + await this._withAbort(locator.click({ timeout: 5000 }), signal); + if (clear) { + await this._withAbort(locator.fill(text, { timeout: 5000 }), signal); + } + else { + await this._withAbort(locator.type(text, { timeout: 5000 }), signal); + } + } + async _click_element_node(node, options = {}) { + const signal = options.signal ?? null; + this._throwIfAborted(signal); + const locator = await this.get_locate_element(node); + if (!locator) { + throw new Error('Element not found'); + } + const page = await this._withAbort(this.get_current_page(), signal); + const performClick = async () => { + await this._withAbort(locator.click({ timeout: 5000 }), signal); + }; + const downloadsDir = this.browser_profile.downloads_path; + if (downloadsDir && page?.waitForEvent) { + const downloadPromise = page.waitForEvent('download', { timeout: 5000 }); + await performClick(); + try { + const download = await this._withAbort(downloadPromise, signal); + const downloadGuid = uuid7str(); + const suggested = typeof download.suggestedFilename === 'function' + ? download.suggestedFilename() + : 'download'; + const downloadUrl = typeof download.url === 'function' + ? download.url() + : (this.currentUrl ?? ''); + await this.event_bus.dispatch(new DownloadStartedEvent({ + guid: downloadGuid, + url: downloadUrl, + suggested_filename: suggested, + auto_download: false, + })); + const uniqueFilename = await BrowserSession.get_unique_filename(downloadsDir, suggested); + const downloadPath = path.join(downloadsDir, uniqueFilename); + if (typeof download.saveAs === 'function') { + await download.saveAs(downloadPath); + } + const stats = fs.existsSync(downloadPath) + ? fs.statSync(downloadPath) + : null; + await this.event_bus.dispatch(new DownloadProgressEvent({ + guid: downloadGuid, + received_bytes: stats?.size ?? 0, + total_bytes: stats?.size ?? 0, + state: 'completed', + })); + const fileDownloadedResult = await this.event_bus.dispatch(new FileDownloadedEvent({ + guid: downloadGuid, + url: downloadUrl, + path: downloadPath, + file_name: uniqueFilename, + file_size: stats?.size ?? 0, + file_type: path.extname(uniqueFilename).replace('.', '') || null, + mime_type: null, + auto_download: false, + })); + if (fileDownloadedResult.handler_results.length === 0) { + this.add_downloaded_file(downloadPath); + } + return downloadPath; + } + catch (error) { + if (this._isAbortError(error)) { + throw error; + } + this.logger.debug(`No download triggered within timeout: ${error.message}`); + } + } + else { + await performClick(); + } + await this._waitForLoad(page, 5000, signal); + if (page) { + await this._syncCurrentTabFromPage(page); + if (this.historyStack[this.historyStack.length - 1] !== this.currentUrl) { + this.historyStack.push(this.currentUrl); + } + } + this.cachedBrowserState = null; + return null; + } + async _waitForLoad(page, timeout = 5000, signal = null) { + if (!page || typeof page.waitForLoadState !== 'function') { + return; + } + try { + await this._withAbort(page.waitForLoadState('domcontentloaded', { timeout }), signal); + } + catch (error) { + if (this._isAbortError(error)) { + throw error; + } + this.logger.debug(`waitForLoadState failed: ${error.message}`); + } + } + // ==================== Cookie Management ==================== + /** + * Get all cookies from the current browser context + */ + async get_cookies() { + if (this.browser_context?.cookies) { + return await this.browser_context.cookies(); + } + return []; + } + /** + * Save cookies to a file (deprecated, use save_storage_state instead) + * @deprecated Use save_storage_state() instead + */ + async save_cookies(...args) { + return this.save_storage_state(...args); + } + /** + * Load cookies from a file (deprecated, use load_storage_state instead) + * @deprecated Use load_storage_state() instead + */ + async load_cookies_from_file(...args) { + return this.load_storage_state(...args); + } + /** + * Save the current storage state (cookies, localStorage, sessionStorage) to a file + */ + async save_storage_state(filePath) { + if (!this.browser_context) { + this.logger.warning('Cannot save storage state: browser context not initialized'); + return; + } + const targetPath = filePath || this.browser_profile.cookies_file; + if (!targetPath) { + return; + } + try { + const resolvedPath = path.resolve(targetPath); + const dirPath = path.dirname(resolvedPath); + // Create directory if it doesn't exist + if (!fs.existsSync(dirPath)) { + fs.mkdirSync(dirPath, { recursive: true }); + } + // Get storage state from browser context + const storageState = await this.browser_context.storageState(); + // Write to temporary file first + const tempPath = `${resolvedPath}.tmp`; + fs.writeFileSync(tempPath, JSON.stringify(storageState, null, 2)); + // Backup existing file if present + if (fs.existsSync(resolvedPath)) { + const backupPath = `${resolvedPath}.bak`; + try { + fs.renameSync(resolvedPath, backupPath); + } + catch (error) { + // Ignore backup errors + } + } + // Move temp file to target + fs.renameSync(tempPath, resolvedPath); + const cookieCount = storageState.cookies?.length || 0; + this.logger.info(`🍪 Saved ${cookieCount} cookies to ${path.basename(resolvedPath)}`); + } + catch (error) { + this.logger.warning(`❌ Failed to save storage state: ${error.message}`); + } + } + /** + * Load storage state (cookies, localStorage, sessionStorage) from a file + */ + async load_storage_state(filePath) { + const targetPath = filePath || this.browser_profile.cookies_file; + if (!targetPath) { + return; + } + try { + const resolvedPath = path.resolve(targetPath); + if (!fs.existsSync(resolvedPath)) { + this.logger.warning(`Storage state file not found: ${resolvedPath}`); + return; + } + const storageStateContent = fs.readFileSync(resolvedPath, 'utf-8'); + const storageState = JSON.parse(storageStateContent); + if (this.browser_context?.addCookies) { + // Add cookies to context + if (storageState.cookies && Array.isArray(storageState.cookies)) { + await this.browser_context.addCookies(storageState.cookies); + this.logger.info(`🍪 Loaded ${storageState.cookies.length} cookies from ${path.basename(resolvedPath)}`); + } + } + } + catch (error) { + this.logger.warning(`❌ Failed to load storage state: ${error.message}`); + } + } + // ==================== JavaScript Execution ==================== + /** + * Execute JavaScript in the current page context + */ + async execute_javascript(script) { + const page = await this.get_current_page(); + if (!page) { + throw new Error('No page available to execute JavaScript'); + } + return await page.evaluate(script); + } + // ==================== Page Information ==================== + /** + * Get comprehensive page information (size, scroll position, etc.) + */ + async get_page_info(page) { + const targetPage = page || (await this.get_current_page()); + if (!targetPage) { + return null; + } + const pageData = await targetPage.evaluate(() => { + return { + // Current viewport dimensions + viewport_width: window.innerWidth, + viewport_height: window.innerHeight, + // Total page dimensions + page_width: Math.max(document.documentElement.scrollWidth, document.body.scrollWidth || 0), + page_height: Math.max(document.documentElement.scrollHeight, document.body.scrollHeight || 0), + // Current scroll position + scroll_x: window.scrollX || + window.pageXOffset || + document.documentElement.scrollLeft || + 0, + scroll_y: window.scrollY || + window.pageYOffset || + document.documentElement.scrollTop || + 0, + }; + }); + // Calculate derived values + const viewport_width = Math.floor(pageData.viewport_width); + const viewport_height = Math.floor(pageData.viewport_height); + const page_width = Math.floor(pageData.page_width); + const page_height = Math.floor(pageData.page_height); + const scroll_x = Math.floor(pageData.scroll_x); + const scroll_y = Math.floor(pageData.scroll_y); + // Calculate scroll information + const pixels_above = scroll_y; + const pixels_below = Math.max(0, page_height - (scroll_y + viewport_height)); + const pixels_left = scroll_x; + const pixels_right = Math.max(0, page_width - (scroll_x + viewport_width)); + return { + viewport_width, + viewport_height, + page_width, + page_height, + scroll_x, + scroll_y, + pixels_above, + pixels_below, + pixels_left, + pixels_right, + }; + } + /** + * Get the HTML content of the current page + */ + async get_page_html() { + const page = await this.get_current_page(); + if (!page) { + return ''; + } + return await page.content(); + } + /** + * Get a debug view of the page structure including iframes + */ + async get_page_structure() { + const page = await this.get_current_page(); + if (!page) { + return ''; + } + const debug_script = `(() => { + function getPageStructure(element = document, depth = 0, maxDepth = 10) { + if (depth >= maxDepth) return ''; + + const indent = ' '.repeat(depth); + let structure = ''; + + // Skip certain elements that clutter the output + const skipTags = new Set(['script', 'style', 'link', 'meta', 'noscript']); + + // Add current element info if it's not the document + if (element !== document) { + const tagName = element.tagName.toLowerCase(); + + // Skip uninteresting elements + if (skipTags.has(tagName)) return ''; + + const id = element.id ? \`#\${element.id}\` : ''; + const classes = element.className && typeof element.className === 'string' ? + \`.\${element.className.split(' ').filter(c => c).join('.')}\` : ''; + + // Get additional useful attributes + const attrs = []; + if (element.getAttribute('role')) attrs.push(\`role="\${element.getAttribute('role')}"\`); + if (element.getAttribute('aria-label')) attrs.push(\`aria-label="\${element.getAttribute('aria-label')}"\`); + if (element.getAttribute('type')) attrs.push(\`type="\${element.getAttribute('type')}"\`); + if (element.getAttribute('name')) attrs.push(\`name="\${element.getAttribute('name')}"\`); + if (element.getAttribute('src')) { + const src = element.getAttribute('src'); + attrs.push(\`src="\${src.substring(0, 50)}\${src.length > 50 ? '...' : ''}"\`); + } + + // Add element info + structure += \`\${indent}\${tagName}\${id}\${classes}\${attrs.length ? ' [' + attrs.join(', ') + ']' : ''}\\n\`; + + // Handle iframes specially + if (tagName === 'iframe') { + try { + const iframeDoc = element.contentDocument || element.contentWindow?.document; + if (iframeDoc) { + structure += \`\${indent} [IFRAME CONTENT]:\\n\`; + structure += getPageStructure(iframeDoc, depth + 2, maxDepth); + } else { + structure += \`\${indent} [CROSS-ORIGIN IFRAME - Cannot access]\\n\`; + } + } catch (e) { + structure += \`\${indent} [IFRAME - Access denied]\\n\`; + } + return structure; + } + } + + // Process children + const children = element.children || element.documentElement?.children || []; + for (let i = 0; i < children.length; i++) { + structure += getPageStructure(children[i], depth + 1, maxDepth); + } + + return structure; + } + + return getPageStructure(); + })()`; + return await page.evaluate(debug_script); + } + // ==================== Navigation & History ==================== + /** + * Navigate forward in browser history + */ + async go_forward() { + try { + const page = await this.get_current_page(); + if (page?.goForward) { + await page.goForward({ timeout: 10000, waitUntil: 'load' }); + } + } + catch (error) { + this.logger.debug(`⏭️ Error during go_forward: ${error.message}`); + // Verify page is still usable after navigation error + if (error.message.toLowerCase().includes('timeout')) { + const page = await this.get_current_page(); + try { + await page?.evaluate('1'); + } + catch (evalError) { + this.logger.error(`❌ Page crashed after go_forward timeout: ${evalError.message}`); + } + } + } + } + /** + * Refresh the current page + */ + async refresh() { + try { + const page = await this.get_current_page(); + if (page?.reload) { + this.currentPageLoadingStatus = null; + await page.reload({ waitUntil: 'domcontentloaded' }); + await this._waitForStableNetwork(page); + } + } + catch (error) { + this.logger.debug(`🔄 Error during refresh: ${error.message}`); + } + } + // ==================== Element Waiting ==================== + /** + * Wait for an element to appear on the page + */ + async wait_for_element(selector, timeout = 10000) { + const page = await this.get_current_page(); + if (!page) { + throw new Error('No page available'); + } + await page.waitForSelector(selector, { state: 'visible', timeout }); + } + // ==================== Screenshots ==================== + /** + * Take a screenshot of the current page. + * @param full_page Whether to capture the full scrollable page + * @param clip Optional clip region for partial screenshots + * @returns Base64 encoded PNG screenshot + */ + async take_screenshot(full_page = false, clip = null) { + const page = await this.get_current_page(); + if (!page) { + throw new Error('No page available for screenshot'); + } + if (!this.browser_context) { + throw new Error('Browser context is not set'); + } + // Check if it's a new tab page + const url = page.url(); + if (url === 'about:blank' || + url === 'chrome://newtab/' || + url === 'edge://newtab/') { + this.logger.warning(`▫️ Skipping screenshot of empty page: ${url}`); + // Return a 4px placeholder + return 'iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAYAAACp8Z5+AAAAD0lEQVQIHWP8//8/AxYMACgtBP9g8jqYAAAAAElFTkSuQmCC'; + } + // Bring page to front before rendering + try { + await page.bringToFront(); + } + catch (error) { + // Ignore errors + } + // Take screenshot using CDP for better performance + let cdp_session = null; + try { + this.logger.debug(`📸 Taking ${full_page ? 'full-page' : 'viewport'} PNG screenshot via CDP: ${url}`); + // Create CDP session for the screenshot + cdp_session = await this.get_or_create_cdp_session(page); + // Capture screenshot via CDP + const screenshotParams = { + captureBeyondViewport: full_page, + fromSurface: true, + format: 'png', + }; + if (clip) { + screenshotParams.clip = { + x: clip.x, + y: clip.y, + width: clip.width, + height: clip.height, + scale: 1, + }; + } + const screenshot_response = await cdp_session.send('Page.captureScreenshot', screenshotParams); + const screenshot_b64 = screenshot_response.data; + if (!screenshot_b64) { + throw new Error(`CDP returned empty screenshot data for page ${url}`); + } + return screenshot_b64; + } + catch (error) { + const error_str = error.message || String(error); + if (error_str.toLowerCase().includes('timeout')) { + this.logger.warning(`⏱️ Screenshot timed out on page ${url}: ${error_str}`); + } + else { + this.logger.error(`❌ Screenshot failed on page ${url}: ${error_str}`); + } + throw error; + } + finally { + if (cdp_session) { + try { + await cdp_session.detach(); + } + catch (error) { + // Ignore detach errors + } + } + } + } + // ==================== Event Listeners ==================== + /** + * Add a request event listener to the current page + */ + async on_request(callback) { + const page = await this.get_current_page(); + if (page && typeof page.on === 'function') { + page.on('request', callback); + } + } + /** + * Add a response event listener to the current page + */ + async on_response(callback) { + const page = await this.get_current_page(); + if (page && typeof page.on === 'function') { + page.on('response', callback); + } + } + /** + * Remove a request event listener from the current page + */ + async off_request(callback) { + const page = await this.get_current_page(); + if (page && typeof page.off === 'function') { + page.off('request', callback); + } + } + /** + * Remove a response event listener from the current page + */ + async off_response(callback) { + const page = await this.get_current_page(); + if (page && typeof page.off === 'function') { + page.off('response', callback); + } + } + // ==================== P2 Additional Functions ==================== + /** + * Get information about all open tabs + * @returns Array of tab information including page_id, tab_id, url, and title + */ + async get_tabs_info() { + if (!this.browser_context) { + return []; + } + this._syncTabsWithBrowserPages(); + const tabs_info = []; + for (const tab of this._tabs) { + const page_id = tab.page_id; + const page = this.tabPages.get(page_id) ?? null; + const tab_id = tab.tab_id || this._formatTabId(page_id); + if (!tab.tab_id) { + tab.tab_id = tab_id; + } + this._attachDialogHandler(page); + let currentUrl = tab.url; + if (page?.url) { + try { + currentUrl = normalize_url(page.url()); + } + catch { + // Keep tab url fallback when page url is unavailable. + } + } + // Skip chrome:// pages and new tab pages + const isNewTab = currentUrl === 'about:blank' || + currentUrl.startsWith('chrome://newtab'); + if (isNewTab || currentUrl.startsWith('chrome://')) { + if (isNewTab) { + tabs_info.push({ + page_id, + tab_id, + url: currentUrl, + title: 'ignore this tab and do not use it', + }); + } + else { + tabs_info.push({ + page_id, + tab_id, + url: currentUrl, + title: currentUrl, + }); + } + continue; + } + // Normal pages - try to get title with timeout + try { + if (!page?.title) { + throw new Error('page_title_unavailable'); + } + const titlePromise = page.title(); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error('timeout')), 2000); + }); + const title = await Promise.race([titlePromise, timeoutPromise]); + tabs_info.push({ page_id, tab_id, url: currentUrl, title }); + } + catch (error) { + this.logger.debug(`⚠️ Failed to get tab info for tab #${page_id}: ${currentUrl} (using fallback title)`); + if (isNewTab) { + tabs_info.push({ + page_id, + tab_id, + url: currentUrl, + title: 'ignore this tab and do not use it', + }); + } + else { + tabs_info.push({ + page_id, + tab_id, + url: currentUrl, + title: tab.title || currentUrl, + }); + } + } + } + return tabs_info; + } + /** + * Check if a page is responsive by trying to evaluate simple JavaScript + * @param page - The page to check + * @param timeout - Timeout in seconds (default: 5) + * @returns True if page is responsive, false otherwise + */ + async _is_page_responsive(page, timeout = 5.0) { + try { + const evalPromise = page.evaluate('1'); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error('timeout')), timeout * 1000); + }); + await Promise.race([evalPromise, timeoutPromise]); + return true; + } + catch (error) { + return false; + } + } + /** + * Get scroll information for the current page + * @returns Object with scroll position and page dimensions + */ + async get_scroll_info() { + const page = await this.get_current_page(); + if (!page) { + return { + scroll_x: 0, + scroll_y: 0, + page_width: 0, + page_height: 0, + viewport_width: 0, + viewport_height: 0, + }; + } + return await page.evaluate(() => { + return { + scroll_x: window.scrollX || + window.pageXOffset || + document.documentElement.scrollLeft || + 0, + scroll_y: window.scrollY || + window.pageYOffset || + document.documentElement.scrollTop || + 0, + page_width: Math.max(document.documentElement.scrollWidth, document.body.scrollWidth || 0), + page_height: Math.max(document.documentElement.scrollHeight, document.body.scrollHeight || 0), + viewport_width: window.innerWidth, + viewport_height: window.innerHeight, + }; + }); + } + /** + * Get a summary of the current browser state + * @param cache_clickable_elements_hashes - Cache clickable element hashes to detect new elements + * @param include_screenshot - Include screenshot in state summary + * @returns BrowserStateSummary with current page state + */ + async get_state_summary(cache_clickable_elements_hashes = true, include_screenshot = true, include_recent_events = false) { + this.logger.debug('🔄 Starting get_state_summary...'); + const updated_state = await this._get_updated_state(-1, include_screenshot, include_recent_events); + // Implement clickable element hash caching to detect new elements + if (cache_clickable_elements_hashes) { + const page = await this.get_current_page(); + if (page) { + const currentUrl = page.url(); + const currentHashes = this._computeElementHashes(updated_state.selector_map); + // Mark new elements if we have cached hashes for this URL + if (this._cachedClickableElementHashes && + this._cachedClickableElementHashes.url === currentUrl) { + this._markNewElements(updated_state.selector_map, this._cachedClickableElementHashes.hashes); + } + // Update cache with current hashes + this._cachedClickableElementHashes = { + url: currentUrl, + hashes: currentHashes, + }; + } + } + this.cachedBrowserState = updated_state; + return this.cachedBrowserState; + } + /** + * Get minimal state summary without DOM processing, but with screenshot + * Used when page is in error state or unresponsive + */ + async get_minimal_state_summary(include_recent_events = false) { + try { + const page = await this.get_current_page(); + const url = page ? page.url() : 'unknown'; + // Try to get title safely + let title = 'Page Load Error'; + try { + if (page) { + const titlePromise = page.title(); + const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 2000)); + title = await Promise.race([titlePromise, timeoutPromise]); + } + } + catch (error) { + // Keep default title + } + // Try to get tabs info safely + let tabs_info = []; + try { + const tabsPromise = this.get_tabs_info(); + const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 2000)); + tabs_info = await Promise.race([tabsPromise, timeoutPromise]); + } + catch (error) { + // Keep empty tabs + } + // Create minimal DOM element for error state + const minimal_element_tree = new DOMElementNode(true, null, 'body', '/body', {}, []); + // Try to get screenshot + let screenshot_b64 = null; + try { + screenshot_b64 = await this.take_screenshot(); + } + catch (error) { + this.logger.debug(`Screenshot failed in minimal state: ${error.message}`); + } + // Use default viewport dimensions + const viewport = this.browser_profile.viewport || { + width: 1280, + height: 720, + }; + const dom_state = new DOMState(minimal_element_tree, {}); + this._original_viewport_size = [viewport.width, viewport.height]; + return new BrowserStateSummary(dom_state, { + url, + title, + tabs: tabs_info, + screenshot: screenshot_b64, + page_info: { + viewport_width: viewport.width, + viewport_height: viewport.height, + page_width: viewport.width, + page_height: viewport.height, + scroll_x: 0, + scroll_y: 0, + pixels_above: 0, + pixels_below: 0, + pixels_left: 0, + pixels_right: 0, + }, + pixels_above: 0, + pixels_below: 0, + browser_errors: ['Page in error state - minimal navigation available'], + is_pdf_viewer: false, + loading_status: this.currentPageLoadingStatus, + recent_events: include_recent_events + ? this._getRecentEventsSummary() + : null, + pending_network_requests: [], + pagination_buttons: [], + closed_popup_messages: this._getClosedPopupMessagesSnapshot(), + }); + } + catch (error) { + this.logger.error(`Failed to get minimal state summary: ${error.message}`); + throw error; + } + } + /** + * Internal method to get updated browser state with DOM processing + * @param focus_element - Element index to focus on (default: -1) + * @param include_screenshot - Whether to include screenshot + */ + async _get_updated_state(focus_element = -1, include_screenshot = true, include_recent_events = false) { + const page = await this.get_current_page(); + if (!page) { + throw new Error('No current page available'); + } + const page_url = page.url(); + // Check for new tab or chrome:// pages - fast path + const is_empty_page = this._is_new_tab_page(page_url) || page_url.startsWith('chrome://'); + if (is_empty_page) { + this.logger.debug(`⚡ Fast path for empty page: ${page_url}`); + // Create minimal DOM state + const minimal_element_tree = new DOMElementNode(false, null, 'body', '', {}, []); + const tabs_info = await this.get_tabs_info(); + const viewport = this.browser_profile.viewport || { + width: 1280, + height: 720, + }; + const dom_state = new DOMState(minimal_element_tree, {}); + this._original_viewport_size = [viewport.width, viewport.height]; + return new BrowserStateSummary(dom_state, { + url: page_url, + title: this._is_new_tab_page(page_url) ? 'New Tab' : 'Chrome Page', + tabs: tabs_info, + screenshot: null, + page_info: { + viewport_width: viewport.width, + viewport_height: viewport.height, + page_width: viewport.width, + page_height: viewport.height, + scroll_x: 0, + scroll_y: 0, + pixels_above: 0, + pixels_below: 0, + pixels_left: 0, + pixels_right: 0, + }, + pixels_above: 0, + pixels_below: 0, + browser_errors: [], + is_pdf_viewer: false, + loading_status: this.currentPageLoadingStatus, + recent_events: include_recent_events + ? this._getRecentEventsSummary() + : null, + pending_network_requests: [], + pagination_buttons: [], + closed_popup_messages: this._getClosedPopupMessagesSnapshot(), + }); + } + // Normal path for regular pages + this.logger.debug('🧹 Removing highlights...'); + try { + await this.remove_highlights(); + } + catch (error) { + this.logger.debug('Timeout removing highlights'); + } + // Check for PDF and auto-download if needed + try { + const pdf_path = await this._auto_download_pdf_if_needed(page); + if (pdf_path) { + this.logger.info(`📄 PDF auto-downloaded: ${pdf_path}`); + } + } + catch (error) { + this.logger.debug(`PDF auto-download check failed: ${error.message}`); + } + // DOM processing + this.logger.debug('🌳 Starting DOM processing...'); + const dom_service = new DomService(page, this.logger); + let content; + try { + const domPromise = dom_service.get_clickable_elements(this.browser_profile.highlight_elements, focus_element, this.browser_profile.viewport_expansion); + const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('DOM processing timeout')), 45000)); + content = await Promise.race([domPromise, timeoutPromise]); + this.logger.debug('✅ DOM processing completed'); + } + catch (error) { + this.logger.warning(`DOM processing timed out for ${page_url}`); + this.logger.warning('🔄 Falling back to minimal DOM state...'); + // Create minimal DOM state for fallback + const minimal_element_tree = new DOMElementNode(true, null, 'body', '/body', {}, []); + content = new DOMState(minimal_element_tree, {}); + } + // Get tabs info + this.logger.debug('📋 Getting tabs info...'); + const tabs_info = await this.get_tabs_info(); + this.logger.debug('✅ Tabs info completed'); + // Screenshot + let screenshot_b64 = null; + if (include_screenshot) { + try { + this.logger.debug('📸 Capturing screenshot...'); + screenshot_b64 = await this.take_screenshot(); + } + catch (error) { + this.logger.warning(`❌ Screenshot failed for ${page_url}: ${error.message}`); + } + } + // Get page info and scroll info + const page_info = await this.get_page_info(page); + if (page_info && + Number.isFinite(page_info.viewport_width) && + Number.isFinite(page_info.viewport_height)) { + this._original_viewport_size = [ + Math.floor(page_info.viewport_width), + Math.floor(page_info.viewport_height), + ]; + } + let pixels_above = 0; + let pixels_below = 0; + try { + this.logger.debug('📏 Getting scroll info...'); + const scroll_info = await Promise.race([ + this.get_scroll_info(), + new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 5000)), + ]); + // Calculate pixels above/below viewport + pixels_above = Math.max(0, scroll_info.scroll_y); + const viewport_bottom = scroll_info.scroll_y + scroll_info.viewport_height; + pixels_below = Math.max(0, scroll_info.page_height - viewport_bottom); + this.logger.debug('✅ Scroll info completed'); + } + catch (error) { + this.logger.warning(`Failed to get scroll info: ${error.message}`); + } + // Get title + let title = 'Title unavailable'; + try { + const titlePromise = page.title(); + const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 3000)); + title = await Promise.race([titlePromise, timeoutPromise]); + } + catch (error) { + // Keep default title + } + // Check for errors + const browser_errors = []; + if (Object.keys(content.selector_map).length === 0) { + browser_errors.push(`DOM processing timed out for ${page_url} - using minimal state. Basic navigation still available.`); + } + // Check if PDF viewer + const is_pdf_viewer = await this._is_pdf_viewer(page); + const pendingNetworkRequests = await this._getPendingNetworkRequests(page); + const paginationButtons = DomService.detect_pagination_buttons(content.selector_map); + const browser_state = new BrowserStateSummary(content, { + url: page_url, + title, + tabs: tabs_info, + screenshot: screenshot_b64, + page_info, + pixels_above, + pixels_below, + browser_errors, + is_pdf_viewer, + loading_status: this.currentPageLoadingStatus, + recent_events: include_recent_events + ? this._getRecentEventsSummary() + : null, + pending_network_requests: pendingNetworkRequests, + pagination_buttons: paginationButtons, + closed_popup_messages: this._getClosedPopupMessagesSnapshot(), + }); + this.logger.debug('✅ get_state_summary completed successfully'); + return browser_state; + } + /** + * Check if a URL is a new tab page + */ + _is_new_tab_page(url) { + return (url === 'about:blank' || + url === 'about:newtab' || + url === 'chrome://newtab/' || + url === 'chrome://new-tab-page/' || + url === 'chrome://new-tab-page'); + } + _is_ip_address_host(hostname) { + const normalized = hostname.startsWith('[') && hostname.endsWith(']') + ? hostname.slice(1, -1) + : hostname; + return isIP(normalized) !== 0; + } + _get_domain_variants(hostname) { + const host = hostname.toLowerCase(); + if (host.startsWith('www.')) { + return [host, host.slice(4)]; + } + return [host, `www.${host}`]; + } + _setEntryMatchesUrl(domains, hostVariant, hostAlt, protocol) { + const matchedHost = domains.has(hostVariant) || domains.has(hostAlt); + if (!matchedHost) { + return false; + } + // Set-optimized entries are exact hostnames without explicit schemes, + // so keep parity with pattern matching default: https-only. + return protocol.toLowerCase() === 'https:'; + } + /** + * Check if page is displaying a PDF + */ + async _is_pdf_viewer(page) { + try { + const url = page.url(); + if (url.endsWith('.pdf') || url.includes('.pdf?')) { + return true; + } + // Check for PDF viewer in page content + const is_pdf = await page.evaluate(() => { + return (document.querySelector('embed[type="application/pdf"]') !== null || + document.querySelector('object[type="application/pdf"]') !== null); + }); + return is_pdf; + } + catch (error) { + return false; + } + } + /** + * Auto-download PDF if detected and auto-download is enabled + */ + async _auto_download_pdf_if_needed(page) { + const downloadsPath = this.browser_profile.downloads_path; + if (!downloadsPath || !this._autoDownloadPdfs) { + return null; + } + try { + const is_pdf = await this._is_pdf_viewer(page); + if (!is_pdf) { + return null; + } + const url = page.url(); + this.logger.info(`📄 PDF detected: ${url}`); + let pdfFilename = path.basename(url.split('?')[0]); + if (!pdfFilename || !pdfFilename.toLowerCase().endsWith('.pdf')) { + const parsed = new URL(url); + pdfFilename = path.basename(parsed.pathname) || 'document.pdf'; + if (!pdfFilename.toLowerCase().endsWith('.pdf')) { + pdfFilename += '.pdf'; + } + } + if (this.downloaded_files.some((downloaded) => path.basename(downloaded) === pdfFilename)) { + this.logger.debug(`📄 PDF already downloaded: ${pdfFilename}`); + return null; + } + this.logger.info(`📄 Auto-downloading PDF from: ${url}`); + const downloadResult = await page.evaluate(async (pdfUrl) => { + try { + const response = await fetch(pdfUrl, { + cache: 'force-cache', + }); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + const blob = await response.blob(); + const arrayBuffer = await blob.arrayBuffer(); + const uint8Array = new Uint8Array(arrayBuffer); + const cacheHeader = response.headers.get('x-cache') || ''; + const fromCache = response.headers.has('age') || + cacheHeader.toLowerCase().includes('hit'); + return { + data: Array.from(uint8Array), + fromCache, + responseSize: uint8Array.length, + }; + } + catch (error) { + return { + data: [], + fromCache: false, + responseSize: 0, + error: error instanceof Error ? error.message : 'Unknown fetch error', + }; + } + }, url); + if (downloadResult?.error) { + this.logger.warning(`⚠️ Failed to auto-download PDF from ${url}: ${downloadResult.error}`); + return null; + } + if (!downloadResult || + !Array.isArray(downloadResult.data) || + downloadResult.data.length === 0) { + this.logger.warning(`⚠️ No data received when downloading PDF from ${url}`); + return null; + } + await fs.promises.mkdir(downloadsPath, { recursive: true }); + const uniqueFilename = await BrowserSession.get_unique_filename(downloadsPath, pdfFilename); + const downloadPath = path.join(downloadsPath, uniqueFilename); + await fs.promises.writeFile(downloadPath, Buffer.from(downloadResult.data)); + this.add_downloaded_file(downloadPath); + const cacheStatus = downloadResult.fromCache + ? 'from cache' + : 'from network'; + const responseSize = Number(downloadResult.responseSize || 0); + this.logger.info(`📄 Auto-downloaded PDF (${cacheStatus}, ${responseSize.toLocaleString()} bytes): ${downloadPath}`); + return downloadPath; + } + catch (error) { + this.logger.debug(`PDF detection failed: ${error.message}`); + return null; + } + } + /** + * Check if an element is visible on the page + */ + async _is_visible(element) { + try { + const is_hidden = await element.isHidden(); + const bbox = await element.boundingBox(); + return !is_hidden && bbox !== null && bbox.width > 0 && bbox.height > 0; + } + catch (error) { + return false; + } + } + /** + * Locate an element by XPath + */ + async get_locate_element_by_xpath(xpath) { + const page = await this.get_current_page(); + if (!page) { + return null; + } + try { + // Use XPath to locate the element + const element_handle = await page + .locator(`xpath=${xpath}`) + .elementHandle(); + if (element_handle) { + const is_visible = await this._is_visible(element_handle); + if (is_visible) { + await element_handle.scrollIntoViewIfNeeded({ timeout: 1000 }); + } + return element_handle; + } + return null; + } + catch (error) { + this.logger.error(`❌ Failed to locate xpath ${xpath}: ${error.message}`); + return null; + } + } + /** + * Locate an element by CSS selector + */ + async get_locate_element_by_css_selector(css_selector) { + const page = await this.get_current_page(); + if (!page) { + return null; + } + try { + // Use CSS selector to locate the element + const element_handle = await page.locator(css_selector).elementHandle(); + if (element_handle) { + const is_visible = await this._is_visible(element_handle); + if (is_visible) { + await element_handle.scrollIntoViewIfNeeded({ timeout: 1000 }); + } + return element_handle; + } + return null; + } + catch (error) { + this.logger.error(`❌ Failed to locate element ${css_selector}: ${error.message}`); + return null; + } + } + /** + * Locate an element by text content + * @param text - Text to search for + * @param nth - Which matching element to return (0-based index) + * @param element_type - Optional tag name to filter by (e.g., 'button', 'span') + */ + async get_locate_element_by_text(text, nth = 0, element_type = null) { + const page = await this.get_current_page(); + if (!page) { + return null; + } + try { + // Build selector: filter by element type and text + const selector = element_type + ? `${element_type}:text("${text}")` + : `:text("${text}")`; + // Get all matching elements + const locator = page.locator(selector); + const count = await locator.count(); + if (count === 0) { + this.logger.error(`❌ No element with text '${text}' found`); + return null; + } + // Filter visible elements + const visible_elements = []; + for (let i = 0; i < count; i++) { + const element_handle = await locator.nth(i).elementHandle(); + if (element_handle && (await this._is_visible(element_handle))) { + visible_elements.push(element_handle); + } + } + if (visible_elements.length === 0) { + this.logger.error(`❌ No visible element with text '${text}' found`); + return null; + } + if (nth >= visible_elements.length) { + this.logger.error(`❌ Element with text '${text}' not found at index #${nth}`); + return null; + } + const element_handle = visible_elements[nth]; + const is_visible = await this._is_visible(element_handle); + if (is_visible) { + await element_handle.scrollIntoViewIfNeeded({ timeout: 1000 }); + } + return element_handle; + } + catch (error) { + this.logger.error(`❌ Failed to locate element by text '${text}': ${error.message}`); + return null; + } + } + /** + * Check if browser session is connected and has valid browser/context objects + * @param restart - If true, attempt to create a new tab if no pages exist + */ + async is_connected(restart = true) { + if (!this.browser_context) { + return false; + } + try { + // Check if browser is connected + if (this.browser && !this.browser.isConnected()) { + return false; + } + // Check if browser context's browser is connected (context may reference a different browser object) + const context_browser = this.browser_context.browser?.(); + if (context_browser && !context_browser.isConnected()) { + return false; + } + // Check if context has at least one page + const pages = this.browser_context.pages(); + if (pages.length === 0) { + if (restart) { + // Try to create a new page to keep context alive + try { + await this.browser_context.newPage(); + } + catch (error) { + return false; + } + } + else { + return false; + } + } + return true; + } + catch (error) { + return false; + } + } + /** + * Check if a URL is allowed based on allowed_domains configuration + * @param url - URL to check + */ + _get_url_access_denial_reason(url) { + // Always allow new tab pages and browser-internal pages we intentionally use. + if (this._is_new_tab_page(url)) { + return null; + } + let parsed; + try { + parsed = new URL(url); + } + catch { + return 'invalid_url'; + } + if (parsed.protocol === 'data:' || parsed.protocol === 'blob:') { + return null; + } + if (!parsed.hostname) { + return 'missing_host'; + } + const [hostVariant, hostAlt] = this._get_domain_variants(parsed.hostname); + if (this.browser_profile.block_ip_addresses && + this._is_ip_address_host(parsed.hostname)) { + return 'ip_address_blocked'; + } + const allowedDomains = this.browser_profile.allowed_domains; + if (allowedDomains && + ((Array.isArray(allowedDomains) && allowedDomains.length > 0) || + (allowedDomains instanceof Set && allowedDomains.size > 0))) { + if (allowedDomains instanceof Set) { + if (this._setEntryMatchesUrl(allowedDomains, hostVariant, hostAlt, parsed.protocol)) { + return null; + } + } + else { + for (const allowedDomain of allowedDomains) { + try { + if (match_url_with_domain_pattern(url, allowedDomain, true)) { + return null; + } + } + catch { + this.logger.warning(`Invalid domain pattern: ${allowedDomain}`); + } + } + } + return 'not_in_allowed_domains'; + } + const prohibitedDomains = this.browser_profile.prohibited_domains; + if (prohibitedDomains && + ((Array.isArray(prohibitedDomains) && prohibitedDomains.length > 0) || + (prohibitedDomains instanceof Set && prohibitedDomains.size > 0))) { + if (prohibitedDomains instanceof Set) { + if (this._setEntryMatchesUrl(prohibitedDomains, hostVariant, hostAlt, parsed.protocol)) { + return 'in_prohibited_domains'; + } + } + else { + for (const prohibitedDomain of prohibitedDomains) { + try { + if (match_url_with_domain_pattern(url, prohibitedDomain, true)) { + return 'in_prohibited_domains'; + } + } + catch { + this.logger.warning(`Invalid domain pattern: ${prohibitedDomain}`); + } + } + } + } + return null; + } + _is_url_allowed(url) { + return this._get_url_access_denial_reason(url) === null; + } + _formatDomainCollection(value) { + if (value instanceof Set) { + return JSON.stringify(Array.from(value)); + } + return JSON.stringify(value ?? null); + } + _assert_url_allowed(url) { + const denialReason = this._get_url_access_denial_reason(url); + if (!denialReason) { + return; + } + this._recordRecentEvent('navigation_blocked', { + url, + error_message: denialReason, + }); + if (denialReason === 'not_in_allowed_domains') { + throw new URLNotAllowedError(`URL ${url} is not in allowed_domains. Current allowed_domains: ${this._formatDomainCollection(this.browser_profile.allowed_domains)}`); + } + if (denialReason === 'in_prohibited_domains') { + throw new URLNotAllowedError(`URL ${url} is blocked by prohibited_domains. Current prohibited_domains: ${this._formatDomainCollection(this.browser_profile.prohibited_domains)}`); + } + if (denialReason === 'ip_address_blocked') { + throw new URLNotAllowedError(`URL ${url} is blocked because block_ip_addresses=true`); + } + throw new URLNotAllowedError(`URL ${url} is not allowed (${denialReason})`); + } + /** + * Navigate helper with URL validation + */ + async navigate(url) { + this._assert_url_allowed(url); + await this.navigate_to(url); + } + /** + * Kill the browser session (force close even if keep_alive=true) + */ + async kill() { + this.logger.info('💀 Force killing browser session...'); + // Temporarily disable keep_alive to ensure browser closes + const original_keep_alive = this.browser_profile.keep_alive; + this.browser_profile.keep_alive = false; + try { + await this.close(); + } + finally { + // Restore original keep_alive setting + this.browser_profile.keep_alive = original_keep_alive; + } + } + /** + * Alias for close() to match Python API + */ + async stop() { + if (this.browser_profile.keep_alive) { + this.logger.info('🕊️ BrowserSession.stop() called but keep_alive=true, leaving browser running. Use .kill() to force close.'); + return; + } + if (this._stoppingPromise) { + await this._stoppingPromise; + return; + } + const hasActiveResources = this.initialized || + Boolean(this.browser || + this.browser_context || + this.browser_pid || + this._subprocess || + this._childProcesses.size > 0); + if (!hasActiveResources) { + return; + } + this._stoppingPromise = Promise.resolve().then(async () => { + await this.event_bus.dispatch(new BrowserStopEvent()); + await this._shutdown_browser_session(); + }); + try { + await this._stoppingPromise; + this._recordRecentEvent('browser_stopped'); + await this.event_bus.dispatch(new BrowserStoppedEvent()); + } + finally { + this.detach_all_watchdogs(); + await this.event_bus.stop(); + this._stoppingPromise = null; + } + } + /** + * Perform a click action with download and navigation handling + * @param element_node - DOM element to click + */ + async perform_click(element_node) { + const page = await this.get_current_page(); + if (!page) { + throw new Error('No current page available'); + } + const element_handle = await this.get_locate_element(element_node); + if (!element_handle) { + throw new Error(`Element not found: ${JSON.stringify(element_node)}`); + } + // Check if downloads are enabled + const downloads_path = this.browser_profile.downloads_path; + if (downloads_path) { + fs.mkdirSync(downloads_path, { recursive: true }); + // Try to detect file download. + const download_promise = page.waitForEvent('download', { + timeout: 5000, + }); + // Click failures should bubble to the caller. + try { + await element_handle.click(); + } + catch (error) { + void download_promise.catch(() => undefined); + throw error; + } + let download; + try { + download = await download_promise; + } + catch (error) { + const message = error instanceof Error ? error.message : String(error); + const isDownloadTimeout = error instanceof Error && + (error.name === 'TimeoutError' || + message.toLowerCase().includes('timeout')); + if (!isDownloadTimeout) { + throw error; + } + this.logger.debug('No download triggered within timeout. Checking navigation...'); + try { + await page.waitForLoadState(); + } + catch (e) { + this.logger.warning(`Navigation check failed: ${e.message}`); + } + return null; + } + // Save the downloaded file. + const suggested_filename = download.suggestedFilename(); + const unique_filename = await BrowserSession.get_unique_filename(downloads_path, suggested_filename); + const download_path = path.join(downloads_path, unique_filename); + const download_guid = uuid7str(); + const download_url = typeof download.url === 'function' + ? download.url() + : (this.currentUrl ?? ''); + await this.event_bus.dispatch(new DownloadStartedEvent({ + guid: download_guid, + url: download_url, + suggested_filename, + auto_download: false, + })); + await download.saveAs(download_path); + this.logger.info(`⬇️ Downloaded file to: ${download_path}`); + const stats = fs.existsSync(download_path) + ? fs.statSync(download_path) + : null; + await this.event_bus.dispatch(new DownloadProgressEvent({ + guid: download_guid, + received_bytes: stats?.size ?? 0, + total_bytes: stats?.size ?? 0, + state: 'completed', + })); + const fileDownloadedResult = await this.event_bus.dispatch(new FileDownloadedEvent({ + guid: download_guid, + url: download_url, + path: download_path, + file_name: unique_filename, + file_size: stats?.size ?? 0, + file_type: path.extname(unique_filename).replace('.', '') || null, + mime_type: null, + auto_download: false, + })); + if (fileDownloadedResult.handler_results.length === 0) { + this.add_downloaded_file(download_path); + } + return download_path; + } + else { + // No downloads path configured, just click + await element_handle.click(); + } + return null; + } + /** + * Remove all highlights from the current page + */ + async remove_highlights() { + const page = await this.get_current_page(); + if (!page) { + return; + } + try { + await page.evaluate(() => { + const pageWindow = window; + const cleanupFunctions = Array.isArray(pageWindow._highlightCleanupFunctions) + ? pageWindow._highlightCleanupFunctions + : []; + for (const cleanupFn of cleanupFunctions) { + try { + if (typeof cleanupFn === 'function') { + cleanupFn(); + } + } + catch { + // Ignore callback cleanup failures. + } + } + pageWindow._highlightCleanupFunctions = []; + const containers = document.querySelectorAll('#playwright-highlight-container'); + containers.forEach((element) => element.remove()); + const labels = document.querySelectorAll('.playwright-highlight-label'); + labels.forEach((element) => element.remove()); + // Backward compatibility with legacy selectors. + const highlights = document.querySelectorAll('.browser-use-highlight'); + highlights.forEach((element) => element.remove()); + const styled = document.querySelectorAll('[style*="browser-use"]'); + styled.forEach((element) => { + if (element.style) { + element.style.outline = ''; + element.style.border = ''; + } + }); + }); + } + catch (error) { + this.logger.debug(`Failed to remove highlights: ${error.message}`); + } + } + // region - Trace Recording + /** + * Start tracing on browser context if traces_dir is configured + * Note: Currently optional as it may cause performance issues in some cases + */ + async start_trace_recording() { + await this._startContextTracing(); + } + /** + * Save browser trace recording if active + */ + async save_trace_recording() { + await this._saveTraceRecording(); + } + /** + * Start tracing on browser context if traces_dir is configured + * Note: Currently optional as it may cause performance issues in some cases + */ + async _startContextTracing() { + if (this.browser_profile.traces_dir && this.browser_context) { + try { + this.logger.debug(`📽️ Starting tracing (will save to: ${this.browser_profile.traces_dir})`); + await this.browser_context.tracing.start({ + screenshots: true, + snapshots: true, + sources: false, // Reduce trace size + }); + } + catch (error) { + this.logger.warning(`Failed to start tracing: ${error.message}`); + throw error; + } + } + } + /** + * Save browser trace recording + */ + async _saveTraceRecording() { + if (this.browser_profile.traces_dir && this.browser_context) { + try { + const tracesPath = this.browser_profile.traces_dir; + let finalTracePath; + // Check if path has extension + if (path.extname(tracesPath)) { + // Path has extension, use as-is (user specified exact file path) + finalTracePath = tracesPath; + } + else { + // Path has no extension, treat as directory and create filename + const traceFilename = `BrowserSession_${this.id}.zip`; + finalTracePath = path.join(tracesPath, traceFilename); + } + this.logger.info(`🎥 Saving browser_context trace to ${finalTracePath}...`); + await this.browser_context.tracing.stop({ path: finalTracePath }); + } + catch (error) { + this.logger.warning(`Failed to save trace recording: ${error.message}`); + throw error; + } + } + } + // endregion + // region - CDP Advanced Integration + /** + * Scroll using CDP Input.synthesizeScrollGesture for universal compatibility + * @param page - The page to scroll + * @param pixels - Number of pixels to scroll (positive = up, negative = down) + * @returns true if successful, false if failed + */ + async _scrollWithCdpGesture(page, pixels) { + try { + // Use CDP to synthesize scroll gesture - works in all contexts including PDFs + const cdpSession = await this.get_or_create_cdp_session(page); + // Get viewport center for scroll origin + const viewport = await page.evaluate(() => ({ + width: window.innerWidth, + height: window.innerHeight, + })); + const centerX = Math.floor(viewport.width / 2); + const centerY = Math.floor(viewport.height / 2); + await cdpSession.send('Input.synthesizeScrollGesture', { + x: centerX, + y: centerY, + xDistance: 0, + yDistance: -pixels, // Negative = scroll down, Positive = scroll up + gestureSourceType: 'mouse', // Use mouse gestures for better compatibility + speed: 3000, // Pixels per second + }); + try { + await Promise.race([ + cdpSession.detach(), + new Promise((resolve) => setTimeout(resolve, 1000)), + ]); + } + catch { + // Ignore detach errors + } + this.logger.debug(`📄 Scrolled via CDP Input.synthesizeScrollGesture: ${pixels}px`); + return true; + } + catch (error) { + this.logger.warning(`❌ Scrolling via CDP Input.synthesizeScrollGesture failed: ${error.message}`); + return false; + } + } + /** + * Scroll the current page container + * @param pixels - Number of pixels to scroll (positive = up, negative = down) + */ + async _scrollContainer(pixels) { + const page = await this.getCurrentPage(); + if (!page) { + throw new Error('No active page available for scrolling'); + } + // Try CDP scroll gesture first (works universally including PDFs) + if (await this._scrollWithCdpGesture(page, pixels)) { + return; + } + // Fallback to JavaScript for older browsers or when CDP fails + this.logger.debug('Falling back to JavaScript scrolling'); + const SMART_SCROLL_JS = `(dy) => { + const bigEnough = el => el.clientHeight >= window.innerHeight * 0.5; + const canScroll = el => + el && + /(auto|scroll|overlay)/.test(getComputedStyle(el).overflowY) && + el.scrollHeight > el.clientHeight && + bigEnough(el); + + let el = document.activeElement; + while (el && !canScroll(el) && el !== document.body) el = el.parentElement; + + el = canScroll(el) + ? el + : [...document.querySelectorAll('*')].find(canScroll) + || document.scrollingElement + || document.documentElement; + + if (el === document.scrollingElement || + el === document.documentElement || + el === document.body) { + window.scrollBy(0, dy); + } else { + el.scrollBy(0, dy); + } + }`; + await page.evaluate(SMART_SCROLL_JS, pixels); + } + /** + * Compute hashes for all clickable elements in the selector map + * @param selectorMap - Selector map from DOM state + * @returns Set of element hashes + */ + _computeElementHashes(selectorMap) { + const hashes = new Set(); + for (const [index, element] of Object.entries(selectorMap)) { + if (element instanceof DOMElementNode) { + // Create hash from element's xpath and key attributes + const hashParts = [ + element.xpath || '', + element.tag_name || '', + JSON.stringify(element.attributes || {}), + ]; + const hash = hashParts.join('|'); + hashes.add(hash); + } + } + return hashes; + } + /** + * Mark elements in the selector map as new if they weren't in the cached hashes + * @param selectorMap - Selector map to update + * @param cachedHashes - Previously cached element hashes + */ + _markNewElements(selectorMap, cachedHashes) { + for (const [index, element] of Object.entries(selectorMap)) { + if (element instanceof DOMElementNode) { + // Create hash for current element + const hashParts = [ + element.xpath || '', + element.tag_name || '', + JSON.stringify(element.attributes || {}), + ]; + const hash = hashParts.join('|'); + // Mark as new if not in cached hashes + if (!cachedHashes.has(hash)) { + // Add a marker to the element's attributes to indicate it's new + element.attributes = element.attributes || {}; + element.attributes['__browser_use_new_element'] = true; + } + } + } + } + /** + * Helper to get a safe method name from the calling context + * Used for recovery error messages + */ + _getCurrentMethodName() { + try { + const stack = new Error().stack; + if (!stack) + return 'unknown'; + const lines = stack.split('\n'); + // Skip first 3 lines: Error, this method, and the caller + const callerLine = lines[3] || ''; + const match = callerLine.match(/at (?:BrowserSession\.)?(\w+)/); + return match ? match[1] : 'unknown'; + } + catch { + return 'unknown'; + } + } + /** + * Get current page with fallback logic + * Alias for compatibility with Python API + */ + async getCurrentPage() { + return await this.get_current_page(); + } + /** + * Log warning about unsafe glob patterns + * @param pattern - The glob pattern being used + */ + _logGlobWarning(pattern) { + const unsafePatterns = [ + '**/*', + '**/.*', + '~/*', + '/etc/*', + '/sys/*', + '/proc/*', + ]; + const isUnsafe = unsafePatterns.some((unsafe) => pattern.includes(unsafe) || + pattern.startsWith(unsafe.replace('**/', ''))); + if (isUnsafe) { + this.logger.warning(`⚠️ Potentially unsafe glob pattern detected: "${pattern}". ` + + `This could access system files or expose sensitive data.`); + } + } + /** + * Create a shallow copy of the browser session + * Note: This doesn't copy the actual browser instance, just the session metadata + * @returns A new BrowserSession instance with copied state + */ + modelCopy() { + const copy = new BrowserSession({ + id: this.id, + browser_profile: this.browser_profile, + browser: this.browser, + browser_context: this.browser_context, + page: this.agent_current_page, + title: this.currentTitle, + url: this.currentUrl, + wss_url: this.wss_url, + cdp_url: this.cdp_url, + browser_pid: this.browser_pid, + playwright: this.playwright, + downloaded_files: [...this.downloaded_files], + closed_popup_messages: [...this._closedPopupMessages], + }); + copy.llm_screenshot_size = this.llm_screenshot_size + ? [...this.llm_screenshot_size] + : null; + copy._original_viewport_size = this._original_viewport_size + ? [...this._original_viewport_size] + : null; + return copy; + } + model_copy() { + return this.modelCopy(); + } + // endregion + // region - Page Health Check and Recovery + _inRecovery = false; + /** + * Check if a page is responsive by trying to evaluate simple JavaScript + * @param page - The page to check + * @param timeout - Timeout in seconds (default: 5.0) + * @returns true if page is responsive, false otherwise + */ + async _isPageResponsive(page, timeout = 5.0) { + try { + const timeoutMs = timeout * 1000; + await Promise.race([ + page.evaluate('1'), + new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), timeoutMs)), + ]); + return true; + } + catch (error) { + return false; + } + } + /** + * Force close a crashed page using CDP from a clean temporary page + * @param pageUrl - The URL of the page to force close + * @returns true if successful, false otherwise + */ + async _forceClosePageViaCdp(pageUrl) { + try { + if (!this.browser_context) { + throw new Error('Browser context is not set up yet'); + } + // Create a clean page for CDP operations + const tempPage = await Promise.race([ + this.browser_context.newPage(), + new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout creating temp page')), 5000)), + ]); + await Promise.race([ + tempPage.goto('about:blank'), + new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout navigating to blank')), 2000)), + ]); + try { + // Create CDP session from the clean page + const cdpSession = await Promise.race([ + this.get_or_create_cdp_session(tempPage), + new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout creating CDP session')), 5000)), + ]); + try { + // Get all browser targets + const targets = (await Promise.race([ + cdpSession.send('Target.getTargets'), + new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout getting targets')), 2000)), + ])); + // Find the crashed page target + let blockedTargetId = null; + const targetInfos = targets.targetInfos || []; + for (const target of targetInfos) { + if (target.type === 'page' && target.url === pageUrl) { + blockedTargetId = target.targetId; + break; + } + } + if (blockedTargetId) { + // Force close the target + this.logger.warning(`🪓 Force-closing crashed page target_id=${blockedTargetId} via CDP: ${pageUrl.substring(0, 50)}...`); + await Promise.race([ + cdpSession.send('Target.closeTarget', { + targetId: blockedTargetId, + }), + new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout closing target')), 2000)), + ]); + return true; + } + else { + this.logger.debug(`❌ Could not find CDP page target_id to force-close: ${pageUrl.substring(0, 50)} (concurrency issues?)`); + return false; + } + } + finally { + try { + await Promise.race([ + cdpSession.detach(), + new Promise((resolve) => setTimeout(resolve, 1000)), + ]); + } + catch { + // Ignore detach errors + } + } + } + finally { + await tempPage.close(); + } + } + catch (error) { + this.logger.error(`❌ Using raw CDP to force-close crashed page failed: ${error.message}`); + return false; + } + } + /** + * Try to reopen a URL in a new page and check if it's responsive + * @param url - The URL to reopen + * @param timeoutMs - Navigation timeout in milliseconds + * @returns true if successful and responsive, false otherwise + */ + async _tryReopenUrl(url, timeoutMs) { + if (!url || + url.startsWith('about:') || + url.startsWith('chrome://') || + url.startsWith('edge://')) { + return false; + } + const timeout = timeoutMs || this.browser_profile.default_navigation_timeout || 6000; + try { + this.logger.debug(`🔄 Attempting to reload URL that crashed: ${url.substring(0, 50)}`); + if (!this.browser_context) { + throw new Error('Browser context is not set'); + } + // Create new page directly to avoid circular dependency + const newPage = await this.browser_context.newPage(); + this.agent_current_page = newPage; + // Update human tab reference if there is no human tab yet + if (!this.human_current_page || this.human_current_page.isClosed()) { + this.human_current_page = newPage; + } + // Set viewport for new tab + if (this.browser_profile.window_size) { + await newPage.setViewportSize(this.browser_profile.window_size); + } + // Navigate with timeout + try { + await Promise.race([ + newPage.goto(url, { waitUntil: 'load', timeout }), + new Promise((_, reject) => setTimeout(() => reject(new Error('Navigation timeout')), timeout + 500)), + ]); + } + catch (error) { + this.logger.debug(`⚠️ Attempting to reload previously crashed URL ${url.substring(0, 50)} failed again: ${error.name}`); + } + // Wait a bit for any transient blocking to resolve + await new Promise((resolve) => setTimeout(resolve, 1000)); + // Check if the reopened page is responsive + const isResponsive = await this._isPageResponsive(newPage, 2.0); + if (isResponsive) { + this.logger.info(`✅ Page recovered and is now responsive after reopening on: ${url.substring(0, 50)}`); + return true; + } + else { + this.logger.warning(`⚠️ Reopened page ${url.substring(0, 50)} is still unresponsive`); + // Close the unresponsive page before returning + try { + await this._forceClosePageViaCdp(newPage.url()); + } + catch (error) { + this.logger.error(`❌ Failed to close crashed page ${url.substring(0, 50)} via CDP: ${error.message} (something is very wrong or system is extremely overloaded)`); + } + this.agent_current_page = null; // Clear reference to closed page + return false; + } + } + catch (error) { + this.logger.error(`❌ Retrying crashed page ${url.substring(0, 50)} failed: ${error.message}`); + return false; + } + } + /** + * Create a new blank page as a fallback when recovery fails + * @param url - The original URL that failed + */ + async _createBlankFallbackPage(url) { + this.logger.warning(`⚠️ Resetting to about:blank as fallback because browser is unable to load the original URL without crashing: ${url.substring(0, 50)}`); + // Close any existing broken page + if (this.agent_current_page && !this.agent_current_page.isClosed()) { + try { + await this.agent_current_page.close(); + } + catch { + // Ignore close errors + } + } + if (!this.browser_context) { + throw new Error('Browser context is not set'); + } + // Create fresh page directly (avoid decorated methods to prevent circular dependency) + const newPage = await this.browser_context.newPage(); + this.agent_current_page = newPage; + // Update human tab reference if there is no human tab yet + if (!this.human_current_page || this.human_current_page.isClosed()) { + this.human_current_page = newPage; + } + // Set viewport for new tab + if (this.browser_profile.window_size) { + await newPage.setViewportSize(this.browser_profile.window_size); + } + // Navigate to blank + try { + await newPage.goto('about:blank', { waitUntil: 'load', timeout: 5000 }); + } + catch (error) { + this.logger.error(`❌ Failed to navigate to about:blank: ${error.message} (something is very wrong or system is extremely overloaded)`); + throw error; + } + // Verify it's responsive + if (!(await this._isPageResponsive(newPage, 1.0))) { + throw new BrowserError('Browser is unable to load any new about:blank pages (something is very wrong or browser is extremely overloaded)'); + } + } + /** + * Recover from an unresponsive page by closing and reopening it + * @param callingMethod - The name of the method that detected the unresponsive page + * @param timeoutMs - Navigation timeout in milliseconds + */ + async _recoverUnresponsivePage(callingMethod, timeoutMs) { + this.logger.warning(`⚠️ Page JS engine became unresponsive in ${callingMethod}(), attempting recovery...`); + const timeout = Math.min(3000, timeoutMs || this.browser_profile.default_navigation_timeout || 5000); + // Check if browser connection is still alive + if (this.browser && !this.browser.isConnected()) { + this.logger.error('❌ Browser connection lost - browser process may have crashed'); + throw new Error('Browser connection lost - cannot recover unresponsive page'); + } + // Prevent re-entrance + if (this._inRecovery) { + this.logger.debug('Already in recovery, skipping nested recovery attempt'); + return; + } + this._inRecovery = true; + try { + // Get current URL before recovery + if (!this.agent_current_page) { + throw new Error('Agent current page is not set'); + } + const currentUrl = this.agent_current_page.url(); + // Clear page references + const blockedPage = this.agent_current_page; + this.agent_current_page = null; + if (blockedPage === this.human_current_page) { + this.human_current_page = null; + } + // Force-close the crashed page via CDP + this.logger.debug('🪓 Page Recovery Step 1/3: Force-closing crashed page via CDP...'); + await this._forceClosePageViaCdp(currentUrl); + // Remove the closed page from browser_context.pages by forcing a refresh + if (this.browser_context && this.browser_context.pages()) { + for (const page of this.browser_context.pages().slice()) { + const pageUrl = page.url(); + if (pageUrl === currentUrl && + !page.isClosed() && + !pageUrl.startsWith('about:') && + !pageUrl.startsWith('chrome://') && + !pageUrl.startsWith('edge://')) { + try { + await page.close(); + this.logger.debug(`🪓 Closed page because it has a known crash-causing URL: ${pageUrl.substring(0, 50)}`); + } + catch { + // Page might already be closed via CDP + } + } + } + } + // Try to reopen the URL (in case blocking was transient) + this.logger.debug('🍼 Page Recovery Step 2/3: Trying to reopen the URL again...'); + if (await this._tryReopenUrl(currentUrl, timeout)) { + this.logger.debug('✅ Page Recovery Step 3/3: Page loading succeeded after 2nd attempt!'); + return; // Success! + } + // If that failed, fall back to blank page + this.logger.debug('❌ Page Recovery Step 3/3: Loading the page a 2nd time failed as well, browser seems unable to load this URL without getting stuck, retreating to a safe page...'); + await this._createBlankFallbackPage(currentUrl); + } + finally { + // Always clear recovery flag + this._inRecovery = false; + } + } + // endregion + // region - Enhanced CSS Selector Generation + /** + * Generate enhanced CSS selector for an element + * Handles special characters and provides fallback strategies + * @param xpath - XPath of the element + * @param element - Optional element node for additional context + * @returns Enhanced CSS selector string + */ + _enhancedCssSelectorForElement(xpath, element) { + // Try to convert XPath to CSS selector + const cssSelector = this._xpathToCss(xpath); + if (cssSelector) { + return cssSelector; + } + // Fallback: use element attributes if available + if (element) { + const selectors = []; + // Try ID first (most specific) + if (element.attributes?.id) { + const id = this._escapeSelector(element.attributes.id); + selectors.push(`#${id}`); + } + // Try class names + if (element.attributes?.class) { + const classes = element.attributes.class + .split(/\s+/) + .filter((c) => c.length > 0) + .map((c) => `.${this._escapeSelector(c)}`) + .join(''); + if (classes) { + selectors.push(`${element.tag_name}${classes}`); + } + } + // Try name attribute + if (element.attributes?.name) { + const name = this._escapeSelector(element.attributes.name); + selectors.push(`${element.tag_name}[name="${name}"]`); + } + // Try data attributes + for (const [key, value] of Object.entries(element.attributes || {})) { + if (key.startsWith('data-')) { + const escaped = this._escapeSelector(String(value)); + selectors.push(`${element.tag_name}[${key}="${escaped}"]`); + } + } + if (selectors.length > 0) { + return selectors[0]; + } + // Last resort: just the tag name + return element.tag_name || 'div'; + } + // Ultimate fallback + return 'body'; + } + /** + * Convert XPath to CSS selector + * Handles simple XPath expressions + */ + _xpathToCss(xpath) { + try { + // Remove leading slashes + let path = xpath.replace(/^\/+/, ''); + // Handle simple cases like /html/body/div[1]/span[2] + const parts = path.split('/'); + const cssparts = []; + for (const part of parts) { + // Extract tag and index: div[1] -> {tag: 'div', index: 1} + const match = part.match(/^([a-zA-Z0-9_-]+)(?:\[(\d+)\])?$/); + if (match) { + const [, tag, index] = match; + if (index) { + // CSS uses nth-of-type (1-indexed like XPath) + cssparts.push(`${tag}:nth-of-type(${index})`); + } + else { + cssparts.push(tag); + } + } + else { + // Complex XPath, can't convert + return null; + } + } + return cssparts.join(' > '); + } + catch { + return null; + } + } + /** + * Escape special characters in CSS selectors + * Handles characters that need escaping in CSS + */ + _escapeSelector(selector) { + // Escape special CSS characters + return selector.replace(/[!"#$%&'()*+,./:;<=>?@[\\\]^`{|}~]/g, '\\$&'); + } + // endregion + // region - User Data Directory Management + /** + * Prepare user data directory for browser profile + * Handles singleton lock conflicts and creates temp profiles if needed + */ + async prepareUserDataDir(userDataDir) { + if (!userDataDir) { + // Use profile's user data dir or create temp one + userDataDir = + this.browser_profile.user_data_dir || + (await this._createTempUserDataDir()); + } + // Check for singleton lock conflicts + const hasConflict = await this._checkForSingletonLockConflict(userDataDir); + if (hasConflict) { + this.logger.warning(`Singleton lock detected in ${userDataDir}, falling back to temp profile`); + userDataDir = await this._fallbackToTempProfile(); + } + // Ensure directory exists + if (!fs.existsSync(userDataDir)) { + fs.mkdirSync(userDataDir, { recursive: true }); + this.logger.debug(`Created user data directory: ${userDataDir}`); + } + return userDataDir; + } + /** + * Check if user data directory has a singleton lock + * This happens when another Chrome instance is using the profile + */ + async _checkForSingletonLockConflict(userDataDir) { + try { + const singletonLockFile = path.join(userDataDir, 'SingletonLock'); + const singletonSocketFile = path.join(userDataDir, 'SingletonSocket'); + const singletonCookieFile = path.join(userDataDir, 'SingletonCookie'); + // Check if any singleton lock files exist + if (fs.existsSync(singletonLockFile) || + fs.existsSync(singletonSocketFile) || + fs.existsSync(singletonCookieFile)) { + // Try to detect if process is still alive (Unix-like systems) + if (process.platform !== 'win32' && fs.existsSync(singletonLockFile)) { + try { + // Try to read the lock file to get PID + const lockContent = fs.readFileSync(singletonLockFile, 'utf-8'); + const pidMatch = lockContent.match(/(\d+)/); + if (pidMatch) { + const pid = parseInt(pidMatch[1], 10); + try { + // Check if process exists (signal 0 doesn't kill, just checks) + process.kill(pid, 0); + return true; // Process exists, lock is valid + } + catch { + // Process doesn't exist, stale lock + this.logger.debug(`Stale singleton lock detected, removing`); + fs.unlinkSync(singletonLockFile); + return false; + } + } + } + catch { + // Couldn't read lock file + } + } + return true; + } + return false; + } + catch (error) { + this.logger.debug(`Error checking singleton lock: ${error.message}`); + return false; + } + } + /** + * Fallback to a temporary profile when the primary one is locked + */ + async _fallbackToTempProfile() { + const tempDir = await this._createTempUserDataDir(); + this.logger.info(`Using temporary profile: ${tempDir}`); + return tempDir; + } + /** + * Create a temporary user data directory + */ + async _createTempUserDataDir() { + const osTempDir = os.tmpdir(); + const tempDir = path.join(osTempDir, `browser-use-${Date.now()}-${Math.random().toString(36).slice(2)}`); + fs.mkdirSync(tempDir, { recursive: true }); + return tempDir; + } + // endregion + // region - Page Visibility Listeners + /** + * Setup listeners for page visibility changes + * Tracks when user switches tabs to update human_current_page + */ + async _setupCurrentPageChangeListeners() { + if (!this.browser_context) { + return; + } + // Listen for page events to track which page the user is viewing + this.browser_context.on?.('page', (page) => { + this.logger.debug(`New page created: ${page.url?.() || 'about:blank'}`); + // Note: 'visibilitychange' is not a standard Playwright page event + // Visibility tracking would need to be implemented differently + // (e.g., through page.evaluate polling or browser context events) + // Track new page + if (page.url && !page.url().startsWith('about:')) { + this.human_current_page = page; + } + }); + } + /** + * Callback when tab visibility changes + * Updates human_current_page to reflect which tab the user is viewing + */ + _onTabVisibilityChange(page) { + try { + // Check if page is visible + page + .evaluate?.(() => document.visibilityState === 'visible') + .then((isVisible) => { + if (isVisible) { + this.logger.debug(`Tab became visible: ${page.url?.() || 'unknown'}`); + this.human_current_page = page; + } + }) + .catch(() => { + // Ignore errors from closed pages + }); + } + catch { + // Ignore errors + } + } + // endregion + // region - Process Management + /** + * Normalize pid values before issuing process operations. + */ + _normalizePid(pid) { + if (!Number.isSafeInteger(pid) || pid <= 0) { + this.logger.debug(`Skipping process operation for invalid pid: ${String(pid)}`); + return null; + } + return pid; + } + /** + * Kill all child processes spawned by this browser session + */ + async _killChildProcesses() { + if (this._childProcesses.size === 0) { + return; + } + this.logger.debug(`Killing ${this._childProcesses.size} child processes`); + for (const trackedPid of this._childProcesses) { + const pid = this._normalizePid(trackedPid); + if (!pid) { + continue; + } + try { + process.kill(pid, 'SIGTERM'); + this.logger.debug(`Sent SIGTERM to process ${pid}`); + await new Promise((resolve) => setTimeout(resolve, 500)); + try { + process.kill(pid, 0); + process.kill(pid, 'SIGKILL'); + this.logger.debug(`Sent SIGKILL to process ${pid}`); + } + catch { + // Process is dead, ignore + } + } + catch (error) { + this.logger.debug(`Could not kill process ${pid}: ${error.message}`); + } + } + this._childProcesses.clear(); + } + /** + * Terminate the browser process and all its children + */ + async _terminateBrowserProcess() { + const browserPid = this._normalizePid(this.browser_pid); + if (!browserPid) { + return; + } + try { + this.logger.debug(`Terminating browser process ${browserPid}`); + if (process.platform === 'win32') { + await execFileAsync('taskkill', [ + '/PID', + String(browserPid), + '/T', + '/F', + ]).catch(() => { + // Ignore errors if process already dead + }); + } + else { + try { + process.kill(-browserPid, 'SIGTERM'); + await new Promise((resolve) => setTimeout(resolve, 1000)); + try { + process.kill(-browserPid, 0); + process.kill(-browserPid, 'SIGKILL'); + } + catch { + // Process is dead + } + } + catch { + try { + process.kill(browserPid, 'SIGTERM'); + await new Promise((resolve) => setTimeout(resolve, 1000)); + process.kill(browserPid, 'SIGKILL'); + } + catch { + // Process doesn't exist + } + } + } + } + catch (error) { + this.logger.debug(`Error terminating browser process: ${error.message}`); + } + } + /** + * Get child processes of a given PID + * Cross-platform implementation using ps on Unix-like systems and WMIC on Windows + */ + async _getChildProcesses(pid) { + const normalizedPid = this._normalizePid(pid); + if (!normalizedPid) { + return []; + } + try { + if (process.platform === 'win32') { + const { stdout } = await execFileAsync('wmic', [ + 'process', + 'where', + `ParentProcessId=${normalizedPid}`, + 'get', + 'ProcessId', + ]); + const pids = stdout + .split('\n') + .slice(1) + .map((line) => parseInt(line.trim(), 10)) + .filter((p) => Number.isFinite(p)); + return pids; + } + const { stdout } = await execFileAsync('ps', [ + '-o', + 'pid=', + '--ppid', + String(normalizedPid), + ]); + const pids = stdout + .split('\n') + .map((line) => parseInt(line.trim(), 10)) + .filter((p) => Number.isFinite(p)); + return pids; + } + catch { + return []; + } + } + /** + * Track a child process + */ + _trackChildProcess(pid) { + const normalizedPid = this._normalizePid(pid); + if (normalizedPid) { + this._childProcesses.add(normalizedPid); + } + } + /** + * Untrack a child process + */ + _untrackChildProcess(pid) { + const normalizedPid = this._normalizePid(pid); + if (normalizedPid) { + this._childProcesses.delete(normalizedPid); + } + } + // region: Loading Animations + /** + * Show DVD screensaver loading animation + * Returns a function to stop the animation + * + * @param message - Message to display (default: 'Loading...') + * @param fps - Frames per second (default: 10) + * @returns Function to stop the animation + * + * @example + * const stopAnimation = this._showDvdScreensaverLoadingAnimation('Loading page...'); + * await someLongOperation(); + * stopAnimation(); + */ + _showDvdScreensaverLoadingAnimation(message = 'Loading...', fps = 10) { + return showDVDScreensaver(message, fps); + } + /** + * Show simple spinner loading animation + * Returns a function to stop the animation + * + * @param message - Message to display (default: 'Loading...') + * @param fps - Frames per second (default: 10) + * @returns Function to stop the animation + * + * @example + * const stopSpinner = this._showSpinnerLoadingAnimation('Processing...'); + * await someLongOperation(); + * stopSpinner(); + */ + _showSpinnerLoadingAnimation(message = 'Loading...', fps = 10) { + return showSpinner(message, fps); + } + /** + * Execute an async operation with DVD screensaver animation + * + * @param operation - Async operation to execute + * @param message - Message to display during operation + * @returns Result of the operation + * + * @example + * const page = await this._withDvdScreensaver( + * async () => await this.browser_context!.newPage(), + * 'Opening new page...' + * ); + */ + async _withDvdScreensaver(operation, message = 'Loading...') { + return withDVDScreensaver(operation, message); + } +} +export { DEFAULT_BROWSER_PROFILE }; diff --git a/dist/browser/types.d.ts b/dist/browser/types.d.ts new file mode 100644 index 00000000..5d4dce05 --- /dev/null +++ b/dist/browser/types.d.ts @@ -0,0 +1,1181 @@ +import type { Browser as PlaywrightBrowser, BrowserContextOptions, BrowserContext as PlaywrightBrowserContext, ElementHandle as PlaywrightElementHandle, FrameLocator as PlaywrightFrameLocator, LaunchOptions, Page as PlaywrightPage, Locator as PlaywrightLocator } from 'playwright'; +export type Browser = PlaywrightBrowser; +export type BrowserContext = PlaywrightBrowserContext; +export type Page = PlaywrightPage; +export type ElementHandle = PlaywrightElementHandle; +export type FrameLocator = PlaywrightFrameLocator; +export type Locator = PlaywrightLocator; +export type PlaywrightModule = typeof import('playwright'); +export type Playwright = PlaywrightModule; +export type PlaywrightOrPatchright = PlaywrightModule; +export declare const async_playwright: () => Promise<{ + default: typeof import("playwright"); + errors: typeof import("playwright").errors; + devices: { + [key: string]: { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Blackberry PlayBook": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Blackberry PlayBook landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "BlackBerry Z30": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "BlackBerry Z30 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy Note 3": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy Note 3 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy Note II": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy Note II landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy S III": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy S III landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy S5": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy S5 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy S8": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy S8 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy S9+": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy S9+ landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy S24": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy S24 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy A55": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy A55 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy Tab S4": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy Tab S4 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy Tab S9": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Galaxy Tab S9 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPad (gen 5)": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPad (gen 5) landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPad (gen 6)": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPad (gen 6) landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPad (gen 7)": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPad (gen 7) landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPad (gen 11)": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPad (gen 11) landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPad Mini": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPad Mini landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPad Pro 11": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPad Pro 11 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 6": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 6 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 6 Plus": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 6 Plus landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 7": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 7 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 7 Plus": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 7 Plus landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 8": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 8 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 8 Plus": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 8 Plus landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone SE": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone SE landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone SE (3rd gen)": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone SE (3rd gen) landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone X": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone X landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone XR": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone XR landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 11": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 11 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 11 Pro": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 11 Pro landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 11 Pro Max": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 11 Pro Max landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 12": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 12 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 12 Pro": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 12 Pro landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 12 Pro Max": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 12 Pro Max landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 12 Mini": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 12 Mini landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 13": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 13 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 13 Pro": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 13 Pro landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 13 Pro Max": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 13 Pro Max landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 13 Mini": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 13 Mini landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 14": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 14 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 14 Plus": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 14 Plus landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 14 Pro": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 14 Pro landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 14 Pro Max": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 14 Pro Max landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 15": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 15 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 15 Plus": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 15 Plus landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 15 Pro": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 15 Pro landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 15 Pro Max": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "iPhone 15 Pro Max landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Kindle Fire HDX": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Kindle Fire HDX landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "LG Optimus L70": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "LG Optimus L70 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Microsoft Lumia 550": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Microsoft Lumia 550 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Microsoft Lumia 950": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Microsoft Lumia 950 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 10": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 10 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 4": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 4 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 5": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 5 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 5X": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 5X landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 6": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 6 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 6P": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 6P landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 7": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nexus 7 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nokia Lumia 520": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nokia Lumia 520 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nokia N9": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Nokia N9 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 2": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 2 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 2 XL": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 2 XL landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 3": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 3 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 4": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 4 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 4a (5G)": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 4a (5G) landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 5": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 5 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 7": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Pixel 7 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Moto G4": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Moto G4 landscape": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Desktop Chrome HiDPI": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Desktop Edge HiDPI": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Desktop Firefox HiDPI": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Desktop Safari": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Desktop Chrome": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Desktop Edge": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + "Desktop Firefox": { + viewport: import("playwright").ViewportSize; + userAgent: string; + deviceScaleFactor: number; + isMobile: boolean; + hasTouch: boolean; + defaultBrowserType: "chromium" | "firefox" | "webkit"; + }; + }; + _electron: import("playwright").Electron; + _android: import("playwright").Android; + chromium: import("playwright").BrowserType; + firefox: import("playwright").BrowserType; + request: import("playwright").APIRequest; + selectors: import("playwright").Selectors; + webkit: import("playwright").BrowserType; +}>; +export type ProxySettings = NonNullable; +export type HttpCredentials = NonNullable; +export type Geolocation = NonNullable; +export type ViewportSize = NonNullable; +export type StorageState = Exclude; +export type ClientCertificate = NonNullable[number]>; diff --git a/dist/browser/types.js b/dist/browser/types.js new file mode 100644 index 00000000..384be271 --- /dev/null +++ b/dist/browser/types.js @@ -0,0 +1 @@ +export const async_playwright = async () => import('playwright'); diff --git a/dist/browser/utils.d.ts b/dist/browser/utils.d.ts new file mode 100644 index 00000000..6da95b41 --- /dev/null +++ b/dist/browser/utils.d.ts @@ -0,0 +1 @@ +export declare const normalize_url: (url: string) => string; diff --git a/dist/browser/utils.js b/dist/browser/utils.js new file mode 100644 index 00000000..22f9a1fe --- /dev/null +++ b/dist/browser/utils.js @@ -0,0 +1,19 @@ +const SPECIAL_PROTOCOLS = [ + 'about:', + 'mailto:', + 'tel:', + 'ftp:', + 'file:', + 'data:', + 'javascript:', +]; +export const normalize_url = (url) => { + const normalized = url.trim(); + if (normalized.includes('://')) { + return normalized; + } + if (SPECIAL_PROTOCOLS.some((protocol) => normalized.startsWith(protocol))) { + return normalized; + } + return `https://${normalized}`; +}; diff --git a/dist/browser/views.d.ts b/dist/browser/views.d.ts new file mode 100644 index 00000000..787d1351 --- /dev/null +++ b/dist/browser/views.d.ts @@ -0,0 +1,117 @@ +import { DOMState } from '../dom/views.js'; +import type { DOMHistoryElement } from '../dom/history-tree-processor/view.js'; +export declare const PLACEHOLDER_4PX_SCREENSHOT = "iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAFElEQVR4nGP8//8/AwwwMSAB3BwAlm4DBfIlvvkAAAAASUVORK5CYII="; +export interface TabInfo { + page_id: number; + tab_id?: string; + target_id?: string; + url: string; + title: string; + parent_page_id?: number | null; +} +export interface PageInfo { + viewport_width: number; + viewport_height: number; + page_width: number; + page_height: number; + scroll_x: number; + scroll_y: number; + pixels_above: number; + pixels_below: number; + pixels_left: number; + pixels_right: number; +} +export interface NetworkRequest { + url: string; + method?: string; + loading_duration_ms?: number; + resource_type?: string | null; +} +export interface PaginationButton { + button_type: string; + backend_node_id: number; + text: string; + selector: string; + is_disabled?: boolean; +} +interface BrowserStateSummaryInit { + url: string; + title: string; + tabs: TabInfo[]; + screenshot?: string | null; + page_info?: PageInfo | null; + pixels_above?: number; + pixels_below?: number; + browser_errors?: string[]; + is_pdf_viewer?: boolean; + loading_status?: string | null; + recent_events?: string | null; + pending_network_requests?: NetworkRequest[]; + pagination_buttons?: PaginationButton[]; + closed_popup_messages?: string[]; +} +export declare class BrowserStateSummary extends DOMState { + url: string; + title: string; + tabs: TabInfo[]; + screenshot: string | null; + page_info: PageInfo | null; + pixels_above: number; + pixels_below: number; + browser_errors: string[]; + is_pdf_viewer: boolean; + loading_status: string | null; + recent_events: string | null; + pending_network_requests: NetworkRequest[]; + pagination_buttons: PaginationButton[]; + closed_popup_messages: string[]; + constructor(dom_state: DOMState, init: BrowserStateSummaryInit); +} +export declare class BrowserStateHistory { + url: string; + title: string; + tabs: TabInfo[]; + interacted_element: Array; + screenshot_path: string | null; + constructor(url: string, title: string, tabs: TabInfo[], interacted_element: Array, screenshot_path?: string | null); + get_screenshot(): string | null; + to_dict(): { + tabs: TabInfo[]; + screenshot_path: string | null; + interacted_element: ({ + tag_name: string; + xpath: string; + highlight_index: number | null; + entire_parent_branch_path: string[]; + attributes: Record; + shadow_root: boolean; + css_selector: string | null; + page_coordinates: import("../index.js").CoordinateSet | null; + viewport_coordinates: import("../index.js").CoordinateSet | null; + viewport_info: import("../index.js").ViewportInfo | null; + element_hash: string | null; + stable_hash: string | null; + ax_name: string | null; + } | null)[]; + url: string; + title: string; + }; +} +export interface BrowserErrorInit { + message: string; + short_term_memory?: string | null; + long_term_memory?: string | null; + details?: Record | null; + event?: unknown; +} +export declare class BrowserError extends Error { + short_term_memory: string | null; + long_term_memory: string | null; + details: Record | null; + while_handling_event: unknown; + constructor(messageOrInit: string | BrowserErrorInit, options?: Omit); + toString(): string; +} +export declare class URLNotAllowedError extends BrowserError { +} +export {}; diff --git a/dist/browser/views.js b/dist/browser/views.js new file mode 100644 index 00000000..1769eb8c --- /dev/null +++ b/dist/browser/views.js @@ -0,0 +1,104 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { DOMState } from '../dom/views.js'; +export const PLACEHOLDER_4PX_SCREENSHOT = 'iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAFElEQVR4nGP8//8/AwwwMSAB3BwAlm4DBfIlvvkAAAAASUVORK5CYII='; +export class BrowserStateSummary extends DOMState { + url; + title; + tabs; + screenshot; + page_info; + pixels_above; + pixels_below; + browser_errors; + is_pdf_viewer; + loading_status; + recent_events; + pending_network_requests; + pagination_buttons; + closed_popup_messages; + constructor(dom_state, init) { + super(dom_state.element_tree, dom_state.selector_map); + this.url = init.url; + this.title = init.title; + this.tabs = init.tabs; + this.screenshot = init.screenshot ?? null; + this.page_info = init.page_info ?? null; + this.pixels_above = init.pixels_above ?? 0; + this.pixels_below = init.pixels_below ?? 0; + this.browser_errors = init.browser_errors ?? []; + this.is_pdf_viewer = init.is_pdf_viewer ?? false; + this.loading_status = init.loading_status ?? null; + this.recent_events = init.recent_events ?? null; + this.pending_network_requests = init.pending_network_requests ?? []; + this.pagination_buttons = init.pagination_buttons ?? []; + this.closed_popup_messages = init.closed_popup_messages ?? []; + } +} +export class BrowserStateHistory { + url; + title; + tabs; + interacted_element; + screenshot_path; + constructor(url, title, tabs, interacted_element, screenshot_path = null) { + this.url = url; + this.title = title; + this.tabs = tabs; + this.interacted_element = interacted_element; + this.screenshot_path = screenshot_path; + } + get_screenshot() { + if (!this.screenshot_path) { + return null; + } + const resolved = path.resolve(this.screenshot_path); + if (!fs.existsSync(resolved)) { + return null; + } + try { + const data = fs.readFileSync(resolved); + return data.toString('base64'); + } + catch { + return null; + } + } + to_dict() { + return { + tabs: this.tabs, + screenshot_path: this.screenshot_path, + interacted_element: this.interacted_element.map((element) => element?.to_dict?.() ?? null), + url: this.url, + title: this.title, + }; + } +} +export class BrowserError extends Error { + short_term_memory; + long_term_memory; + details; + while_handling_event; + constructor(messageOrInit, options) { + const init = typeof messageOrInit === 'string' + ? { message: messageOrInit, ...(options ?? {}) } + : messageOrInit; + super(init.message); + this.name = 'BrowserError'; + this.short_term_memory = init.short_term_memory ?? null; + this.long_term_memory = init.long_term_memory ?? null; + this.details = init.details ?? null; + this.while_handling_event = init.event ?? null; + } + toString() { + if (this.details) { + return `${this.message} (${JSON.stringify(this.details)})`; + } + if (this.while_handling_event) { + return `${this.message} (while handling: ${String(this.while_handling_event)})`; + } + return this.message; + } +} +export class URLNotAllowedError extends BrowserError { +} diff --git a/dist/browser/watchdogs/aboutblank-watchdog.d.ts b/dist/browser/watchdogs/aboutblank-watchdog.d.ts new file mode 100644 index 00000000..b2fea2e4 --- /dev/null +++ b/dist/browser/watchdogs/aboutblank-watchdog.d.ts @@ -0,0 +1,12 @@ +import { AboutBlankDVDScreensaverShownEvent, BrowserStopEvent, BrowserStoppedEvent, NavigateToUrlEvent, TabClosedEvent, TabCreatedEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export declare class AboutBlankWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof BrowserStopEvent | typeof BrowserStoppedEvent | typeof TabClosedEvent)[]; + static EMITS: (typeof NavigateToUrlEvent | typeof AboutBlankDVDScreensaverShownEvent)[]; + private _stopping; + on_BrowserStopEvent(): Promise; + on_BrowserStoppedEvent(): Promise; + on_TabClosedEvent(): Promise; + on_TabCreatedEvent(event: TabCreatedEvent): Promise; + private _injectDvdScreensaverOverlay; +} diff --git a/dist/browser/watchdogs/aboutblank-watchdog.js b/dist/browser/watchdogs/aboutblank-watchdog.js new file mode 100644 index 00000000..91f8c8af --- /dev/null +++ b/dist/browser/watchdogs/aboutblank-watchdog.js @@ -0,0 +1,131 @@ +import { AboutBlankDVDScreensaverShownEvent, BrowserStopEvent, BrowserStoppedEvent, NavigateToUrlEvent, TabClosedEvent, TabCreatedEvent, } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class AboutBlankWatchdog extends BaseWatchdog { + static LISTENS_TO = [ + BrowserStopEvent, + BrowserStoppedEvent, + TabClosedEvent, + TabCreatedEvent, + ]; + static EMITS = [ + NavigateToUrlEvent, + AboutBlankDVDScreensaverShownEvent, + ]; + _stopping = false; + async on_BrowserStopEvent() { + this._stopping = true; + } + async on_BrowserStoppedEvent() { + this._stopping = true; + } + async on_TabClosedEvent() { + if (this._stopping) { + return; + } + if (this.browser_session.tabs.length > 0) { + return; + } + await this.event_bus.dispatch(new NavigateToUrlEvent({ + url: 'about:blank', + new_tab: true, + })); + } + async on_TabCreatedEvent(event) { + if (this._stopping) { + return; + } + if (event.url !== 'about:blank') { + return; + } + let injectionError = null; + try { + await this._injectDvdScreensaverOverlay(); + } + catch (error) { + injectionError = + error.message || 'DVD overlay injection failed'; + } + await this.event_bus.dispatch(new AboutBlankDVDScreensaverShownEvent({ + target_id: event.target_id, + error: injectionError, + })); + } + async _injectDvdScreensaverOverlay() { + const page = await this.browser_session.get_current_page(); + if (!page?.evaluate || !page.url) { + return; + } + const currentUrl = page.url(); + if (currentUrl !== 'about:blank') { + return; + } + await page.evaluate(() => { + if (window.__dvdAnimationRunning) { + return; + } + window.__dvdAnimationRunning = true; + const existing = document.getElementById('pretty-loading-animation'); + if (existing) { + return; + } + const ensureBody = () => { + if (!document.body) { + return false; + } + const overlay = document.createElement('div'); + overlay.id = 'pretty-loading-animation'; + overlay.style.position = 'fixed'; + overlay.style.top = '0'; + overlay.style.left = '0'; + overlay.style.width = '100vw'; + overlay.style.height = '100vh'; + overlay.style.background = '#000'; + overlay.style.zIndex = '99999'; + overlay.style.overflow = 'hidden'; + const img = document.createElement('img'); + img.src = 'https://cf.browser-use.com/logo.svg'; + img.alt = 'Browser-Use'; + img.style.width = '180px'; + img.style.height = 'auto'; + img.style.position = 'absolute'; + img.style.left = '0px'; + img.style.top = '0px'; + img.style.opacity = '0.85'; + overlay.appendChild(img); + document.body.appendChild(overlay); + let x = Math.random() * Math.max(20, window.innerWidth - 200); + let y = Math.random() * Math.max(20, window.innerHeight - 120); + let dx = 1.4; + let dy = 1.2; + const animate = () => { + if (!document.getElementById('pretty-loading-animation')) { + window.__dvdAnimationRunning = false; + return; + } + const imgWidth = img.offsetWidth || 180; + const imgHeight = img.offsetHeight || 80; + x += dx; + y += dy; + if (x <= 0 || x + imgWidth >= window.innerWidth) { + dx = -dx; + x = Math.max(0, Math.min(x, window.innerWidth - imgWidth)); + } + if (y <= 0 || y + imgHeight >= window.innerHeight) { + dy = -dy; + y = Math.max(0, Math.min(y, window.innerHeight - imgHeight)); + } + img.style.left = `${x}px`; + img.style.top = `${y}px`; + requestAnimationFrame(animate); + }; + requestAnimationFrame(animate); + return true; + }; + if (!ensureBody() && document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', ensureBody, { + once: true, + }); + } + }); + } +} diff --git a/dist/browser/watchdogs/base.d.ts b/dist/browser/watchdogs/base.d.ts new file mode 100644 index 00000000..38c63629 --- /dev/null +++ b/dist/browser/watchdogs/base.d.ts @@ -0,0 +1,21 @@ +import type { EventBus, EventPayload, EventTypeReference } from '../../event-bus.js'; +import type { BrowserSession } from '../session.js'; +export interface BaseWatchdogInit { + browser_session: BrowserSession; + event_bus?: EventBus; +} +export declare abstract class BaseWatchdog { + static LISTENS_TO: EventTypeReference[]; + static EMITS: EventTypeReference[]; + protected readonly browser_session: BrowserSession; + protected readonly event_bus: EventBus; + private _attached; + private _registeredHandlers; + constructor(init: BaseWatchdogInit); + get is_attached(): boolean; + attach_to_session(): void; + detach_from_session(): void; + protected onAttached(): void; + protected onDetached(): void; + private _collectHandlerMethods; +} diff --git a/dist/browser/watchdogs/base.js b/dist/browser/watchdogs/base.js new file mode 100644 index 00000000..e82306bb --- /dev/null +++ b/dist/browser/watchdogs/base.js @@ -0,0 +1,114 @@ +const resolveEventType = (eventTypeRef) => typeof eventTypeRef === 'string' ? eventTypeRef : eventTypeRef.name; +const LIFECYCLE_EVENT_NAMES = new Set([ + 'BrowserStartEvent', + 'BrowserStopEvent', + 'BrowserStoppedEvent', + 'BrowserLaunchEvent', + 'BrowserKillEvent', + 'BrowserConnectedEvent', + 'BrowserReconnectingEvent', + 'BrowserReconnectedEvent', + 'BrowserErrorEvent', +]); +const createConnectionError = (message) => { + const error = new Error(message); + error.name = 'ConnectionError'; + return error; +}; +export class BaseWatchdog { + static LISTENS_TO = []; + static EMITS = []; + browser_session; + event_bus; + _attached = false; + _registeredHandlers = []; + constructor(init) { + this.browser_session = init.browser_session; + this.event_bus = init.event_bus ?? init.browser_session.event_bus; + } + get is_attached() { + return this._attached; + } + attach_to_session() { + if (this._attached) { + throw new Error(`[${this.constructor.name}] attach_to_session() called twice`); + } + const handlerMethods = this._collectHandlerMethods(); + const declaredListenEvents = new Set(this.constructor.LISTENS_TO.map(resolveEventType)); + const registeredEventTypes = new Set(); + for (const methodName of handlerMethods) { + const event_type = methodName.slice(3); + if (declaredListenEvents.size > 0 && + !declaredListenEvents.has(event_type)) { + throw new Error(`[${this.constructor.name}] Handler ${methodName} listens to ${event_type} but ${event_type} is not declared in LISTENS_TO`); + } + const handler_id = `${this.constructor.name}.${methodName}`; + const method = this[methodName]; + if (typeof method !== 'function') { + continue; + } + const bound = method.bind(this); + const wrapped = async (event) => { + if (!LIFECYCLE_EVENT_NAMES.has(event_type) && + this.browser_session.should_gate_watchdog_events && + !this.browser_session.is_cdp_connected) { + if (this.browser_session.is_reconnecting) { + await this.browser_session.wait_for_reconnect(); + if (!this.browser_session.is_cdp_connected) { + throw createConnectionError(`[${this.constructor.name}.${methodName}] Reconnection failed; browser connection is still unavailable`); + } + } + else { + this.browser_session.logger.debug(`[${this.constructor.name}.${methodName}] Skipped because browser connection is not available`); + return null; + } + } + return await bound(event); + }; + this.event_bus.on(event_type, wrapped, { handler_id }); + this._registeredHandlers.push({ event_type, handler_id }); + registeredEventTypes.add(event_type); + } + if (declaredListenEvents.size > 0) { + const missing = [...declaredListenEvents].filter((eventType) => !registeredEventTypes.has(eventType)); + if (missing.length > 0) { + throw new Error(`[${this.constructor.name}] LISTENS_TO declares ${missing.join(', ')} but no matching on_ handlers were found`); + } + } + this._attached = true; + this.onAttached(); + } + detach_from_session() { + if (!this._attached) { + return; + } + for (const { event_type, handler_id } of this._registeredHandlers) { + this.event_bus.off(event_type, handler_id); + } + this._registeredHandlers = []; + this._attached = false; + this.onDetached(); + } + onAttached() { } + onDetached() { } + _collectHandlerMethods() { + const methodNames = new Set(); + let prototype = Object.getPrototypeOf(this); + while (prototype && prototype !== BaseWatchdog.prototype) { + for (const name of Object.getOwnPropertyNames(prototype)) { + if (!name.startsWith('on_')) { + continue; + } + if (name.length <= 3) { + continue; + } + const descriptor = Object.getOwnPropertyDescriptor(prototype, name); + if (typeof descriptor?.value === 'function') { + methodNames.add(name); + } + } + prototype = Object.getPrototypeOf(prototype); + } + return [...methodNames]; + } +} diff --git a/dist/browser/watchdogs/captcha-watchdog.d.ts b/dist/browser/watchdogs/captcha-watchdog.d.ts new file mode 100644 index 00000000..c4145ce7 --- /dev/null +++ b/dist/browser/watchdogs/captcha-watchdog.d.ts @@ -0,0 +1,26 @@ +import { BrowserConnectedEvent, BrowserStoppedEvent, CaptchaSolverFinishedEvent, CaptchaSolverStartedEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +type CaptchaResultType = 'success' | 'failed' | 'timeout' | 'unknown'; +export interface CaptchaWaitResult { + waited: boolean; + vendor: string; + url: string; + duration_ms: number; + result: CaptchaResultType; +} +export declare class CaptchaWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof BrowserConnectedEvent | typeof BrowserStoppedEvent)[]; + static EMITS: (typeof CaptchaSolverStartedEvent | typeof CaptchaSolverFinishedEvent)[]; + private _cdpSession; + private _handlers; + private _captchaSolving; + private _captchaInfo; + private _captchaResult; + private _captchaDurationMs; + private _waiters; + on_BrowserConnectedEvent(): Promise; + on_BrowserStoppedEvent(): Promise; + protected onDetached(): void; + wait_if_captcha_solving(timeoutSeconds?: number): Promise; +} +export {}; diff --git a/dist/browser/watchdogs/captcha-watchdog.js b/dist/browser/watchdogs/captcha-watchdog.js new file mode 100644 index 00000000..ac22f6ac --- /dev/null +++ b/dist/browser/watchdogs/captcha-watchdog.js @@ -0,0 +1,151 @@ +import { BrowserConnectedEvent, BrowserStoppedEvent, CaptchaSolverFinishedEvent, CaptchaSolverStartedEvent, } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class CaptchaWatchdog extends BaseWatchdog { + static LISTENS_TO = [BrowserConnectedEvent, BrowserStoppedEvent]; + static EMITS = [ + CaptchaSolverStartedEvent, + CaptchaSolverFinishedEvent, + ]; + _cdpSession = null; + _handlers = []; + _captchaSolving = false; + _captchaInfo = { + vendor: 'unknown', + url: '', + target_id: '', + }; + _captchaResult = 'unknown'; + _captchaDurationMs = 0; + _waiters = new Set(); + async on_BrowserConnectedEvent() { + if (this._cdpSession) { + return; + } + const page = await this.browser_session.get_current_page(); + if (!page) { + return; + } + try { + const cdpSession = (await this.browser_session.get_or_create_cdp_session(page)); + const onStarted = (payload) => { + this._captchaSolving = true; + this._captchaResult = 'unknown'; + this._captchaDurationMs = 0; + this._captchaInfo = { + vendor: String(payload?.vendor ?? 'unknown'), + url: String(payload?.url ?? ''), + target_id: String(payload?.targetId ?? ''), + }; + void this.event_bus.dispatch(new CaptchaSolverStartedEvent({ + target_id: this._captchaInfo.target_id, + vendor: this._captchaInfo.vendor, + url: this._captchaInfo.url, + started_at: Number(payload?.startedAt ?? Date.now()), + })); + }; + const onFinished = (payload) => { + this._captchaSolving = false; + this._captchaDurationMs = Number(payload?.durationMs ?? 0); + this._captchaResult = payload?.success ? 'success' : 'failed'; + const vendor = String(payload?.vendor ?? this._captchaInfo.vendor); + const url = String(payload?.url ?? this._captchaInfo.url); + const targetId = String(payload?.targetId ?? this._captchaInfo.target_id); + for (const resolve of this._waiters) { + resolve(); + } + this._waiters.clear(); + void this.event_bus.dispatch(new CaptchaSolverFinishedEvent({ + target_id: targetId, + vendor, + url, + duration_ms: this._captchaDurationMs, + finished_at: Number(payload?.finishedAt ?? Date.now()), + success: Boolean(payload?.success), + })); + }; + cdpSession.on?.('BrowserUse.captchaSolverStarted', onStarted); + cdpSession.on?.('BrowserUse.captchaSolverFinished', onFinished); + this._cdpSession = cdpSession; + this._handlers = [ + { event: 'BrowserUse.captchaSolverStarted', handler: onStarted }, + { event: 'BrowserUse.captchaSolverFinished', handler: onFinished }, + ]; + } + catch (error) { + this.browser_session.logger.debug(`CaptchaWatchdog monitoring unavailable: ${error.message}`); + await this.on_BrowserStoppedEvent(); + } + } + async on_BrowserStoppedEvent() { + this._captchaSolving = false; + this._captchaResult = 'unknown'; + this._captchaDurationMs = 0; + this._captchaInfo = { vendor: 'unknown', url: '', target_id: '' }; + for (const resolve of this._waiters) { + resolve(); + } + this._waiters.clear(); + if (!this._cdpSession) { + return; + } + for (const { event, handler } of this._handlers) { + this._cdpSession.off?.(event, handler); + } + this._handlers = []; + try { + await this._cdpSession.detach?.(); + } + catch { + // Ignore detach errors during shutdown. + } + finally { + this._cdpSession = null; + } + } + onDetached() { + void this.on_BrowserStoppedEvent(); + } + async wait_if_captcha_solving(timeoutSeconds = 120) { + if (!this._captchaSolving) { + return null; + } + const vendor = this._captchaInfo.vendor; + const url = this._captchaInfo.url; + const timeoutMs = Math.max(0, timeoutSeconds * 1000); + try { + await new Promise((resolve, reject) => { + const onResolved = () => { + cleanup(); + resolve(); + }; + const timeoutHandle = setTimeout(() => { + cleanup(); + reject(new Error('timeout')); + }, timeoutMs); + const cleanup = () => { + clearTimeout(timeoutHandle); + this._waiters.delete(onResolved); + }; + this._waiters.add(onResolved); + }); + return { + waited: true, + vendor, + url, + duration_ms: this._captchaDurationMs, + result: this._captchaResult, + }; + } + catch { + this._captchaSolving = false; + this._waiters.clear(); + return { + waited: true, + vendor, + url, + duration_ms: timeoutMs, + result: 'timeout', + }; + } + } +} diff --git a/dist/browser/watchdogs/cdp-session-watchdog.d.ts b/dist/browser/watchdogs/cdp-session-watchdog.d.ts new file mode 100644 index 00000000..4ee134d2 --- /dev/null +++ b/dist/browser/watchdogs/cdp-session-watchdog.d.ts @@ -0,0 +1,14 @@ +import { BrowserConnectedEvent, BrowserStoppedEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export declare class CDPSessionWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof BrowserConnectedEvent | typeof BrowserStoppedEvent)[]; + private _rootCdpSession; + private _listeners; + private _knownTargets; + on_BrowserConnectedEvent(): Promise; + on_BrowserStoppedEvent(): Promise; + protected onDetached(): void; + private _ensureCdpMonitoring; + private _teardownCdpMonitoring; + private _dispatchEventSafely; +} diff --git a/dist/browser/watchdogs/cdp-session-watchdog.js b/dist/browser/watchdogs/cdp-session-watchdog.js new file mode 100644 index 00000000..405904ac --- /dev/null +++ b/dist/browser/watchdogs/cdp-session-watchdog.js @@ -0,0 +1,177 @@ +import { BrowserConnectedEvent, BrowserStoppedEvent, NavigationCompleteEvent, TabClosedEvent, TabCreatedEvent, } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class CDPSessionWatchdog extends BaseWatchdog { + static LISTENS_TO = [BrowserConnectedEvent, BrowserStoppedEvent]; + _rootCdpSession = null; + _listeners = []; + _knownTargets = new Map(); + async on_BrowserConnectedEvent() { + await this._ensureCdpMonitoring(); + } + async on_BrowserStoppedEvent() { + await this._teardownCdpMonitoring(); + } + onDetached() { + void this._teardownCdpMonitoring(); + } + async _ensureCdpMonitoring() { + if (this._rootCdpSession) { + return; + } + if (!this.browser_session.browser_context?.newCDPSession) { + return; + } + const page = await this.browser_session.get_current_page(); + if (!page) { + return; + } + try { + const cdpSession = (await this.browser_session.get_or_create_cdp_session(page)); + this._rootCdpSession = cdpSession; + await cdpSession.send?.('Target.setDiscoverTargets', { + discover: true, + filter: [{ type: 'page' }, { type: 'iframe' }], + }); + const targetsPayload = await cdpSession.send?.('Target.getTargets'); + const targetInfos = Array.isArray(targetsPayload?.targetInfos) + ? targetsPayload.targetInfos + : []; + for (const targetInfo of targetInfos) { + const target_id = String(targetInfo?.targetId ?? ''); + if (!target_id) { + continue; + } + this.browser_session.session_manager.handle_target_info_changed({ + target_id, + target_type: typeof targetInfo?.type === 'string' ? targetInfo.type : 'page', + url: typeof targetInfo?.url === 'string' ? targetInfo.url : '', + title: typeof targetInfo?.title === 'string' ? targetInfo.title : '', + }); + this._knownTargets.set(target_id, { + target_type: typeof targetInfo?.type === 'string' ? targetInfo.type : 'page', + url: typeof targetInfo?.url === 'string' ? targetInfo.url : '', + }); + } + const onAttached = (payload) => { + const targetInfo = payload?.targetInfo ?? {}; + const target_id = String(targetInfo?.targetId ?? ''); + if (!target_id) { + return; + } + const target_type = typeof targetInfo?.type === 'string' ? targetInfo.type : 'page'; + const url = typeof targetInfo?.url === 'string' ? targetInfo.url : ''; + const isNewTarget = !this._knownTargets.has(target_id); + this._knownTargets.set(target_id, { + target_type, + url, + }); + this.browser_session.session_manager.handle_target_attached({ + target_id, + session_id: typeof payload?.sessionId === 'string' ? payload.sessionId : null, + target_type, + url, + title: typeof targetInfo?.title === 'string' ? targetInfo.title : '', + }); + if (isNewTarget && target_type === 'page') { + this._dispatchEventSafely(new TabCreatedEvent({ + target_id, + url, + })); + } + }; + const onDetached = (payload) => { + const target_id = String(payload?.targetId ?? ''); + if (!target_id) { + return; + } + const knownTarget = this._knownTargets.get(target_id); + this._knownTargets.delete(target_id); + this.browser_session.session_manager.handle_target_detached({ + target_id, + session_id: typeof payload?.sessionId === 'string' ? payload.sessionId : null, + }); + if (knownTarget?.target_type === 'page') { + this._dispatchEventSafely(new TabClosedEvent({ + target_id, + })); + } + }; + const onTargetInfoChanged = (payload) => { + const targetInfo = payload?.targetInfo ?? {}; + const target_id = String(targetInfo?.targetId ?? ''); + if (!target_id) { + return; + } + const knownTarget = this._knownTargets.get(target_id); + const target_type = typeof targetInfo?.type === 'string' + ? targetInfo.type + : (knownTarget?.target_type ?? 'page'); + const url = typeof targetInfo?.url === 'string' + ? targetInfo.url + : (knownTarget?.url ?? ''); + this._knownTargets.set(target_id, { + target_type, + url, + }); + this.browser_session.session_manager.handle_target_info_changed({ + target_id, + target_type, + url, + title: typeof targetInfo?.title === 'string' ? targetInfo.title : '', + }); + if (!knownTarget && target_type === 'page') { + this._dispatchEventSafely(new TabCreatedEvent({ + target_id, + url, + })); + return; + } + if (knownTarget?.target_type === 'page' && knownTarget.url !== url) { + this._dispatchEventSafely(new NavigationCompleteEvent({ + target_id, + url, + status: null, + error_message: null, + loading_status: null, + })); + } + }; + cdpSession.on?.('Target.attachedToTarget', onAttached); + cdpSession.on?.('Target.detachedFromTarget', onDetached); + cdpSession.on?.('Target.targetInfoChanged', onTargetInfoChanged); + this._listeners = [ + { event: 'Target.attachedToTarget', handler: onAttached }, + { event: 'Target.detachedFromTarget', handler: onDetached }, + { event: 'Target.targetInfoChanged', handler: onTargetInfoChanged }, + ]; + } + catch (error) { + this.browser_session.logger.debug(`CDPSessionWatchdog monitoring unavailable: ${error.message}`); + await this._teardownCdpMonitoring(); + } + } + async _teardownCdpMonitoring() { + if (!this._rootCdpSession) { + return; + } + for (const listener of this._listeners) { + this._rootCdpSession.off?.(listener.event, listener.handler); + } + this._listeners = []; + try { + await this._rootCdpSession.detach?.(); + } + catch { + // Ignore CDP detach errors during shutdown. + } + finally { + this._rootCdpSession = null; + this._knownTargets.clear(); + } + } + _dispatchEventSafely(event) { + void this.event_bus.dispatch(event).catch((error) => { + this.browser_session.logger.debug(`CDPSessionWatchdog failed to dispatch ${String(event?.event_name ?? 'event')}: ${error.message}`); + }); + } +} diff --git a/dist/browser/watchdogs/crash-watchdog.d.ts b/dist/browser/watchdogs/crash-watchdog.d.ts new file mode 100644 index 00000000..c6ce1fb6 --- /dev/null +++ b/dist/browser/watchdogs/crash-watchdog.d.ts @@ -0,0 +1,38 @@ +import { BrowserConnectedEvent, BrowserErrorEvent, BrowserStoppedEvent, TabClosedEvent, TargetCrashedEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export declare class CrashWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof BrowserConnectedEvent | typeof BrowserStoppedEvent | typeof TabClosedEvent)[]; + static EMITS: (typeof TargetCrashedEvent | typeof BrowserErrorEvent)[]; + private _pageListeners; + private _pendingRequests; + private _requestIds; + private _requestCounter; + private _healthInterval; + private _networkTimeoutMs; + private _healthCheckIntervalMs; + private _consecutiveUnresponsiveChecks; + private _unresponsiveThreshold; + private _monitoringInProgress; + on_BrowserConnectedEvent(): Promise; + on_TabCreatedEvent(): Promise; + on_TabClosedEvent(): Promise; + on_BrowserStoppedEvent(): Promise; + protected onDetached(): void; + private _attachToKnownPages; + private _dropDetachedPages; + private _detachAllPages; + private _attachPage; + private _detachPageListeners; + private _getKnownPages; + private _handlePageCrash; + private _resolveTargetId; + private _safePageUrl; + private _normalizeCrashError; + private _trackRequestStart; + private _trackRequestDone; + private _startHealthMonitor; + private _stopHealthMonitor; + private _runHealthCheck; + private _checkNetworkTimeouts; + private _checkPageResponsiveness; +} diff --git a/dist/browser/watchdogs/crash-watchdog.js b/dist/browser/watchdogs/crash-watchdog.js new file mode 100644 index 00000000..05a1d988 --- /dev/null +++ b/dist/browser/watchdogs/crash-watchdog.js @@ -0,0 +1,296 @@ +import { BrowserConnectedEvent, BrowserErrorEvent, BrowserStoppedEvent, TabClosedEvent, TabCreatedEvent, TargetCrashedEvent, } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class CrashWatchdog extends BaseWatchdog { + static LISTENS_TO = [ + BrowserConnectedEvent, + BrowserStoppedEvent, + TabCreatedEvent, + TabClosedEvent, + ]; + static EMITS = [TargetCrashedEvent, BrowserErrorEvent]; + _pageListeners = new Map(); + _pendingRequests = new Map(); + _requestIds = new WeakMap(); + _requestCounter = 0; + _healthInterval = null; + _networkTimeoutMs = 10_000; + _healthCheckIntervalMs = 5_000; + _consecutiveUnresponsiveChecks = 0; + _unresponsiveThreshold = 2; + _monitoringInProgress = false; + async on_BrowserConnectedEvent() { + this._attachToKnownPages(); + this._startHealthMonitor(); + } + async on_TabCreatedEvent() { + this._attachToKnownPages(); + } + async on_TabClosedEvent() { + this._dropDetachedPages(); + } + async on_BrowserStoppedEvent() { + this._detachAllPages(); + this._stopHealthMonitor(); + this._pendingRequests.clear(); + this._consecutiveUnresponsiveChecks = 0; + } + onDetached() { + this._detachAllPages(); + this._stopHealthMonitor(); + this._pendingRequests.clear(); + this._consecutiveUnresponsiveChecks = 0; + } + _attachToKnownPages() { + for (const page of this._getKnownPages()) { + this._attachPage(page); + } + } + _dropDetachedPages() { + const livePages = new Set(this._getKnownPages()); + for (const [page, listeners] of [...this._pageListeners.entries()]) { + if (livePages.has(page)) { + continue; + } + this._detachPageListeners(page, listeners); + this._pageListeners.delete(page); + } + } + _detachAllPages() { + for (const [page, listeners] of [...this._pageListeners.entries()]) { + this._detachPageListeners(page, listeners); + } + this._pageListeners.clear(); + } + _attachPage(page) { + if (this._pageListeners.has(page)) { + return; + } + if (typeof page?.on !== 'function') { + return; + } + const crashListener = (payload) => { + void this._handlePageCrash(page, payload); + }; + const requestListener = (payload) => { + this._trackRequestStart(payload); + }; + const requestFinishedListener = (payload) => { + this._trackRequestDone(payload); + }; + const requestFailedListener = (payload) => { + this._trackRequestDone(payload); + }; + const responseListener = () => { + // Keep hook for future detailed response-based crash heuristics. + }; + page.on('crash', crashListener); + page.on('request', requestListener); + page.on('requestfinished', requestFinishedListener); + page.on('requestfailed', requestFailedListener); + page.on('response', responseListener); + this._pageListeners.set(page, { + crash: crashListener, + request: requestListener, + requestfinished: requestFinishedListener, + requestfailed: requestFailedListener, + response: responseListener, + }); + } + _detachPageListeners(page, listeners) { + if (typeof page.off === 'function') { + page.off('crash', listeners.crash); + page.off('request', listeners.request); + page.off('requestfinished', listeners.requestfinished); + page.off('requestfailed', listeners.requestfailed); + page.off('response', listeners.response); + return; + } + if (typeof page.removeListener === 'function') { + page.removeListener('crash', listeners.crash); + page.removeListener('request', listeners.request); + page.removeListener('requestfinished', listeners.requestfinished); + page.removeListener('requestfailed', listeners.requestfailed); + page.removeListener('response', listeners.response); + } + } + _getKnownPages() { + const pagesFromContext = typeof this.browser_session.browser_context?.pages === 'function' + ? this.browser_session.browser_context.pages() + : []; + const activePage = this.browser_session.agent_current_page; + if (!activePage) { + return pagesFromContext; + } + if (pagesFromContext.includes(activePage)) { + return pagesFromContext; + } + return [...pagesFromContext, activePage]; + } + async _handlePageCrash(page, payload) { + const target_id = this._resolveTargetId(page); + const url = this._safePageUrl(page) ?? this.browser_session.active_tab?.url ?? ''; + const errorMessage = this._normalizeCrashError(payload); + await this.event_bus.dispatch(new TargetCrashedEvent({ + target_id, + error: errorMessage, + })); + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'TargetCrash', + message: errorMessage, + details: { + target_id, + url, + }, + })); + } + _resolveTargetId(page) { + const pageUrl = this._safePageUrl(page); + if (pageUrl) { + const tabByUrl = this.browser_session.tabs.find((tab) => tab.url === pageUrl && tab.target_id); + if (tabByUrl?.target_id) { + return tabByUrl.target_id; + } + } + const activeTargetId = this.browser_session.active_tab?.target_id; + if (activeTargetId) { + return activeTargetId; + } + return (this.browser_session.session_manager.get_focused_target_id() ?? + 'unknown_target'); + } + _safePageUrl(page) { + try { + return typeof page.url === 'function' ? page.url() : null; + } + catch { + return null; + } + } + _normalizeCrashError(payload) { + if (payload instanceof Error) { + return payload.message || 'Target crashed'; + } + if (typeof payload === 'string' && payload.trim().length > 0) { + return payload.trim(); + } + if (payload && typeof payload === 'object' && 'message' in payload) { + const candidate = payload.message; + if (typeof candidate === 'string' && candidate.trim().length > 0) { + return candidate.trim(); + } + } + return 'Target crashed'; + } + _trackRequestStart(request) { + const requestObject = request; + if (!requestObject || typeof requestObject !== 'object') { + return; + } + const requestId = `req-${this._requestCounter++}`; + const url = typeof request?.url === 'function' + ? request.url() + : (this.browser_session.active_tab?.url ?? ''); + const method = typeof request?.method === 'function' ? request.method() : 'GET'; + this._requestIds.set(requestObject, requestId); + this._pendingRequests.set(requestId, { + url: typeof url === 'string' ? url : '', + method: typeof method === 'string' ? method : 'GET', + started_at: Date.now(), + }); + } + _trackRequestDone(request) { + const requestObject = request; + if (!requestObject || typeof requestObject !== 'object') { + return; + } + const requestId = this._requestIds.get(requestObject); + if (!requestId) { + return; + } + this._requestIds.delete(requestObject); + this._pendingRequests.delete(requestId); + } + _startHealthMonitor() { + if (this._healthInterval) { + return; + } + this._healthInterval = setInterval(() => { + void this._runHealthCheck(); + }, this._healthCheckIntervalMs); + } + _stopHealthMonitor() { + if (!this._healthInterval) { + return; + } + clearInterval(this._healthInterval); + this._healthInterval = null; + } + async _runHealthCheck() { + if (this._monitoringInProgress) { + return; + } + this._monitoringInProgress = true; + try { + await this._checkNetworkTimeouts(); + await this._checkPageResponsiveness(); + } + finally { + this._monitoringInProgress = false; + } + } + async _checkNetworkTimeouts() { + const now = Date.now(); + for (const [requestId, metadata] of [...this._pendingRequests.entries()]) { + const ageMs = now - metadata.started_at; + if (ageMs < this._networkTimeoutMs) { + continue; + } + this._pendingRequests.delete(requestId); + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'NetworkTimeout', + message: `Request timed out after ${Math.round(ageMs / 1000)}s`, + details: { + request_id: requestId, + url: metadata.url, + method: metadata.method, + timeout_ms: this._networkTimeoutMs, + }, + })); + } + } + async _checkPageResponsiveness() { + const page = (await this.browser_session.get_current_page()); + if (!page?.evaluate) { + return; + } + const timeoutMs = 2_000; + try { + await Promise.race([ + page.evaluate(() => document.readyState), + new Promise((_, reject) => { + setTimeout(() => { + reject(new Error('Page responsiveness check timed out')); + }, timeoutMs); + }), + ]); + this._consecutiveUnresponsiveChecks = 0; + } + catch (error) { + this._consecutiveUnresponsiveChecks += 1; + if (this._consecutiveUnresponsiveChecks < this._unresponsiveThreshold) { + return; + } + this._consecutiveUnresponsiveChecks = 0; + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'TargetUnresponsive', + message: error.message || 'Target became unresponsive', + details: { + url: this._safePageUrl(page) ?? + this.browser_session.active_tab?.url ?? + '', + timeout_ms: timeoutMs, + }, + })); + } + } +} diff --git a/dist/browser/watchdogs/default-action-watchdog.d.ts b/dist/browser/watchdogs/default-action-watchdog.d.ts new file mode 100644 index 00000000..297ba50a --- /dev/null +++ b/dist/browser/watchdogs/default-action-watchdog.d.ts @@ -0,0 +1,49 @@ +import { ClickCoordinateEvent, ClickElementEvent, CloseTabEvent, GetDropdownOptionsEvent, GoBackEvent, NavigateToUrlEvent, ScrollEvent, ScrollToTextEvent, SelectDropdownOptionEvent, SendKeysEvent, SwitchTabEvent, TypeTextEvent, UploadFileEvent, WaitEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export declare class DefaultActionWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof ClickElementEvent | typeof ClickCoordinateEvent | typeof TypeTextEvent | typeof SwitchTabEvent | typeof GoBackEvent | typeof GetDropdownOptionsEvent)[]; + on_NavigateToUrlEvent(event: NavigateToUrlEvent): Promise; + on_SwitchTabEvent(event: SwitchTabEvent): Promise; + on_CloseTabEvent(event: CloseTabEvent): Promise; + on_GoBackEvent(): Promise; + on_GoForwardEvent(): Promise; + on_RefreshEvent(): Promise; + on_WaitEvent(event: WaitEvent): Promise; + on_SendKeysEvent(event: SendKeysEvent): Promise; + on_ScrollEvent(event: ScrollEvent): Promise; + on_ScrollToTextEvent(event: ScrollToTextEvent): Promise; + on_ClickElementEvent(event: ClickElementEvent): Promise; + on_ClickCoordinateEvent(event: ClickCoordinateEvent): Promise<{ + coordinate_x: number; + coordinate_y: number; + }>; + on_TypeTextEvent(event: TypeTextEvent): Promise; + on_UploadFileEvent(event: UploadFileEvent): Promise; + on_GetDropdownOptionsEvent(event: GetDropdownOptionsEvent): Promise<{ + type: string; + options: string; + formatted_options: any; + message: any; + short_term_memory: any; + long_term_memory: string; + }>; + on_SelectDropdownOptionEvent(event: SelectDropdownOptionEvent): Promise<{ + message: string; + short_term_memory: string; + long_term_memory: string; + matched_text: string; + matched_value: string; + } | { + message: string; + short_term_memory: string; + long_term_memory: string; + matched_text: string; + matched_value?: undefined; + }>; + private _isPrintRelatedElement; + private _handlePrintButtonClick; + private _getUniqueFilename; + private _sanitizeFilename; +} diff --git a/dist/browser/watchdogs/default-action-watchdog.js b/dist/browser/watchdogs/default-action-watchdog.js new file mode 100644 index 00000000..4f21d0c0 --- /dev/null +++ b/dist/browser/watchdogs/default-action-watchdog.js @@ -0,0 +1,212 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { ClickCoordinateEvent, ClickElementEvent, CloseTabEvent, FileDownloadedEvent, GetDropdownOptionsEvent, GoBackEvent, GoForwardEvent, NavigateToUrlEvent, RefreshEvent, ScrollEvent, ScrollToTextEvent, SelectDropdownOptionEvent, SendKeysEvent, SwitchTabEvent, TypeTextEvent, UploadFileEvent, WaitEvent, } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class DefaultActionWatchdog extends BaseWatchdog { + static LISTENS_TO = [ + NavigateToUrlEvent, + SwitchTabEvent, + CloseTabEvent, + GoBackEvent, + GoForwardEvent, + RefreshEvent, + WaitEvent, + SendKeysEvent, + ScrollEvent, + ScrollToTextEvent, + ClickElementEvent, + TypeTextEvent, + UploadFileEvent, + ClickCoordinateEvent, + GetDropdownOptionsEvent, + SelectDropdownOptionEvent, + ]; + async on_NavigateToUrlEvent(event) { + if (event.new_tab) { + await this.browser_session.create_new_tab(event.url, { + wait_until: event.wait_until, + timeout_ms: event.timeout_ms, + }); + return; + } + await this.browser_session.navigate_to(event.url, { + wait_until: event.wait_until, + timeout_ms: event.timeout_ms, + }); + } + async on_SwitchTabEvent(event) { + let identifier; + if (event.target_id) { + identifier = event.target_id; + } + else { + const tabs = this.browser_session.tabs; + if (tabs.length === 0) { + await this.browser_session.create_new_tab('about:blank'); + return (this.browser_session.active_tab?.target_id ?? + this.browser_session.active_tab?.tab_id ?? + 'unknown_target'); + } + const latestTab = tabs[tabs.length - 1]; + identifier = latestTab.target_id ?? latestTab.tab_id ?? latestTab.page_id; + } + const page = await this.browser_session.switch_to_tab(identifier); + const activeTargetId = this.browser_session.active_tab?.target_id ?? null; + return (activeTargetId ?? (page ? this.browser_session.active_tab?.tab_id : null)); + } + async on_CloseTabEvent(event) { + await this.browser_session.close_tab(event.target_id); + } + async on_GoBackEvent() { + await this.browser_session.go_back(); + } + async on_GoForwardEvent() { + await this.browser_session.go_forward(); + } + async on_RefreshEvent() { + await this.browser_session.refresh(); + } + async on_WaitEvent(event) { + const seconds = Math.min(Math.max(event.seconds, 0), event.max_seconds); + await this.browser_session.wait(seconds); + } + async on_SendKeysEvent(event) { + await this.browser_session.send_keys(event.keys); + } + async on_ScrollEvent(event) { + await this.browser_session.scroll(event.direction, event.amount, { + node: event.node ?? null, + }); + } + async on_ScrollToTextEvent(event) { + await this.browser_session.scroll_to_text(event.text, { + direction: event.direction, + }); + } + async on_ClickElementEvent(event) { + if (this.browser_session.is_file_input(event.node)) { + return { + validation_error: 'The target element is a file input. Use upload_file action instead of click.', + }; + } + if (this._isPrintRelatedElement(event.node)) { + const pdfPath = await this._handlePrintButtonClick(); + if (pdfPath) { + return pdfPath; + } + } + return this.browser_session._click_element_node(event.node); + } + async on_ClickCoordinateEvent(event) { + await this.browser_session.click_coordinates(event.coordinate_x, event.coordinate_y, { + button: event.button, + }); + return { + coordinate_x: event.coordinate_x, + coordinate_y: event.coordinate_y, + }; + } + async on_TypeTextEvent(event) { + return this.browser_session._input_text_element_node(event.node, event.text, { + clear: event.clear, + }); + } + async on_UploadFileEvent(event) { + await this.browser_session.upload_file(event.node, event.file_path); + } + async on_GetDropdownOptionsEvent(event) { + return this.browser_session.get_dropdown_options(event.node); + } + async on_SelectDropdownOptionEvent(event) { + return this.browser_session.select_dropdown_option(event.node, event.text); + } + _isPrintRelatedElement(node) { + if (!node || typeof node !== 'object') { + return false; + } + const attributes = 'attributes' in node && node.attributes + ? node.attributes + : {}; + const onclick = String(attributes.onclick ?? '').toLowerCase(); + if (onclick.includes('print')) { + return true; + } + const textMethods = [ + node.get_all_text_till_next_clickable_element, + node.get_all_children_text, + ]; + for (const textMethod of textMethods) { + if (typeof textMethod !== 'function') { + continue; + } + try { + const text = String(textMethod.call(node, 2) ?? '').toLowerCase(); + if (text.includes('print')) { + return true; + } + } + catch { + // Ignore text extraction failures. + } + } + return false; + } + async _handlePrintButtonClick() { + const page = await this.browser_session.get_current_page(); + if (!page) { + return null; + } + try { + const cdpSession = await this.browser_session.get_or_create_cdp_session(page); + const result = await cdpSession.send?.('Page.printToPDF', { + printBackground: true, + preferCSSPageSize: true, + }); + const pdfBase64 = result && typeof result.data === 'string' ? result.data : null; + if (!pdfBase64) { + return null; + } + const downloadsPath = this.browser_session.browser_profile.downloads_path || os.tmpdir(); + fs.mkdirSync(downloadsPath, { recursive: true }); + const title = typeof page.title === 'function' ? await page.title() : 'document'; + const suggestedName = this._sanitizeFilename(`${title || 'document'}.pdf`); + const uniqueFilename = await this._getUniqueFilename(downloadsPath, suggestedName); + const finalPath = path.join(downloadsPath, uniqueFilename); + const content = Buffer.from(pdfBase64, 'base64'); + fs.writeFileSync(finalPath, content); + await this.event_bus.dispatch(new FileDownloadedEvent({ + guid: null, + url: typeof page.url === 'function' ? page.url() : '', + path: finalPath, + file_name: uniqueFilename, + file_size: content.length, + file_type: 'pdf', + mime_type: 'application/pdf', + auto_download: false, + })); + return finalPath; + } + catch (error) { + this.browser_session.logger.debug(`[DefaultActionWatchdog] Print-to-PDF failed: ${error.message}`); + return null; + } + } + async _getUniqueFilename(directory, filename) { + const parsed = path.parse(filename); + let candidate = filename; + let counter = 1; + while (fs.existsSync(path.join(directory, candidate))) { + candidate = `${parsed.name} (${counter})${parsed.ext}`; + counter += 1; + } + return candidate; + } + _sanitizeFilename(filename) { + const sanitized = filename + .replace(/[\\/:*?"<>|]+/g, '_') + .replace(/\s+/g, ' ') + .trim(); + return sanitized || 'document.pdf'; + } +} diff --git a/dist/browser/watchdogs/dom-watchdog.d.ts b/dist/browser/watchdogs/dom-watchdog.d.ts new file mode 100644 index 00000000..e294692c --- /dev/null +++ b/dist/browser/watchdogs/dom-watchdog.d.ts @@ -0,0 +1,8 @@ +import { BrowserErrorEvent, BrowserStateRequestEvent, TabCreatedEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export declare class DOMWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof BrowserStateRequestEvent | typeof TabCreatedEvent)[]; + static EMITS: (typeof BrowserErrorEvent)[]; + on_TabCreatedEvent(): Promise; + on_BrowserStateRequestEvent(event: BrowserStateRequestEvent): Promise; +} diff --git a/dist/browser/watchdogs/dom-watchdog.js b/dist/browser/watchdogs/dom-watchdog.js new file mode 100644 index 00000000..bda6699a --- /dev/null +++ b/dist/browser/watchdogs/dom-watchdog.js @@ -0,0 +1,31 @@ +import { BrowserErrorEvent, BrowserStateRequestEvent, TabCreatedEvent, } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class DOMWatchdog extends BaseWatchdog { + static LISTENS_TO = [TabCreatedEvent, BrowserStateRequestEvent]; + static EMITS = [BrowserErrorEvent]; + async on_TabCreatedEvent() { + // Placeholder hook kept for parity with Python watchdog lifecycle. + return null; + } + async on_BrowserStateRequestEvent(event) { + try { + return await this.browser_session.get_browser_state_with_recovery({ + cache_clickable_elements_hashes: true, + include_screenshot: event.include_screenshot, + include_recent_events: event.include_recent_events, + }); + } + catch (error) { + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'BrowserStateRequestFailed', + message: `DOM state request failed: ${error.message}`, + details: { + include_screenshot: event.include_screenshot, + include_recent_events: event.include_recent_events, + }, + event_parent_id: event.event_id, + })); + throw error; + } + } +} diff --git a/dist/browser/watchdogs/downloads-watchdog.d.ts b/dist/browser/watchdogs/downloads-watchdog.d.ts new file mode 100644 index 00000000..c3fe6726 --- /dev/null +++ b/dist/browser/watchdogs/downloads-watchdog.d.ts @@ -0,0 +1,77 @@ +import { BrowserConnectedEvent, BrowserLaunchEvent, BrowserStateRequestEvent, BrowserStoppedEvent, DownloadProgressEvent, DownloadStartedEvent, FileDownloadedEvent, TabClosedEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +type DownloadStartInfo = { + guid: string; + url: string; + suggested_filename: string; + auto_download: boolean; +}; +type DownloadProgressInfo = { + guid: string; + received_bytes: number; + total_bytes: number; + state: string; +}; +type DownloadCompleteInfo = { + guid: string | null; + url: string; + path: string; + file_name: string; + file_size: number; + file_type: string | null; + mime_type: string | null; + auto_download: boolean; +}; +export declare class DownloadsWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof BrowserStateRequestEvent | typeof BrowserLaunchEvent | typeof BrowserConnectedEvent | typeof BrowserStoppedEvent | typeof TabClosedEvent | typeof DownloadStartedEvent | typeof DownloadProgressEvent | typeof FileDownloadedEvent)[]; + static EMITS: (typeof DownloadStartedEvent | typeof DownloadProgressEvent | typeof FileDownloadedEvent)[]; + private _activeDownloads; + private _downloadStartCallbacks; + private _downloadProgressCallbacks; + private _downloadCompleteCallbacks; + private _cdpSession; + private _cdpListeners; + private _networkDownloads; + private _detectedDownloadUrls; + on_BrowserConnectedEvent(): Promise; + on_BrowserLaunchEvent(): void; + on_BrowserStateRequestEvent(event: BrowserStateRequestEvent): Promise; + on_BrowserStoppedEvent(): void; + on_TabCreatedEvent(): null; + on_TabClosedEvent(): null; + on_NavigationCompleteEvent(): null; + on_DownloadStartedEvent(event: DownloadStartedEvent): void; + on_DownloadProgressEvent(event: DownloadProgressEvent): void; + on_FileDownloadedEvent(event: FileDownloadedEvent): string; + get_active_downloads(): { + url: string; + suggested_filename: string; + started_at: string; + received_bytes: number; + total_bytes: number; + state: string; + guid: string; + }[]; + register_download_callbacks(on_start_or_options?: ((info: DownloadStartInfo) => void) | { + on_start?: ((info: DownloadStartInfo) => void) | null; + on_progress?: ((info: DownloadProgressInfo) => void) | null; + on_complete?: ((info: DownloadCompleteInfo) => void) | null; + } | null, on_progress?: ((info: DownloadProgressInfo) => void) | null, on_complete?: ((info: DownloadCompleteInfo) => void) | null): void; + unregister_download_callbacks(on_start_or_options?: ((info: DownloadStartInfo) => void) | { + on_start?: ((info: DownloadStartInfo) => void) | null; + on_progress?: ((info: DownloadProgressInfo) => void) | null; + on_complete?: ((info: DownloadCompleteInfo) => void) | null; + } | null, on_progress?: ((info: DownloadProgressInfo) => void) | null, on_complete?: ((info: DownloadCompleteInfo) => void) | null): void; + protected onDetached(): void; + private _normalizeCallbackRegistration; + private _startCdpDownloadMonitoring; + private _stopCdpDownloadMonitoring; + private _handleNetworkResponse; + private _handleNetworkLoadingFinished; + private _normalizeHeaders; + private _resolveSuggestedFilename; + private _sanitizeFilename; + private _inferFileType; + private _getUniqueFilename; +} +export {}; diff --git a/dist/browser/watchdogs/downloads-watchdog.js b/dist/browser/watchdogs/downloads-watchdog.js new file mode 100644 index 00000000..c34fc91a --- /dev/null +++ b/dist/browser/watchdogs/downloads-watchdog.js @@ -0,0 +1,409 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { BrowserConnectedEvent, BrowserLaunchEvent, BrowserStateRequestEvent, BrowserStoppedEvent, DownloadProgressEvent, DownloadStartedEvent, FileDownloadedEvent, NavigationCompleteEvent, TabClosedEvent, TabCreatedEvent, } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class DownloadsWatchdog extends BaseWatchdog { + static LISTENS_TO = [ + BrowserConnectedEvent, + BrowserLaunchEvent, + BrowserStateRequestEvent, + BrowserStoppedEvent, + TabCreatedEvent, + TabClosedEvent, + NavigationCompleteEvent, + DownloadStartedEvent, + DownloadProgressEvent, + FileDownloadedEvent, + ]; + static EMITS = [ + DownloadStartedEvent, + DownloadProgressEvent, + FileDownloadedEvent, + ]; + _activeDownloads = new Map(); + _downloadStartCallbacks = []; + _downloadProgressCallbacks = []; + _downloadCompleteCallbacks = []; + _cdpSession = null; + _cdpListeners = []; + _networkDownloads = new Map(); + _detectedDownloadUrls = new Set(); + async on_BrowserConnectedEvent() { + await this._startCdpDownloadMonitoring(); + } + on_BrowserLaunchEvent() { + const downloadsPath = this.browser_session.browser_profile.downloads_path; + if (!downloadsPath) { + return; + } + fs.mkdirSync(downloadsPath, { recursive: true }); + } + async on_BrowserStateRequestEvent(event) { + const activeTab = this.browser_session.active_tab; + if (!activeTab?.target_id || !activeTab.url) { + return; + } + await this.event_bus.dispatch(new NavigationCompleteEvent({ + target_id: activeTab.target_id, + url: activeTab.url, + status: null, + error_message: null, + loading_status: null, + event_parent_id: event.event_id, + })); + } + on_BrowserStoppedEvent() { + this._activeDownloads.clear(); + this._downloadStartCallbacks = []; + this._downloadProgressCallbacks = []; + this._downloadCompleteCallbacks = []; + this._networkDownloads.clear(); + this._detectedDownloadUrls.clear(); + void this._stopCdpDownloadMonitoring(); + } + on_TabCreatedEvent() { + return null; + } + on_TabClosedEvent() { + return null; + } + on_NavigationCompleteEvent() { + return null; + } + on_DownloadStartedEvent(event) { + const startInfo = { + guid: event.guid, + url: event.url, + suggested_filename: event.suggested_filename, + auto_download: event.auto_download, + }; + this._activeDownloads.set(event.guid, { + url: event.url, + suggested_filename: event.suggested_filename, + started_at: new Date().toISOString(), + received_bytes: 0, + total_bytes: 0, + state: 'inProgress', + }); + for (const callback of this._downloadStartCallbacks) { + try { + callback(startInfo); + } + catch (error) { + this.browser_session.logger.debug(`[DownloadsWatchdog] Error in download start callback: ${error.message}`); + } + } + } + on_DownloadProgressEvent(event) { + const existing = this._activeDownloads.get(event.guid); + if (existing) { + existing.received_bytes = event.received_bytes; + existing.total_bytes = event.total_bytes; + existing.state = event.state; + if (event.state === 'completed' || event.state === 'canceled') { + this._activeDownloads.delete(event.guid); + } + } + const progressInfo = { + guid: event.guid, + received_bytes: event.received_bytes, + total_bytes: event.total_bytes, + state: event.state, + }; + for (const callback of this._downloadProgressCallbacks) { + try { + callback(progressInfo); + } + catch (error) { + this.browser_session.logger.debug(`[DownloadsWatchdog] Error in download progress callback: ${error.message}`); + } + } + } + on_FileDownloadedEvent(event) { + if (event.guid) { + this._activeDownloads.delete(event.guid); + } + this.browser_session.add_downloaded_file(event.path); + const completeInfo = { + guid: event.guid, + url: event.url, + path: event.path, + file_name: event.file_name, + file_size: event.file_size, + file_type: event.file_type, + mime_type: event.mime_type, + auto_download: event.auto_download, + }; + for (const callback of this._downloadCompleteCallbacks) { + try { + callback(completeInfo); + } + catch (error) { + this.browser_session.logger.debug(`[DownloadsWatchdog] Error in download complete callback: ${error.message}`); + } + } + return event.path; + } + get_active_downloads() { + return [...this._activeDownloads.entries()].map(([guid, metadata]) => ({ + guid, + ...metadata, + })); + } + register_download_callbacks(on_start_or_options = null, on_progress = null, on_complete = null) { + const { on_start, on_progress: resolvedProgress, on_complete: resolvedEnd, } = this._normalizeCallbackRegistration(on_start_or_options, on_progress, on_complete); + if (on_start) { + this._downloadStartCallbacks.push(on_start); + } + if (resolvedProgress) { + this._downloadProgressCallbacks.push(resolvedProgress); + } + if (resolvedEnd) { + this._downloadCompleteCallbacks.push(resolvedEnd); + } + } + unregister_download_callbacks(on_start_or_options = null, on_progress = null, on_complete = null) { + const { on_start, on_progress: resolvedProgress, on_complete: resolvedEnd, } = this._normalizeCallbackRegistration(on_start_or_options, on_progress, on_complete); + if (on_start) { + this._downloadStartCallbacks = this._downloadStartCallbacks.filter((callback) => callback !== on_start); + } + if (resolvedProgress) { + this._downloadProgressCallbacks = this._downloadProgressCallbacks.filter((callback) => callback !== resolvedProgress); + } + if (resolvedEnd) { + this._downloadCompleteCallbacks = this._downloadCompleteCallbacks.filter((callback) => callback !== resolvedEnd); + } + } + onDetached() { + this._activeDownloads.clear(); + this._downloadStartCallbacks = []; + this._downloadProgressCallbacks = []; + this._downloadCompleteCallbacks = []; + this._networkDownloads.clear(); + this._detectedDownloadUrls.clear(); + void this._stopCdpDownloadMonitoring(); + } + _normalizeCallbackRegistration(on_start_or_options, on_progress, on_complete) { + if (on_start_or_options && + typeof on_start_or_options === 'object' && + !Array.isArray(on_start_or_options)) { + return { + on_start: on_start_or_options.on_start ?? null, + on_progress: on_start_or_options.on_progress ?? null, + on_complete: on_start_or_options.on_complete ?? null, + }; + } + return { + on_start: typeof on_start_or_options === 'function' ? on_start_or_options : null, + on_progress, + on_complete, + }; + } + async _startCdpDownloadMonitoring() { + if (this._cdpSession) { + return; + } + if (!this.browser_session.browser_context?.newCDPSession) { + return; + } + try { + const session = (await this.browser_session.get_or_create_cdp_session(null)); + await session.send?.('Network.enable'); + this._cdpSession = session; + const onResponseReceived = (payload) => { + void this._handleNetworkResponse(payload); + }; + const onLoadingFinished = (payload) => { + void this._handleNetworkLoadingFinished(payload); + }; + session.on?.('Network.responseReceived', onResponseReceived); + session.on?.('Network.loadingFinished', onLoadingFinished); + this._cdpListeners = [ + { + event: 'Network.responseReceived', + handler: onResponseReceived, + }, + { + event: 'Network.loadingFinished', + handler: onLoadingFinished, + }, + ]; + } + catch (error) { + this.browser_session.logger.debug(`[DownloadsWatchdog] CDP download monitoring unavailable: ${error.message}`); + await this._stopCdpDownloadMonitoring(); + } + } + async _stopCdpDownloadMonitoring() { + if (!this._cdpSession) { + return; + } + for (const listener of this._cdpListeners) { + this._cdpSession.off?.(listener.event, listener.handler); + } + this._cdpListeners = []; + try { + await this._cdpSession.detach?.(); + } + catch { + // Ignore detach errors during shutdown. + } + finally { + this._cdpSession = null; + } + } + async _handleNetworkResponse(payload) { + const requestId = String(payload?.requestId ?? ''); + if (!requestId) { + return; + } + const response = payload?.response ?? {}; + const url = String(response?.url ?? '').trim(); + if (!url || this._detectedDownloadUrls.has(url)) { + return; + } + const headers = this._normalizeHeaders(response?.headers); + const mimeType = typeof response?.mimeType === 'string' + ? response.mimeType.toLowerCase() + : ''; + const contentDisposition = headers['content-disposition'] ?? ''; + const isPdf = mimeType.includes('application/pdf') || /\.pdf(?:$|\?)/i.test(url); + const isAttachment = /attachment/i.test(contentDisposition); + const isBinary = mimeType.includes('application/octet-stream'); + if (!isPdf && !isAttachment && !isBinary) { + return; + } + this._detectedDownloadUrls.add(url); + const suggestedFilename = this._resolveSuggestedFilename(contentDisposition, url); + const guid = `cdp-${requestId}`; + const autoDownload = isPdf && this.browser_session.auto_download_pdfs(); + const fileType = this._inferFileType(suggestedFilename, mimeType); + this._networkDownloads.set(requestId, { + guid, + url, + suggested_filename: suggestedFilename, + mime_type: mimeType || null, + file_type: fileType, + auto_download: autoDownload, + }); + await this.event_bus.dispatch(new DownloadStartedEvent({ + guid, + url, + suggested_filename: suggestedFilename, + auto_download: autoDownload, + })); + } + async _handleNetworkLoadingFinished(payload) { + const requestId = String(payload?.requestId ?? ''); + if (!requestId) { + return; + } + const metadata = this._networkDownloads.get(requestId); + if (!metadata) { + return; + } + this._networkDownloads.delete(requestId); + const encodedDataLength = typeof payload?.encodedDataLength === 'number' + ? Math.max(0, Math.floor(payload.encodedDataLength)) + : 0; + await this.event_bus.dispatch(new DownloadProgressEvent({ + guid: metadata.guid, + received_bytes: encodedDataLength, + total_bytes: encodedDataLength, + state: 'completed', + })); + if (!metadata.auto_download || + !metadata.mime_type?.includes('application/pdf')) { + return; + } + const downloadsPath = this.browser_session.browser_profile.downloads_path; + if (!downloadsPath || !this._cdpSession?.send) { + return; + } + try { + const responseBody = await this._cdpSession.send('Network.getResponseBody', { + requestId, + }); + const body = typeof responseBody?.body === 'string' ? responseBody.body : ''; + if (!body) { + return; + } + fs.mkdirSync(downloadsPath, { recursive: true }); + const uniqueFilename = await this._getUniqueFilename(downloadsPath, metadata.suggested_filename); + const filePath = path.join(downloadsPath, uniqueFilename); + const content = responseBody?.base64Encoded + ? Buffer.from(body, 'base64') + : Buffer.from(body, 'utf8'); + fs.writeFileSync(filePath, content); + await this.event_bus.dispatch(new FileDownloadedEvent({ + guid: metadata.guid, + url: metadata.url, + path: filePath, + file_name: uniqueFilename, + file_size: content.length, + file_type: metadata.file_type, + mime_type: metadata.mime_type, + auto_download: true, + })); + } + catch (error) { + this.browser_session.logger.debug(`[DownloadsWatchdog] Failed to materialize CDP download body: ${error.message}`); + } + } + _normalizeHeaders(input) { + if (!input || typeof input !== 'object') { + return {}; + } + return Object.fromEntries(Object.entries(input).map(([k, v]) => [ + k.toLowerCase(), + String(v), + ])); + } + _resolveSuggestedFilename(contentDisposition, url) { + const filenameMatch = /filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/i.exec(contentDisposition); + const fromHeader = filenameMatch?.[1] || filenameMatch?.[2] || ''; + const candidate = decodeURIComponent(fromHeader || '').trim(); + if (candidate) { + return this._sanitizeFilename(candidate); + } + try { + const parsed = new URL(url); + const basename = path.basename(parsed.pathname); + if (basename) { + return this._sanitizeFilename(basename); + } + } + catch { + // Ignore URL parsing errors and fallback below. + } + return 'download'; + } + _sanitizeFilename(filename) { + const sanitized = filename + .replace(/[\\/:*?"<>|]+/g, '_') + .replace(/\s+/g, ' ') + .trim(); + return sanitized || 'download'; + } + _inferFileType(filename, mimeType) { + const ext = path.extname(filename).replace('.', '').toLowerCase(); + if (ext) { + return ext; + } + if (mimeType.includes('pdf')) { + return 'pdf'; + } + return null; + } + async _getUniqueFilename(directory, filename) { + const ext = path.extname(filename); + const basename = ext ? filename.slice(0, -ext.length) : filename; + let candidate = filename || 'download'; + let counter = 1; + while (fs.existsSync(path.join(directory, candidate))) { + candidate = `${basename || 'download'}_${counter}${ext}`; + counter += 1; + } + return candidate; + } +} diff --git a/dist/browser/watchdogs/har-recording-watchdog.d.ts b/dist/browser/watchdogs/har-recording-watchdog.d.ts new file mode 100644 index 00000000..cc5d540b --- /dev/null +++ b/dist/browser/watchdogs/har-recording-watchdog.d.ts @@ -0,0 +1,19 @@ +import { BrowserConnectedEvent, BrowserStopEvent, BrowserStartEvent, BrowserStoppedEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export declare class HarRecordingWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof BrowserStartEvent | typeof BrowserStopEvent | typeof BrowserConnectedEvent | typeof BrowserStoppedEvent)[]; + private _harPath; + private _cdpSession; + private _listeners; + private _entries; + on_BrowserStartEvent(): Promise; + on_BrowserConnectedEvent(): Promise; + on_BrowserStopEvent(): Promise; + on_BrowserStoppedEvent(): Promise; + protected onDetached(): void; + private _resolveConfiguredHarPath; + private _resolveAndPrepareHarPath; + private _startCdpCaptureIfNeeded; + private _writeHarFallbackIfNeeded; + private _teardownCapture; +} diff --git a/dist/browser/watchdogs/har-recording-watchdog.js b/dist/browser/watchdogs/har-recording-watchdog.js new file mode 100644 index 00000000..b8c20d70 --- /dev/null +++ b/dist/browser/watchdogs/har-recording-watchdog.js @@ -0,0 +1,317 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { BrowserConnectedEvent, BrowserStopEvent, BrowserErrorEvent, BrowserStartEvent, BrowserStoppedEvent, } from '../events.js'; +import { BaseWatchdog } from './base.js'; +const toIsoFromEpochSeconds = (value) => { + if (typeof value !== 'number' || !Number.isFinite(value)) { + return new Date().toISOString(); + } + return new Date(value * 1000).toISOString(); +}; +const normalizeHeaders = (input) => { + if (!input) { + return {}; + } + if (Array.isArray(input)) { + const out = {}; + for (const header of input) { + if (header && + typeof header === 'object' && + 'name' in header && + 'value' in header) { + out[String(header.name).toLowerCase()] = String(header.value); + } + } + return out; + } + if (typeof input === 'object') { + return Object.fromEntries(Object.entries(input).map(([key, value]) => [ + key.toLowerCase(), + String(value), + ])); + } + return {}; +}; +export class HarRecordingWatchdog extends BaseWatchdog { + static LISTENS_TO = [ + BrowserStartEvent, + BrowserConnectedEvent, + BrowserStopEvent, + BrowserStoppedEvent, + ]; + _harPath = null; + _cdpSession = null; + _listeners = []; + _entries = new Map(); + async on_BrowserStartEvent() { + const resolvedPath = this._resolveAndPrepareHarPath(); + if (!resolvedPath) { + return; + } + this._harPath = resolvedPath; + } + async on_BrowserConnectedEvent() { + await this._startCdpCaptureIfNeeded(); + } + async on_BrowserStopEvent() { + await this._writeHarFallbackIfNeeded(); + } + async on_BrowserStoppedEvent() { + const resolvedPath = this._harPath ?? this._resolveConfiguredHarPath(); + if (!resolvedPath) { + await this._teardownCapture(); + return; + } + if (!fs.existsSync(resolvedPath)) { + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'HarRecordingMissing', + message: `HAR file was not created at ${resolvedPath}`, + details: { + record_har_path: resolvedPath, + }, + })); + await this._teardownCapture(); + return; + } + try { + const stat = fs.statSync(resolvedPath); + if (stat.size === 0) { + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'HarRecordingEmpty', + message: `HAR file is empty at ${resolvedPath}`, + details: { + record_har_path: resolvedPath, + }, + })); + } + } + catch (error) { + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'HarRecordingStatFailed', + message: `Failed to inspect HAR file: ${error.message}`, + details: { + record_har_path: resolvedPath, + }, + })); + } + finally { + await this._teardownCapture(); + } + } + onDetached() { + void this._teardownCapture(); + } + _resolveConfiguredHarPath() { + const configuredPath = this.browser_session.browser_profile.config.record_har_path; + if (typeof configuredPath !== 'string' || configuredPath.trim() === '') { + return null; + } + return path.resolve(configuredPath); + } + _resolveAndPrepareHarPath() { + const resolvedPath = this._resolveConfiguredHarPath(); + if (!resolvedPath) { + return null; + } + fs.mkdirSync(path.dirname(resolvedPath), { recursive: true }); + this.browser_session.browser_profile.config.record_har_path = resolvedPath; + return resolvedPath; + } + async _startCdpCaptureIfNeeded() { + if (!this._harPath || this._cdpSession) { + return; + } + try { + const cdpSession = (await this.browser_session.get_or_create_cdp_session(null)); + this._cdpSession = cdpSession; + await cdpSession.send?.('Network.enable'); + const onRequestWillBeSent = (payload) => { + const requestId = String(payload?.requestId ?? ''); + const request = payload?.request ?? {}; + const url = String(request?.url ?? ''); + if (!requestId || !url.toLowerCase().startsWith('https://')) { + return; + } + const tsRequest = typeof payload?.timestamp === 'number' ? payload.timestamp : null; + this._entries.set(requestId, { + request_id: requestId, + started_date_time: toIsoFromEpochSeconds(typeof payload?.wallTime === 'number' + ? payload.wallTime + : Date.now() / 1000), + method: String(request?.method ?? 'GET'), + url, + request_headers: normalizeHeaders(request?.headers), + status: 0, + status_text: '', + response_headers: {}, + mime_type: '', + failed: false, + ts_request: tsRequest, + ts_response: null, + ts_finished: null, + encoded_data_length: null, + }); + }; + const onResponseReceived = (payload) => { + const requestId = String(payload?.requestId ?? ''); + const entry = this._entries.get(requestId); + if (!entry) { + return; + } + const response = payload?.response ?? {}; + entry.status = + typeof response?.status === 'number' + ? response.status + : Number(response?.status ?? 0); + entry.status_text = String(response?.statusText ?? ''); + entry.response_headers = normalizeHeaders(response?.headers); + entry.mime_type = String(response?.mimeType ?? ''); + entry.ts_response = + typeof payload?.timestamp === 'number' ? payload.timestamp : null; + }; + const onLoadingFinished = (payload) => { + const requestId = String(payload?.requestId ?? ''); + const entry = this._entries.get(requestId); + if (!entry) { + return; + } + entry.ts_finished = + typeof payload?.timestamp === 'number' ? payload.timestamp : null; + entry.encoded_data_length = + typeof payload?.encodedDataLength === 'number' + ? payload.encodedDataLength + : null; + }; + const onLoadingFailed = (payload) => { + const requestId = String(payload?.requestId ?? ''); + const entry = this._entries.get(requestId); + if (!entry) { + return; + } + entry.failed = true; + }; + cdpSession.on?.('Network.requestWillBeSent', onRequestWillBeSent); + cdpSession.on?.('Network.responseReceived', onResponseReceived); + cdpSession.on?.('Network.loadingFinished', onLoadingFinished); + cdpSession.on?.('Network.loadingFailed', onLoadingFailed); + this._listeners = [ + { event: 'Network.requestWillBeSent', handler: onRequestWillBeSent }, + { event: 'Network.responseReceived', handler: onResponseReceived }, + { event: 'Network.loadingFinished', handler: onLoadingFinished }, + { event: 'Network.loadingFailed', handler: onLoadingFailed }, + ]; + } + catch (error) { + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'HarCaptureUnavailable', + message: `CDP HAR capture is unavailable: ${error.message}`, + details: { + record_har_path: this._harPath, + }, + })); + } + } + async _writeHarFallbackIfNeeded() { + const resolvedPath = this._harPath ?? this._resolveConfiguredHarPath(); + if (!resolvedPath) { + return; + } + if (fs.existsSync(resolvedPath)) { + try { + if (fs.statSync(resolvedPath).size > 0) { + return; + } + } + catch { + // Continue into fallback writer. + } + } + const entries = [...this._entries.values()]; + const harEntries = entries.map((entry) => { + const waitMs = entry.ts_request != null && entry.ts_response != null + ? Math.max(0, Math.round((entry.ts_response - entry.ts_request) * 1000)) + : 0; + const receiveMs = entry.ts_response != null && entry.ts_finished != null + ? Math.max(0, Math.round((entry.ts_finished - entry.ts_response) * 1000)) + : 0; + const totalMs = waitMs + receiveMs; + return { + startedDateTime: entry.started_date_time, + time: totalMs, + request: { + method: entry.method, + url: entry.url, + httpVersion: 'HTTP/1.1', + headers: Object.entries(entry.request_headers).map(([name, value]) => ({ + name, + value, + })), + queryString: [], + cookies: [], + headersSize: -1, + bodySize: -1, + }, + response: { + status: entry.status, + statusText: entry.status_text, + httpVersion: 'HTTP/1.1', + headers: Object.entries(entry.response_headers).map(([name, value]) => ({ + name, + value, + })), + cookies: [], + content: { + size: entry.encoded_data_length ?? -1, + mimeType: entry.mime_type, + }, + redirectURL: '', + headersSize: -1, + bodySize: entry.encoded_data_length ?? -1, + }, + cache: {}, + timings: { + dns: -1, + connect: -1, + ssl: -1, + send: 0, + wait: waitMs, + receive: receiveMs, + }, + _failed: entry.failed, + }; + }); + const harObject = { + log: { + version: '1.2', + creator: { + name: 'browser-use-node', + version: 'dev', + }, + pages: [], + entries: harEntries, + }, + }; + const tempPath = `${resolvedPath}.tmp`; + fs.writeFileSync(tempPath, JSON.stringify(harObject, null, 2), 'utf-8'); + fs.renameSync(tempPath, resolvedPath); + } + async _teardownCapture() { + if (!this._cdpSession) { + return; + } + for (const listener of this._listeners) { + this._cdpSession.off?.(listener.event, listener.handler); + } + this._listeners = []; + try { + await this._cdpSession.detach?.(); + } + catch { + // Ignore CDP detach errors during shutdown. + } + finally { + this._cdpSession = null; + this._entries.clear(); + } + } +} diff --git a/dist/browser/watchdogs/index.d.ts b/dist/browser/watchdogs/index.d.ts new file mode 100644 index 00000000..0f637ea0 --- /dev/null +++ b/dist/browser/watchdogs/index.d.ts @@ -0,0 +1,16 @@ +export * from './base.js'; +export * from './aboutblank-watchdog.js'; +export * from './cdp-session-watchdog.js'; +export * from './captcha-watchdog.js'; +export * from './crash-watchdog.js'; +export * from './default-action-watchdog.js'; +export * from './dom-watchdog.js'; +export * from './downloads-watchdog.js'; +export * from './har-recording-watchdog.js'; +export * from './local-browser-watchdog.js'; +export * from './permissions-watchdog.js'; +export * from './popups-watchdog.js'; +export * from './recording-watchdog.js'; +export * from './screenshot-watchdog.js'; +export * from './security-watchdog.js'; +export * from './storage-state-watchdog.js'; diff --git a/dist/browser/watchdogs/index.js b/dist/browser/watchdogs/index.js new file mode 100644 index 00000000..0f637ea0 --- /dev/null +++ b/dist/browser/watchdogs/index.js @@ -0,0 +1,16 @@ +export * from './base.js'; +export * from './aboutblank-watchdog.js'; +export * from './cdp-session-watchdog.js'; +export * from './captcha-watchdog.js'; +export * from './crash-watchdog.js'; +export * from './default-action-watchdog.js'; +export * from './dom-watchdog.js'; +export * from './downloads-watchdog.js'; +export * from './har-recording-watchdog.js'; +export * from './local-browser-watchdog.js'; +export * from './permissions-watchdog.js'; +export * from './popups-watchdog.js'; +export * from './recording-watchdog.js'; +export * from './screenshot-watchdog.js'; +export * from './security-watchdog.js'; +export * from './storage-state-watchdog.js'; diff --git a/dist/browser/watchdogs/local-browser-watchdog.d.ts b/dist/browser/watchdogs/local-browser-watchdog.d.ts new file mode 100644 index 00000000..881dbec5 --- /dev/null +++ b/dist/browser/watchdogs/local-browser-watchdog.d.ts @@ -0,0 +1,10 @@ +import { BrowserKillEvent, BrowserLaunchEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export declare class LocalBrowserWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof BrowserLaunchEvent | typeof BrowserKillEvent)[]; + on_BrowserLaunchEvent(): Promise<{ + cdp_url: string; + }>; + on_BrowserKillEvent(): Promise; + on_BrowserStopEvent(): void; +} diff --git a/dist/browser/watchdogs/local-browser-watchdog.js b/dist/browser/watchdogs/local-browser-watchdog.js new file mode 100644 index 00000000..9a1e1cd1 --- /dev/null +++ b/dist/browser/watchdogs/local-browser-watchdog.js @@ -0,0 +1,32 @@ +import { BrowserKillEvent, BrowserLaunchEvent, BrowserStopEvent, } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class LocalBrowserWatchdog extends BaseWatchdog { + static LISTENS_TO = [ + BrowserLaunchEvent, + BrowserKillEvent, + BrowserStopEvent, + ]; + async on_BrowserLaunchEvent() { + await this.browser_session.start(); + return { + cdp_url: this.browser_session.cdp_url ?? + this.browser_session.wss_url ?? + 'playwright', + }; + } + async on_BrowserKillEvent() { + if (this.browser_session.is_stopping) { + return; + } + await this.browser_session.kill(); + } + on_BrowserStopEvent() { + if (!this.browser_session._owns_browser_resources) { + return; + } + // Fire-and-forget to avoid blocking BrowserStopEvent handler completion. + void this.event_bus.dispatch(new BrowserKillEvent()).catch(() => { + // Ignore shutdown re-entrancy errors during stop lifecycle. + }); + } +} diff --git a/dist/browser/watchdogs/permissions-watchdog.d.ts b/dist/browser/watchdogs/permissions-watchdog.d.ts new file mode 100644 index 00000000..61621301 --- /dev/null +++ b/dist/browser/watchdogs/permissions-watchdog.d.ts @@ -0,0 +1,8 @@ +import { BrowserConnectedEvent, BrowserErrorEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export declare class PermissionsWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof BrowserConnectedEvent)[]; + static EMITS: (typeof BrowserErrorEvent)[]; + on_BrowserConnectedEvent(): Promise; + private _grantPermissionsViaCdp; +} diff --git a/dist/browser/watchdogs/permissions-watchdog.js b/dist/browser/watchdogs/permissions-watchdog.js new file mode 100644 index 00000000..07618a1d --- /dev/null +++ b/dist/browser/watchdogs/permissions-watchdog.js @@ -0,0 +1,73 @@ +import { BrowserConnectedEvent, BrowserErrorEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class PermissionsWatchdog extends BaseWatchdog { + static LISTENS_TO = [BrowserConnectedEvent]; + static EMITS = [BrowserErrorEvent]; + async on_BrowserConnectedEvent() { + const permissions = this.browser_session.browser_profile.config.permissions; + if (!Array.isArray(permissions) || permissions.length === 0) { + return; + } + let cdpError = null; + try { + const grantedWithCdp = await this._grantPermissionsViaCdp(permissions); + if (grantedWithCdp) { + return; + } + } + catch (error) { + cdpError = error; + } + const context = this.browser_session.browser_context; + if (!context?.grantPermissions) { + if (!cdpError) { + return; + } + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'PermissionsWatchdogError', + message: cdpError.message || 'Failed to grant permissions via CDP', + details: { + permissions, + mode: 'cdp', + }, + })); + return; + } + try { + await context.grantPermissions(permissions); + } + catch (error) { + const message = error.message ?? 'Failed to grant permissions'; + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'PermissionsWatchdogError', + message, + details: { + permissions, + cdp_error: cdpError?.message ?? null, + mode: 'playwright', + }, + })); + } + } + async _grantPermissionsViaCdp(permissions) { + const browser = this.browser_session.browser; + if (!browser?.newBrowserCDPSession) { + return false; + } + const cdpSession = await browser.newBrowserCDPSession(); + try { + await cdpSession.send?.('Browser.grantPermissions', { + permissions, + }); + return true; + } + finally { + try { + await cdpSession.detach?.(); + } + catch { + // Ignore detach failures. + } + } + } +} diff --git a/dist/browser/watchdogs/popups-watchdog.d.ts b/dist/browser/watchdogs/popups-watchdog.d.ts new file mode 100644 index 00000000..52ea5027 --- /dev/null +++ b/dist/browser/watchdogs/popups-watchdog.d.ts @@ -0,0 +1,13 @@ +import { BrowserStoppedEvent, TabCreatedEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export declare class PopupsWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof BrowserStoppedEvent | typeof TabCreatedEvent)[]; + private _dialogListenersRegistered; + private _cdpDialogSessions; + on_TabCreatedEvent(event: TabCreatedEvent): Promise; + on_BrowserStoppedEvent(): Promise; + protected onDetached(): void; + private _attachCdpDialogHandler; + private _detachCdpDialogHandlers; + private _handleJavascriptDialog; +} diff --git a/dist/browser/watchdogs/popups-watchdog.js b/dist/browser/watchdogs/popups-watchdog.js new file mode 100644 index 00000000..33f066dd --- /dev/null +++ b/dist/browser/watchdogs/popups-watchdog.js @@ -0,0 +1,77 @@ +import { BrowserStoppedEvent, TabCreatedEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class PopupsWatchdog extends BaseWatchdog { + static LISTENS_TO = [TabCreatedEvent, BrowserStoppedEvent]; + _dialogListenersRegistered = new Set(); + _cdpDialogSessions = new Map(); + async on_TabCreatedEvent(event) { + const page = (await this.browser_session.get_current_page()); + if (!page) { + return; + } + const attachDialogHandler = this.browser_session + ?._attachDialogHandler; + if (typeof attachDialogHandler === 'function') { + attachDialogHandler.call(this.browser_session, page); + } + await this._attachCdpDialogHandler(event.target_id, page); + } + async on_BrowserStoppedEvent() { + await this._detachCdpDialogHandlers(); + } + onDetached() { + void this._detachCdpDialogHandlers(); + } + async _attachCdpDialogHandler(targetId, page) { + if (this._dialogListenersRegistered.has(targetId)) { + return; + } + try { + const session = (await this.browser_session.get_or_create_cdp_session(page)); + await session.send?.('Page.enable'); + const handler = (payload) => { + void this._handleJavascriptDialog(payload, session); + }; + session.on?.('Page.javascriptDialogOpening', handler); + this._dialogListenersRegistered.add(targetId); + this._cdpDialogSessions.set(targetId, { + session, + handler, + }); + } + catch (error) { + this.browser_session.logger.debug(`[PopupsWatchdog] Failed to attach CDP dialog handler: ${error.message}`); + } + } + async _detachCdpDialogHandlers() { + for (const [targetId, binding] of [...this._cdpDialogSessions.entries()]) { + binding.session.off?.('Page.javascriptDialogOpening', binding.handler); + try { + await binding.session.detach?.(); + } + catch { + // Ignore detach failures during cleanup. + } + this._cdpDialogSessions.delete(targetId); + } + this._dialogListenersRegistered.clear(); + } + async _handleJavascriptDialog(payload, session) { + const dialogType = typeof payload?.type === 'string' ? payload.type : 'alert'; + const message = typeof payload?.message === 'string' ? payload.message : ''; + const shouldAccept = ['alert', 'confirm', 'beforeunload'].includes(dialogType); + const captureClosedPopupMessage = this.browser_session + ?._captureClosedPopupMessage; + if (typeof captureClosedPopupMessage === 'function' && message) { + captureClosedPopupMessage.call(this.browser_session, dialogType, message); + } + try { + await session.send?.('Page.handleJavaScriptDialog', { + accept: shouldAccept, + }); + } + catch (error) { + this.browser_session.logger.debug(`[PopupsWatchdog] Failed to handle JavaScript dialog: ${error.message}`); + } + } +} diff --git a/dist/browser/watchdogs/recording-watchdog.d.ts b/dist/browser/watchdogs/recording-watchdog.d.ts new file mode 100644 index 00000000..66d73a12 --- /dev/null +++ b/dist/browser/watchdogs/recording-watchdog.d.ts @@ -0,0 +1,27 @@ +import { AgentFocusChangedEvent, BrowserConnectedEvent, BrowserStopEvent, BrowserStoppedEvent, TabCreatedEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export declare class RecordingWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof BrowserStopEvent | typeof BrowserConnectedEvent | typeof BrowserStoppedEvent | typeof TabCreatedEvent)[]; + private _traceStarted; + private _videoCloseListeners; + private _cdpScreencastSession; + private _cdpScreencastHandler; + private _cdpScreencastPath; + private _cdpScreencastStream; + on_BrowserConnectedEvent(): Promise; + on_BrowserStopEvent(): Promise; + on_BrowserStoppedEvent(): Promise; + on_AgentFocusChangedEvent(event: AgentFocusChangedEvent): Promise; + on_TabCreatedEvent(): Promise; + protected onDetached(): void; + private _prepareVideoDirectory; + private _startTracingIfConfigured; + private _stopTracingIfStarted; + private _attachVideoListenersToKnownPages; + private _attachVideoListener; + private _detachVideoListeners; + private _getKnownPages; + private _captureVideoArtifact; + private _startCdpScreencastIfConfigured; + private _stopCdpScreencastIfStarted; +} diff --git a/dist/browser/watchdogs/recording-watchdog.js b/dist/browser/watchdogs/recording-watchdog.js new file mode 100644 index 00000000..ebe188d1 --- /dev/null +++ b/dist/browser/watchdogs/recording-watchdog.js @@ -0,0 +1,249 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { AgentFocusChangedEvent, BrowserConnectedEvent, BrowserErrorEvent, BrowserStopEvent, BrowserStoppedEvent, TabCreatedEvent, } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class RecordingWatchdog extends BaseWatchdog { + static LISTENS_TO = [ + BrowserConnectedEvent, + BrowserStopEvent, + BrowserStoppedEvent, + AgentFocusChangedEvent, + TabCreatedEvent, + ]; + _traceStarted = false; + _videoCloseListeners = new Map(); + _cdpScreencastSession = null; + _cdpScreencastHandler = null; + _cdpScreencastPath = null; + _cdpScreencastStream = null; + async on_BrowserConnectedEvent() { + this._prepareVideoDirectory(); + this._attachVideoListenersToKnownPages(); + await this._startCdpScreencastIfConfigured(); + await this._startTracingIfConfigured(); + } + async on_BrowserStopEvent() { + await this._stopCdpScreencastIfStarted(); + await this._stopTracingIfStarted(); + } + async on_BrowserStoppedEvent() { + this._detachVideoListeners(); + } + async on_AgentFocusChangedEvent(event) { + this._attachVideoListenersToKnownPages(); + await this._startCdpScreencastIfConfigured(); + if (!this._traceStarted) { + return; + } + this.browser_session.logger.debug(`[RecordingWatchdog] Focus changed to ${event.target_id}; tracing remains active`); + } + async on_TabCreatedEvent() { + this._attachVideoListenersToKnownPages(); + await this._startCdpScreencastIfConfigured(); + } + onDetached() { + void this._stopCdpScreencastIfStarted(); + this._detachVideoListeners(); + } + _prepareVideoDirectory() { + const configuredPath = this.browser_session.browser_profile.config.record_video_dir; + if (typeof configuredPath !== 'string' || configuredPath.trim() === '') { + return; + } + const resolvedPath = path.resolve(configuredPath); + fs.mkdirSync(resolvedPath, { recursive: true }); + this.browser_session.browser_profile.config.record_video_dir = resolvedPath; + } + async _startTracingIfConfigured() { + if (this._traceStarted || + !this.browser_session.browser_profile.traces_dir) { + return; + } + try { + await this.browser_session.start_trace_recording(); + this._traceStarted = true; + } + catch (error) { + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'RecordingStartFailed', + message: `Failed to start trace recording: ${error.message}`, + details: { + traces_dir: this.browser_session.browser_profile.traces_dir, + }, + })); + } + } + async _stopTracingIfStarted() { + if (!this._traceStarted) { + return; + } + try { + await this.browser_session.save_trace_recording(); + } + catch (error) { + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'RecordingStopFailed', + message: `Failed to save trace recording: ${error.message}`, + details: { + traces_dir: this.browser_session.browser_profile.traces_dir, + }, + })); + } + finally { + this._traceStarted = false; + } + } + _attachVideoListenersToKnownPages() { + const configuredPath = this.browser_session.browser_profile.config.record_video_dir; + if (typeof configuredPath !== 'string' || configuredPath.trim() === '') { + return; + } + for (const page of this._getKnownPages()) { + this._attachVideoListener(page); + } + } + _attachVideoListener(page) { + if (this._videoCloseListeners.has(page) || typeof page?.on !== 'function') { + return; + } + const listener = () => { + this._videoCloseListeners.delete(page); + if (typeof page.off === 'function') { + page.off('close', listener); + } + else if (typeof page.removeListener === 'function') { + page.removeListener('close', listener); + } + void this._captureVideoArtifact(page); + }; + page.on('close', listener); + this._videoCloseListeners.set(page, listener); + } + _detachVideoListeners() { + for (const [page, listener] of [...this._videoCloseListeners.entries()]) { + if (typeof page.off === 'function') { + page.off('close', listener); + } + else if (typeof page.removeListener === 'function') { + page.removeListener('close', listener); + } + } + this._videoCloseListeners.clear(); + } + _getKnownPages() { + const pagesFromContext = typeof this.browser_session.browser_context?.pages === 'function' + ? this.browser_session.browser_context.pages() + : []; + const activePage = this.browser_session + .agent_current_page; + if (!activePage) { + return pagesFromContext; + } + if (pagesFromContext.includes(activePage)) { + return pagesFromContext; + } + return [...pagesFromContext, activePage]; + } + async _captureVideoArtifact(page) { + try { + const video = typeof page.video === 'function' ? page.video() : null; + const videoPath = await video?.path?.(); + if (typeof videoPath === 'string' && videoPath.length > 0) { + this.browser_session.add_downloaded_file(videoPath); + } + } + catch (error) { + this.browser_session.logger.debug(`[RecordingWatchdog] Failed to capture video artifact: ${error.message}`); + } + } + async _startCdpScreencastIfConfigured() { + const configuredPath = this.browser_session.browser_profile.config.record_video_dir; + if (typeof configuredPath !== 'string' || configuredPath.trim() === '') { + return; + } + if (this._cdpScreencastSession || this._cdpScreencastStream) { + return; + } + try { + const page = await this.browser_session.get_current_page(); + if (!page) { + return; + } + const session = (await this.browser_session.get_or_create_cdp_session(page)); + const filePath = path.join(configuredPath, `${Date.now()}-${this.browser_session.id.slice(-6)}.cdp-screencast.ndjson`); + const stream = fs.createWriteStream(filePath, { flags: 'a' }); + const handler = (payload) => { + const frameData = typeof payload?.data === 'string' ? payload.data : ''; + if (frameData && this._cdpScreencastStream) { + this._cdpScreencastStream.write(`${JSON.stringify({ + ts: Date.now(), + session_id: typeof payload?.sessionId === 'number' || + typeof payload?.sessionId === 'string' + ? payload.sessionId + : null, + data: frameData, + })}\n`); + } + const sessionId = typeof payload?.sessionId === 'number' || + typeof payload?.sessionId === 'string' + ? payload.sessionId + : null; + if (!sessionId) { + return; + } + void session.send?.('Page.screencastFrameAck', { + sessionId, + }); + }; + await session.send?.('Page.enable'); + session.on?.('Page.screencastFrame', handler); + await session.send?.('Page.startScreencast', { + format: 'jpeg', + quality: 85, + everyNthFrame: 1, + }); + this._cdpScreencastSession = session; + this._cdpScreencastHandler = handler; + this._cdpScreencastPath = filePath; + this._cdpScreencastStream = stream; + } + catch (error) { + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'RecordingCdpStartFailed', + message: `Failed to start CDP screencast recording: ${error.message}`, + details: { + record_video_dir: this.browser_session.browser_profile.config.record_video_dir, + }, + })); + await this._stopCdpScreencastIfStarted(); + } + } + async _stopCdpScreencastIfStarted() { + if (!this._cdpScreencastSession) { + return; + } + try { + await this._cdpScreencastSession.send?.('Page.stopScreencast'); + } + catch { + // Ignore stop errors. + } + if (this._cdpScreencastHandler) { + this._cdpScreencastSession.off?.('Page.screencastFrame', this._cdpScreencastHandler); + } + if (this._cdpScreencastStream) { + await new Promise((resolve) => { + this._cdpScreencastStream?.end(() => resolve()); + }); + } + if (this._cdpScreencastPath && + fs.existsSync(this._cdpScreencastPath) && + fs.statSync(this._cdpScreencastPath).size > 0) { + this.browser_session.add_downloaded_file(this._cdpScreencastPath); + } + this._cdpScreencastSession = null; + this._cdpScreencastHandler = null; + this._cdpScreencastPath = null; + this._cdpScreencastStream = null; + } +} diff --git a/dist/browser/watchdogs/screenshot-watchdog.d.ts b/dist/browser/watchdogs/screenshot-watchdog.d.ts new file mode 100644 index 00000000..069d398d --- /dev/null +++ b/dist/browser/watchdogs/screenshot-watchdog.d.ts @@ -0,0 +1,6 @@ +import { ScreenshotEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export declare class ScreenshotWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof ScreenshotEvent)[]; + on_ScreenshotEvent(event: ScreenshotEvent): Promise; +} diff --git a/dist/browser/watchdogs/screenshot-watchdog.js b/dist/browser/watchdogs/screenshot-watchdog.js new file mode 100644 index 00000000..d4ddcedd --- /dev/null +++ b/dist/browser/watchdogs/screenshot-watchdog.js @@ -0,0 +1,14 @@ +import { ScreenshotEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class ScreenshotWatchdog extends BaseWatchdog { + static LISTENS_TO = [ScreenshotEvent]; + async on_ScreenshotEvent(event) { + try { + await this.browser_session.remove_highlights(); + } + catch { + // Highlight cleanup is best-effort and should not block screenshots. + } + return await this.browser_session.take_screenshot(event.full_page, event.clip); + } +} diff --git a/dist/browser/watchdogs/security-watchdog.d.ts b/dist/browser/watchdogs/security-watchdog.d.ts new file mode 100644 index 00000000..ecd12b72 --- /dev/null +++ b/dist/browser/watchdogs/security-watchdog.d.ts @@ -0,0 +1,10 @@ +import { BrowserErrorEvent, CloseTabEvent, NavigateToUrlEvent, NavigationCompleteEvent, TabCreatedEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export declare class SecurityWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof NavigateToUrlEvent | typeof TabCreatedEvent)[]; + static EMITS: (typeof CloseTabEvent | typeof BrowserErrorEvent)[]; + on_NavigateToUrlEvent(event: NavigateToUrlEvent): Promise; + on_NavigationCompleteEvent(event: NavigationCompleteEvent): Promise; + on_TabCreatedEvent(event: TabCreatedEvent): Promise; + private _getUrlDenialReason; +} diff --git a/dist/browser/watchdogs/security-watchdog.js b/dist/browser/watchdogs/security-watchdog.js new file mode 100644 index 00000000..22ed5ffe --- /dev/null +++ b/dist/browser/watchdogs/security-watchdog.js @@ -0,0 +1,84 @@ +import { BrowserErrorEvent, CloseTabEvent, NavigateToUrlEvent, NavigationCompleteEvent, TabCreatedEvent, } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class SecurityWatchdog extends BaseWatchdog { + static LISTENS_TO = [ + NavigateToUrlEvent, + NavigationCompleteEvent, + TabCreatedEvent, + ]; + static EMITS = [BrowserErrorEvent, CloseTabEvent]; + async on_NavigateToUrlEvent(event) { + const denialReason = this._getUrlDenialReason(event.url); + if (!denialReason) { + return; + } + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'NavigationBlocked', + message: `Navigation blocked to disallowed URL: ${event.url}`, + details: { + url: event.url, + reason: denialReason, + }, + })); + throw new Error(`Navigation to ${event.url} blocked by security policy`); + } + async on_NavigationCompleteEvent(event) { + const denialReason = this._getUrlDenialReason(event.url); + if (!denialReason) { + return; + } + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'NavigationBlocked', + message: `Navigation blocked to non-allowed URL: ${event.url} - redirecting to about:blank`, + details: { + url: event.url, + target_id: event.target_id, + reason: denialReason, + }, + })); + try { + await this.browser_session.navigate_to('about:blank'); + } + catch (error) { + this.browser_session.logger.debug(`SecurityWatchdog failed to redirect to about:blank: ${error.message}`); + } + } + async on_TabCreatedEvent(event) { + const denialReason = this._getUrlDenialReason(event.url); + if (!denialReason) { + return; + } + await this.event_bus.dispatch(new BrowserErrorEvent({ + error_type: 'TabCreationBlocked', + message: `Tab created with non-allowed URL: ${event.url}`, + details: { + url: event.url, + target_id: event.target_id, + reason: denialReason, + }, + })); + await this.event_bus.dispatch(new CloseTabEvent({ + target_id: event.target_id, + })); + } + _getUrlDenialReason(url) { + const session = this.browser_session; + if (typeof session._get_url_access_denial_reason === 'function') { + try { + return session._get_url_access_denial_reason(url); + } + catch { + // Ignore private method failures and fallback to boolean check. + } + } + if (typeof session._is_url_allowed === 'function') { + try { + return session._is_url_allowed(url) ? null : 'blocked'; + } + catch { + return 'blocked'; + } + } + return null; + } +} diff --git a/dist/browser/watchdogs/storage-state-watchdog.d.ts b/dist/browser/watchdogs/storage-state-watchdog.d.ts new file mode 100644 index 00000000..a2b52166 --- /dev/null +++ b/dist/browser/watchdogs/storage-state-watchdog.d.ts @@ -0,0 +1,24 @@ +import { BrowserConnectedEvent, BrowserStopEvent, LoadStorageStateEvent, SaveStorageStateEvent, StorageStateSavedEvent } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export declare class StorageStateWatchdog extends BaseWatchdog { + static LISTENS_TO: (typeof BrowserStopEvent | typeof BrowserConnectedEvent | typeof SaveStorageStateEvent)[]; + static EMITS: (typeof StorageStateSavedEvent)[]; + private _monitorInterval; + private _autoSaveIntervalMs; + private _monitoring; + private _lastSavedSnapshot; + on_BrowserConnectedEvent(): Promise; + on_BrowserStopEvent(): Promise; + on_SaveStorageStateEvent(event: SaveStorageStateEvent): Promise; + on_LoadStorageStateEvent(event: LoadStorageStateEvent): Promise; + protected onDetached(): void; + private _resolveStoragePath; + private _startMonitoring; + private _stopMonitoring; + private _checkAndAutoSave; + private _snapshotStorageState; + private _readStoredState; + private _mergeStorageStates; + private _applyOriginsStorage; + private _normalizeStorageEntries; +} diff --git a/dist/browser/watchdogs/storage-state-watchdog.js b/dist/browser/watchdogs/storage-state-watchdog.js new file mode 100644 index 00000000..7b65b17c --- /dev/null +++ b/dist/browser/watchdogs/storage-state-watchdog.js @@ -0,0 +1,288 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { BrowserConnectedEvent, BrowserStopEvent, LoadStorageStateEvent, SaveStorageStateEvent, StorageStateLoadedEvent, StorageStateSavedEvent, } from '../events.js'; +import { BaseWatchdog } from './base.js'; +export class StorageStateWatchdog extends BaseWatchdog { + static LISTENS_TO = [ + BrowserConnectedEvent, + BrowserStopEvent, + SaveStorageStateEvent, + LoadStorageStateEvent, + ]; + static EMITS = [StorageStateSavedEvent, StorageStateLoadedEvent]; + _monitorInterval = null; + _autoSaveIntervalMs = 30_000; + _monitoring = false; + _lastSavedSnapshot = null; + async on_BrowserConnectedEvent() { + this._startMonitoring(); + await this.event_bus.dispatch(new LoadStorageStateEvent()); + } + async on_BrowserStopEvent() { + this._stopMonitoring(); + await this.event_bus.dispatch(new SaveStorageStateEvent()); + } + async on_SaveStorageStateEvent(event) { + const targetPath = this._resolveStoragePath(event.path); + if (!targetPath) { + return; + } + const browserContext = this.browser_session.browser_context; + if (!browserContext?.storageState) { + this.browser_session.logger.debug('[StorageStateWatchdog] Browser context unavailable for save'); + return; + } + const storageState = (await browserContext.storageState()); + const normalized = storageState ?? {}; + const merged = this._mergeStorageStates(this._readStoredState(targetPath), { + cookies: Array.isArray(normalized.cookies) ? normalized.cookies : [], + origins: Array.isArray(normalized.origins) ? normalized.origins : [], + }); + this._lastSavedSnapshot = this._snapshotStorageState(merged); + const dirPath = path.dirname(targetPath); + fs.mkdirSync(dirPath, { recursive: true }); + const tempPath = `${targetPath}.tmp`; + fs.writeFileSync(tempPath, JSON.stringify(merged, null, 2)); + if (fs.existsSync(targetPath)) { + const backupPath = `${targetPath}.bak`; + try { + fs.renameSync(targetPath, backupPath); + } + catch { + // Ignore backup failures and continue with atomic swap. + } + } + fs.renameSync(tempPath, targetPath); + await this.event_bus.dispatch(new StorageStateSavedEvent({ + path: targetPath, + cookies_count: Array.isArray(merged.cookies) + ? merged.cookies.length + : 0, + origins_count: Array.isArray(merged.origins) + ? merged.origins.length + : 0, + })); + } + async on_LoadStorageStateEvent(event) { + const targetPath = this._resolveStoragePath(event.path); + if (!targetPath || !fs.existsSync(targetPath)) { + return; + } + const browserContext = this.browser_session.browser_context; + if (!browserContext) { + this.browser_session.logger.debug('[StorageStateWatchdog] Browser context unavailable for load'); + return; + } + const raw = fs.readFileSync(targetPath, 'utf-8'); + const parsed = JSON.parse(raw); + const cookies = Array.isArray(parsed.cookies) ? parsed.cookies : []; + const origins = Array.isArray(parsed.origins) ? parsed.origins : []; + this._lastSavedSnapshot = this._snapshotStorageState({ + cookies, + origins, + }); + if (cookies.length > 0 && typeof browserContext.addCookies === 'function') { + await browserContext.addCookies(cookies); + } + if (origins.length > 0) { + await this._applyOriginsStorage(origins); + } + await this.event_bus.dispatch(new StorageStateLoadedEvent({ + path: targetPath, + cookies_count: cookies.length, + origins_count: origins.length, + })); + } + onDetached() { + this._stopMonitoring(); + } + _resolveStoragePath(pathFromEvent) { + if (typeof pathFromEvent === 'string' && pathFromEvent.trim().length > 0) { + return path.resolve(pathFromEvent); + } + const configured = this.browser_session.browser_profile.config.storage_state; + if (typeof configured === 'string' && configured.trim().length > 0) { + return path.resolve(configured); + } + const cookiesFile = this.browser_session.browser_profile.cookies_file; + if (typeof cookiesFile === 'string' && cookiesFile.trim().length > 0) { + return path.resolve(cookiesFile); + } + return null; + } + _startMonitoring() { + if (this._monitorInterval) { + return; + } + if (!this._resolveStoragePath(null)) { + return; + } + this._monitorInterval = setInterval(() => { + void this._checkAndAutoSave().catch((error) => { + this.browser_session.logger.debug(`[StorageStateWatchdog] Auto-save monitor failed: ${error.message}`); + }); + }, this._autoSaveIntervalMs); + } + _stopMonitoring() { + if (!this._monitorInterval) { + return; + } + clearInterval(this._monitorInterval); + this._monitorInterval = null; + } + async _checkAndAutoSave() { + if (this._monitoring) { + return; + } + const browserContext = this.browser_session.browser_context; + if (!browserContext?.storageState) { + return; + } + this._monitoring = true; + try { + const storageState = (await browserContext.storageState()); + const normalized = storageState ?? {}; + const snapshot = this._snapshotStorageState(normalized); + if (snapshot === this._lastSavedSnapshot) { + return; + } + await this.event_bus.dispatch(new SaveStorageStateEvent()); + } + finally { + this._monitoring = false; + } + } + _snapshotStorageState(state) { + const cookies = Array.isArray(state.cookies) ? state.cookies : []; + const origins = Array.isArray(state.origins) ? state.origins : []; + return JSON.stringify({ + cookies, + origins, + }); + } + _readStoredState(targetPath) { + if (!fs.existsSync(targetPath)) { + return { + cookies: [], + origins: [], + }; + } + try { + const raw = fs.readFileSync(targetPath, 'utf-8'); + const parsed = JSON.parse(raw); + return { + cookies: Array.isArray(parsed.cookies) ? parsed.cookies : [], + origins: Array.isArray(parsed.origins) ? parsed.origins : [], + }; + } + catch (error) { + this.browser_session.logger.debug(`[StorageStateWatchdog] Failed to parse existing storage state: ${error.message}`); + return { + cookies: [], + origins: [], + }; + } + } + _mergeStorageStates(existing, incoming) { + const mergedCookies = new Map(); + const toCookieKey = (cookie) => `${String(cookie?.name ?? '')}::${String(cookie?.domain ?? '')}::${String(cookie?.path ?? '')}`; + for (const cookie of Array.isArray(existing.cookies) + ? existing.cookies + : []) { + mergedCookies.set(toCookieKey(cookie), cookie); + } + for (const cookie of Array.isArray(incoming.cookies) + ? incoming.cookies + : []) { + mergedCookies.set(toCookieKey(cookie), cookie); + } + const mergedOrigins = new Map(); + const toOriginKey = (origin) => String(origin?.origin ?? ''); + for (const origin of Array.isArray(existing.origins) + ? existing.origins + : []) { + mergedOrigins.set(toOriginKey(origin), origin); + } + for (const origin of Array.isArray(incoming.origins) + ? incoming.origins + : []) { + mergedOrigins.set(toOriginKey(origin), origin); + } + return { + cookies: [...mergedCookies.values()], + origins: [...mergedOrigins.values()], + }; + } + async _applyOriginsStorage(origins) { + const browserContext = this.browser_session.browser_context; + if (!browserContext?.newPage) { + return; + } + for (const originState of origins) { + const origin = typeof originState?.origin === 'string' + ? originState.origin.trim() + : ''; + if (!origin || !/^https?:\/\//i.test(origin)) { + continue; + } + const localStorageEntries = this._normalizeStorageEntries(originState?.localStorage); + const sessionStorageEntries = this._normalizeStorageEntries(originState?.sessionStorage); + if (localStorageEntries.length === 0 && + sessionStorageEntries.length === 0) { + continue; + } + let page = null; + try { + page = await browserContext.newPage(); + await page.goto?.(origin, { + waitUntil: 'domcontentloaded', + timeout: 5_000, + }); + await page.evaluate?.((payload) => { + for (const entry of payload.localStorageEntries) { + window.localStorage.setItem(entry.name, entry.value); + } + for (const entry of payload.sessionStorageEntries) { + window.sessionStorage.setItem(entry.name, entry.value); + } + }, { + localStorageEntries, + sessionStorageEntries, + }); + } + catch (error) { + this.browser_session.logger.debug(`[StorageStateWatchdog] Failed to apply origin storage for ${origin}: ${error.message}`); + } + finally { + try { + await page?.close?.(); + } + catch { + // Ignore cleanup errors. + } + } + } + } + _normalizeStorageEntries(entries) { + if (!Array.isArray(entries)) { + return []; + } + const normalized = []; + for (const entry of entries) { + const name = entry && typeof entry === 'object' && 'name' in entry + ? String(entry.name ?? '') + : ''; + if (!name) { + continue; + } + const value = entry && typeof entry === 'object' && 'value' in entry + ? String(entry.value ?? '') + : ''; + normalized.push({ + name, + value, + }); + } + return normalized; + } +} diff --git a/dist/cli.d.ts b/dist/cli.d.ts new file mode 100644 index 00000000..d437786f --- /dev/null +++ b/dist/cli.d.ts @@ -0,0 +1,163 @@ +#!/usr/bin/env node +import { spawnSync } from 'node:child_process'; +import { BrowserProfile } from './browser/profile.js'; +import { CloudBrowserClient } from './browser/cloud/cloud.js'; +import { CloudManagementClient } from './browser/cloud/management.js'; +import type { BaseChatModel } from './llm/base.js'; +import { get_tunnel_manager } from './skill-cli/tunnel.js'; +type CliModelProvider = 'openai' | 'anthropic' | 'google' | 'deepseek' | 'groq' | 'openrouter' | 'azure' | 'mistral' | 'cerebras' | 'vercel' | 'oci' | 'aws-anthropic' | 'aws' | 'ollama' | 'browser-use'; +export interface ParsedCliArgs { + help: boolean; + version: boolean; + debug: boolean; + headless: boolean | null; + window_width: number | null; + window_height: number | null; + user_data_dir: string | null; + profile_directory: string | null; + allowed_domains: string[] | null; + proxy_url: string | null; + no_proxy: string | null; + proxy_username: string | null; + proxy_password: string | null; + cdp_url: string | null; + model: string | null; + provider: CliModelProvider | null; + prompt: string | null; + mcp: boolean; + json: boolean; + yes: boolean; + setup_mode: string | null; + api_key: string | null; + positional: string[]; +} +export declare const CLI_HISTORY_LIMIT = 100; +export declare const parseCliArgs: (argv: string[]) => ParsedCliArgs; +export declare const isInteractiveExitCommand: (value: string) => boolean; +export declare const isInteractiveHelpCommand: (value: string) => boolean; +export declare const normalizeCliHistory: (history: unknown[], maxLength?: number) => string[]; +export declare const getCliHistoryPath: (configDir?: string | null) => string; +export declare const loadCliHistory: (historyPath?: string) => Promise; +export declare const saveCliHistory: (history: string[], historyPath?: string) => Promise; +export declare const shouldStartInteractiveMode: (task: string | null, options?: { + forceInteractive?: boolean; + inputIsTTY?: boolean; + outputIsTTY?: boolean; +}) => boolean; +export declare const getLlmFromCliArgs: (args: ParsedCliArgs) => BaseChatModel; +export declare const buildBrowserProfileFromCliArgs: (args: ParsedCliArgs) => BrowserProfile | null; +export declare const getCliUsage: () => string; +export interface RunInstallCommandOptions { + playwright_cli_path?: string; + spawn_impl?: typeof spawnSync; +} +type WritableLike = { + write(chunk: string): unknown; +}; +export interface RunTunnelCommandOptions { + manager?: Pick, 'start_tunnel' | 'list_tunnels' | 'stop_tunnel' | 'stop_all_tunnels'>; + stdout?: WritableLike; + stderr?: WritableLike; + json_output?: boolean; +} +export interface RunSetupCommandOptions { + run_doctor_checks?: (options?: RunDoctorChecksOptions) => Promise; + install_command?: () => void | Promise; + save_api_key?: (api_key: string) => void; + stdout?: WritableLike; + stderr?: WritableLike; + json_output?: boolean; +} +export interface RunTaskCommandOptions { + client?: Pick; + stdout?: WritableLike; + stderr?: WritableLike; +} +export interface RunSessionCommandOptions { + client?: Pick; + stdout?: WritableLike; + stderr?: WritableLike; +} +export interface RunProfileCommandOptions { + client?: Pick; + profile_lister?: () => Array<{ + directory: string; + name: string; + email?: string; + }>; + local_session_factory?: (profile_directory: string) => { + start: () => Promise; + stop?: () => Promise; + get_cookies?: () => Promise; + }; + remote_session_factory?: (init: { + cdp_url: string; + }) => { + start: () => Promise; + stop?: () => Promise; + browser_context?: { + addCookies?: (cookies: BrowserCookieInit[]) => Promise; + } | null; + }; + cloud_browser_client_factory?: () => Pick; + stdout?: WritableLike; + stderr?: WritableLike; +} +export interface RunCloudTaskCommandOptions { + client?: Pick; + stdout?: WritableLike; + stderr?: WritableLike; + sleep_impl?: (ms: number) => Promise; +} +type BrowserCookieInit = { + name: string; + value: string; + url?: string; + domain?: string; + path?: string; + expires?: number; + httpOnly?: boolean; + secure?: boolean; + sameSite?: 'Strict' | 'Lax' | 'None'; + partitionKey?: string; +}; +export declare const runInstallCommand: (options?: RunInstallCommandOptions) => void; +export declare const runTunnelCommand: (argv: string[], options?: RunTunnelCommandOptions) => Promise<0 | 1>; +export declare const runSetupCommand: (params: { + mode?: string | null; + yes?: boolean; + api_key?: string | null; +}, options?: RunSetupCommandOptions) => Promise; +export declare const runTaskCommand: (argv: string[], options?: RunTaskCommandOptions) => Promise<0 | 1>; +export declare const runSessionCommand: (argv: string[], options?: RunSessionCommandOptions) => Promise<0 | 1>; +export declare const runProfileCommand: (argv: string[], options?: RunProfileCommandOptions) => Promise<0 | 1>; +export declare const hasCloudRunFlags: (argv: string[]) => boolean; +type PrefixedSubcommand = { + command: 'run' | 'task' | 'session' | 'profile'; + argv: string[]; + debug: boolean; + forwardedArgs: string[]; +}; +export declare const extractPrefixedSubcommand: (argv: string[]) => PrefixedSubcommand | null; +export declare const runCloudTaskCommand: (argv: string[], options?: RunCloudTaskCommandOptions) => Promise<0 | 1>; +export interface CliDoctorCheck { + status: 'ok' | 'warning' | 'missing' | 'error'; + message: string; + note?: string; + fix?: string; +} +export interface CliDoctorReport { + status: 'healthy' | 'issues_found'; + checks: Record; + summary: string; +} +export interface RunDoctorChecksOptions { + version?: string; + browser_executable?: string | null; + api_key?: string | null; + cloudflared_path?: string | null; + fetch_impl?: typeof fetch; +} +export declare const runDoctorChecks: (options?: RunDoctorChecksOptions) => Promise; +export declare function main(argv?: string[]): Promise; +export {}; diff --git a/dist/cli.js b/dist/cli.js new file mode 100644 index 00000000..3826b3cd --- /dev/null +++ b/dist/cli.js @@ -0,0 +1,2666 @@ +#!/usr/bin/env node +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { stdin, stdout } from 'node:process'; +import { createInterface } from 'node:readline/promises'; +import { Agent } from './agent/service.js'; +import { BrowserProfile, } from './browser/profile.js'; +import { BrowserSession, systemChrome } from './browser/session.js'; +import { CONFIG } from './config.js'; +import { CloudBrowserClient } from './browser/cloud/cloud.js'; +import { CloudManagementClient, } from './browser/cloud/management.js'; +import { ChatOpenAI } from './llm/openai/chat.js'; +import { ChatAnthropic } from './llm/anthropic/chat.js'; +import { ChatGoogle } from './llm/google/chat.js'; +import { ChatDeepSeek } from './llm/deepseek/chat.js'; +import { ChatGroq } from './llm/groq/chat.js'; +import { ChatOpenRouter } from './llm/openrouter/chat.js'; +import { ChatAzure } from './llm/azure/chat.js'; +import { ChatOllama } from './llm/ollama/chat.js'; +import { ChatMistral } from './llm/mistral/chat.js'; +import { ChatOCIRaw } from './llm/oci-raw/chat.js'; +import { ChatCerebras } from './llm/cerebras/chat.js'; +import { ChatVercel } from './llm/vercel/chat.js'; +import { ChatAnthropicBedrock } from './llm/aws/chat-anthropic.js'; +import { ChatBedrockConverse } from './llm/aws/chat-bedrock.js'; +import { ChatBrowserUse } from './llm/browser-use/chat.js'; +import { MCPServer } from './mcp/server.js'; +import { get_browser_use_version } from './utils.js'; +import { setupLogging } from './logging-config.js'; +import { get_tunnel_manager } from './skill-cli/tunnel.js'; +import { DeviceAuthClient, save_cloud_api_token } from './sync/auth.js'; +import dotenv from 'dotenv'; +dotenv.config(); +const require = createRequire(import.meta.url); +const CLI_PROVIDER_ALIASES = { + openai: 'openai', + anthropic: 'anthropic', + google: 'google', + gemini: 'google', + deepseek: 'deepseek', + groq: 'groq', + openrouter: 'openrouter', + azure: 'azure', + mistral: 'mistral', + cerebras: 'cerebras', + vercel: 'vercel', + oci: 'oci', + ollama: 'ollama', + 'browser-use': 'browser-use', + browseruse: 'browser-use', + bu: 'browser-use', + bedrock: 'aws', + aws: 'aws', + 'aws-anthropic': 'aws-anthropic', + 'bedrock-anthropic': 'aws-anthropic', +}; +export const CLI_HISTORY_LIMIT = 100; +const INTERACTIVE_EXIT_COMMANDS = new Set(['exit', 'quit', ':q', '/q', '.q']); +const INTERACTIVE_HELP_COMMANDS = new Set(['help', '?', ':help']); +const parseAllowedDomains = (value) => { + const domains = value + .split(',') + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); + if (domains.length === 0) { + throw new Error('--allowed-domains must include at least one domain pattern'); + } + return domains; +}; +const parsePositiveInt = (name, value) => { + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || Number.isNaN(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer, got "${value}"`); + } + return parsed; +}; +const parseProvider = (value) => { + const normalized = value.trim().toLowerCase(); + const provider = CLI_PROVIDER_ALIASES[normalized]; + if (!provider) { + throw new Error(`Unsupported provider "${value}". Supported values: openai, anthropic, google, deepseek, groq, openrouter, azure, mistral, cerebras, vercel, oci, ollama, browser-use, aws, aws-anthropic.`); + } + return provider; +}; +const takeOptionValue = (arg, currentIndex, argv) => { + const eqIndex = arg.indexOf('='); + if (eqIndex >= 0) { + const inlineValue = arg.slice(eqIndex + 1).trim(); + if (!inlineValue) { + throw new Error(`Missing value for option: ${arg.slice(0, eqIndex)}`); + } + return { value: inlineValue, nextIndex: currentIndex }; + } + const next = argv[currentIndex + 1]; + if (!next || next.startsWith('-')) { + throw new Error(`Missing value for option: ${arg}`); + } + return { value: next, nextIndex: currentIndex + 1 }; +}; +const expandHome = (value) => { + if (!value.startsWith('~')) { + return value; + } + const home = process.env.HOME || process.env.USERPROFILE || ''; + if (!home) { + return value; + } + if (value === '~') { + return home; + } + if (value.startsWith('~/') || value.startsWith('~\\')) { + return path.join(home, value.slice(2)); + } + return value; +}; +export const parseCliArgs = (argv) => { + const parsed = { + help: false, + version: false, + debug: false, + headless: null, + window_width: null, + window_height: null, + user_data_dir: null, + profile_directory: null, + allowed_domains: null, + proxy_url: null, + no_proxy: null, + proxy_username: null, + proxy_password: null, + cdp_url: null, + model: null, + provider: null, + prompt: null, + mcp: false, + json: false, + yes: false, + setup_mode: null, + api_key: null, + positional: [], + }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--') { + parsed.positional.push(...argv.slice(i + 1)); + break; + } + if (arg === '-h' || arg === '--help') { + parsed.help = true; + continue; + } + if (arg === '--version') { + parsed.version = true; + continue; + } + if (arg === '--debug') { + parsed.debug = true; + continue; + } + if (arg === '--headless') { + parsed.headless = true; + continue; + } + if (arg === '--mcp') { + parsed.mcp = true; + continue; + } + if (arg === '--json') { + parsed.json = true; + continue; + } + if (arg === '-y' || arg === '--yes') { + parsed.yes = true; + continue; + } + if (arg === '-p' || arg === '--prompt' || arg.startsWith('--prompt=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.prompt = value; + i = nextIndex; + continue; + } + if (arg === '--model' || arg.startsWith('--model=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.model = value; + i = nextIndex; + continue; + } + if (arg === '--provider' || arg.startsWith('--provider=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.provider = parseProvider(value); + i = nextIndex; + continue; + } + if (arg === '--mode' || arg.startsWith('--mode=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.setup_mode = value.trim(); + i = nextIndex; + continue; + } + if (arg === '--api-key' || arg.startsWith('--api-key=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.api_key = value.trim(); + i = nextIndex; + continue; + } + if (arg === '--window-width' || arg.startsWith('--window-width=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.window_width = parsePositiveInt('--window-width', value); + i = nextIndex; + continue; + } + if (arg === '--window-height' || arg.startsWith('--window-height=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.window_height = parsePositiveInt('--window-height', value); + i = nextIndex; + continue; + } + if (arg === '--user-data-dir' || arg.startsWith('--user-data-dir=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.user_data_dir = path.resolve(expandHome(value)); + i = nextIndex; + continue; + } + if (arg === '--profile-directory' || + arg.startsWith('--profile-directory=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.profile_directory = value; + i = nextIndex; + continue; + } + if (arg === '--allowed-domains' || arg.startsWith('--allowed-domains=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + const domains = parseAllowedDomains(value); + parsed.allowed_domains = [...(parsed.allowed_domains ?? []), ...domains]; + i = nextIndex; + continue; + } + if (arg === '--proxy-url' || arg.startsWith('--proxy-url=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.proxy_url = value.trim(); + i = nextIndex; + continue; + } + if (arg === '--no-proxy' || arg.startsWith('--no-proxy=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.no_proxy = value; + i = nextIndex; + continue; + } + if (arg === '--proxy-username' || arg.startsWith('--proxy-username=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.proxy_username = value; + i = nextIndex; + continue; + } + if (arg === '--proxy-password' || arg.startsWith('--proxy-password=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.proxy_password = value; + i = nextIndex; + continue; + } + if (arg === '--cdp-url' || arg.startsWith('--cdp-url=')) { + const { value, nextIndex } = takeOptionValue(arg, i, argv); + parsed.cdp_url = value; + i = nextIndex; + continue; + } + if (arg.startsWith('-')) { + throw new Error(`Unknown option: ${arg}`); + } + parsed.positional.push(arg); + } + if (parsed.prompt && parsed.positional.length > 0) { + throw new Error('Use either positional task text or --prompt, not both.'); + } + return parsed; +}; +const resolveTask = (args) => { + if (args.prompt) { + return args.prompt.trim(); + } + if (args.positional.length > 0) { + return args.positional.join(' ').trim(); + } + return null; +}; +export const isInteractiveExitCommand = (value) => INTERACTIVE_EXIT_COMMANDS.has(value.trim().toLowerCase()); +export const isInteractiveHelpCommand = (value) => INTERACTIVE_HELP_COMMANDS.has(value.trim().toLowerCase()); +export const normalizeCliHistory = (history, maxLength = CLI_HISTORY_LIMIT) => { + const normalized = history + .map((entry) => (typeof entry === 'string' ? entry.trim() : '')) + .filter((entry) => entry.length > 0); + return normalized.slice(-maxLength); +}; +export const getCliHistoryPath = (configDir) => { + const baseDir = configDir ?? + CONFIG.BROWSER_USE_CONFIG_DIR ?? + path.join(os.homedir(), '.config', 'browseruse'); + return path.join(baseDir, 'command_history.json'); +}; +export const loadCliHistory = async (historyPath = getCliHistoryPath()) => { + try { + const raw = await fs.readFile(historyPath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) { + return []; + } + return normalizeCliHistory(parsed); + } + catch (error) { + const nodeError = error; + if (nodeError.code === 'ENOENT') { + return []; + } + return []; + } +}; +export const saveCliHistory = async (history, historyPath = getCliHistoryPath()) => { + const normalized = normalizeCliHistory(history); + await fs.mkdir(path.dirname(historyPath), { recursive: true }); + await fs.writeFile(historyPath, JSON.stringify(normalized, null, 2), 'utf-8'); +}; +export const shouldStartInteractiveMode = (task, options = {}) => { + const forceInteractive = options.forceInteractive ?? + process.env.BROWSER_USE_CLI_FORCE_INTERACTIVE === '1'; + const inputIsTTY = options.inputIsTTY ?? Boolean(stdin.isTTY); + const outputIsTTY = options.outputIsTTY ?? Boolean(stdout.isTTY); + return !task && (forceInteractive || (inputIsTTY && outputIsTTY)); +}; +const requireEnv = (name) => { + const value = process.env[name]; + if (!value) { + throw new Error(`Missing environment variable: ${name}`); + } + return value; +}; +const inferProviderFromModel = (model) => { + const lower = model.toLowerCase(); + if (lower.startsWith('gpt') || + lower.startsWith('o1') || + lower.startsWith('o3') || + lower.startsWith('o4') || + lower.startsWith('gpt-5')) { + return 'openai'; + } + if (lower.startsWith('claude')) { + return 'anthropic'; + } + if (lower.startsWith('gemini')) { + return 'google'; + } + if (lower.startsWith('deepseek')) { + return 'deepseek'; + } + if (lower.startsWith('groq:')) { + return 'groq'; + } + if (lower.startsWith('openrouter:')) { + return 'openrouter'; + } + if (lower.startsWith('azure:')) { + return 'azure'; + } + if (lower.startsWith('mistral:')) { + return 'mistral'; + } + if (lower.startsWith('cerebras:')) { + return 'cerebras'; + } + if (lower.startsWith('vercel:')) { + return 'vercel'; + } + if (lower.startsWith('oci:')) { + return 'oci'; + } + if (lower.startsWith('mistral-') || + lower.startsWith('codestral') || + lower.startsWith('pixtral')) { + return 'mistral'; + } + if (lower.startsWith('llama3.') || + lower.startsWith('llama-4-') || + lower.startsWith('gpt-oss-') || + lower.startsWith('qwen-3-')) { + return 'cerebras'; + } + if (lower.startsWith('ollama:')) { + return 'ollama'; + } + if (lower.startsWith('browser-use:') || + lower.startsWith('bu-') || + lower.startsWith('browser-use/')) { + return 'browser-use'; + } + if (lower.startsWith('bedrock:anthropic.')) { + return 'aws-anthropic'; + } + if (lower.startsWith('bedrock:')) { + return 'aws'; + } + if (lower.startsWith('anthropic.')) { + return 'aws-anthropic'; + } + if (lower.includes('/') && + !lower.startsWith('http://') && + !lower.startsWith('https://')) { + return 'openrouter'; + } + return null; +}; +const normalizeModelValue = (model, provider) => { + const lower = model.toLowerCase(); + if (provider === 'groq' && lower.startsWith('groq:')) { + return model.slice('groq:'.length); + } + if (provider === 'openrouter' && lower.startsWith('openrouter:')) { + return model.slice('openrouter:'.length); + } + if (provider === 'azure' && lower.startsWith('azure:')) { + return model.slice('azure:'.length); + } + if (provider === 'mistral' && lower.startsWith('mistral:')) { + return model.slice('mistral:'.length); + } + if (provider === 'cerebras' && lower.startsWith('cerebras:')) { + return model.slice('cerebras:'.length); + } + if (provider === 'vercel' && lower.startsWith('vercel:')) { + return model.slice('vercel:'.length); + } + if (provider === 'oci' && lower.startsWith('oci:')) { + return model.slice('oci:'.length); + } + if (provider === 'ollama' && lower.startsWith('ollama:')) { + return model.slice('ollama:'.length); + } + if (provider === 'browser-use' && lower.startsWith('browser-use:')) { + return model.slice('browser-use:'.length); + } + if (provider === 'browser-use' && lower.startsWith('bu_')) { + return model.replace(/_/g, '-'); + } + if (provider === 'aws-anthropic' && lower.startsWith('bedrock:')) { + return model.slice('bedrock:'.length); + } + if (provider === 'aws' && lower.startsWith('bedrock:')) { + return model.slice('bedrock:'.length); + } + return model; +}; +const providersAreCompatible = (explicitProvider, inferredProvider) => { + if (explicitProvider === inferredProvider) { + return true; + } + if ((explicitProvider === 'aws' && inferredProvider === 'aws-anthropic') || + (explicitProvider === 'aws-anthropic' && inferredProvider === 'aws')) { + return true; + } + return false; +}; +const getDefaultModelForProvider = (provider) => { + switch (provider) { + case 'openai': + return 'gpt-5-mini'; + case 'anthropic': + return 'claude-4-sonnet'; + case 'google': + return 'gemini-2.5-pro'; + case 'deepseek': + return 'deepseek-chat'; + case 'groq': + return 'llama-3.1-70b-versatile'; + case 'openrouter': + return 'openai/gpt-5-mini'; + case 'azure': + return 'gpt-4o'; + case 'mistral': + return 'mistral-large-latest'; + case 'cerebras': + return 'llama3.1-8b'; + case 'vercel': + return 'openai/gpt-5-mini'; + case 'oci': + return null; + case 'aws-anthropic': + return 'anthropic.claude-3-5-sonnet-20241022-v2:0'; + case 'ollama': + return process.env.OLLAMA_MODEL || 'qwen2.5:latest'; + case 'browser-use': + return 'bu-latest'; + case 'aws': + return null; + default: + return null; + } +}; +const createLlmForProvider = (provider, model) => { + switch (provider) { + case 'openai': + return new ChatOpenAI({ + model, + apiKey: requireEnv('OPENAI_API_KEY'), + }); + case 'anthropic': + return new ChatAnthropic({ + model, + apiKey: requireEnv('ANTHROPIC_API_KEY'), + }); + case 'google': + requireEnv('GOOGLE_API_KEY'); + return new ChatGoogle(model); + case 'deepseek': + requireEnv('DEEPSEEK_API_KEY'); + return new ChatDeepSeek(model); + case 'groq': + requireEnv('GROQ_API_KEY'); + return new ChatGroq(model); + case 'openrouter': + requireEnv('OPENROUTER_API_KEY'); + return new ChatOpenRouter(model); + case 'azure': + requireEnv('AZURE_OPENAI_API_KEY'); + requireEnv('AZURE_OPENAI_ENDPOINT'); + return new ChatAzure(model); + case 'mistral': + return new ChatMistral({ + model, + apiKey: requireEnv('MISTRAL_API_KEY'), + baseURL: process.env.MISTRAL_BASE_URL, + }); + case 'cerebras': + return new ChatCerebras({ + model, + apiKey: requireEnv('CEREBRAS_API_KEY'), + baseURL: process.env.CEREBRAS_BASE_URL, + }); + case 'vercel': + return new ChatVercel({ + model, + apiKey: requireEnv('VERCEL_API_KEY'), + baseURL: process.env.VERCEL_BASE_URL, + }); + case 'oci': + return new ChatOCIRaw({ + model, + }); + case 'ollama': { + const host = process.env.OLLAMA_HOST || 'http://localhost:11434'; + return new ChatOllama(model, host); + } + case 'browser-use': + return new ChatBrowserUse({ + model, + apiKey: requireEnv('BROWSER_USE_API_KEY'), + }); + case 'aws-anthropic': + return new ChatAnthropicBedrock({ + model, + region: process.env.AWS_REGION || 'us-east-1', + }); + case 'aws': + return new ChatBedrockConverse(model, process.env.AWS_REGION || 'us-east-1'); + default: + throw new Error(`Unsupported provider "${provider}"`); + } +}; +export const getLlmFromCliArgs = (args) => { + if (args.model) { + const inferredProvider = inferProviderFromModel(args.model); + if (args.provider && + inferredProvider && + !providersAreCompatible(args.provider, inferredProvider)) { + throw new Error(`Provider mismatch: --provider ${args.provider} conflicts with model "${args.model}" (inferred: ${inferredProvider}).`); + } + const provider = args.provider ?? inferredProvider; + if (!provider) { + throw new Error(`Cannot infer provider from model "${args.model}". Provide --provider or use a supported model prefix: gpt*/o*, claude*, gemini*, deepseek*, groq:, openrouter:, azure:, mistral:, cerebras:, vercel:, oci:, ollama:, browser-use:, bu-*, bedrock:.`); + } + const normalizedModel = normalizeModelValue(args.model, provider); + return createLlmForProvider(provider, normalizedModel); + } + if (args.provider) { + const defaultModel = getDefaultModelForProvider(args.provider); + if (!defaultModel) { + throw new Error(`Provider "${args.provider}" requires --model. Example: --provider aws --model bedrock:us.amazon.nova-lite-v1:0`); + } + return createLlmForProvider(args.provider, defaultModel); + } + if (process.env.OPENAI_API_KEY) { + return new ChatOpenAI({ + model: 'gpt-5-mini', + apiKey: process.env.OPENAI_API_KEY, + }); + } + if (process.env.ANTHROPIC_API_KEY) { + return new ChatAnthropic({ + model: 'claude-4-sonnet', + apiKey: process.env.ANTHROPIC_API_KEY, + }); + } + if (process.env.GOOGLE_API_KEY) { + return new ChatGoogle('gemini-2.5-pro'); + } + if (process.env.DEEPSEEK_API_KEY) { + return new ChatDeepSeek('deepseek-chat'); + } + if (process.env.GROQ_API_KEY) { + return new ChatGroq('llama-3.1-70b-versatile'); + } + if (process.env.OPENROUTER_API_KEY) { + return new ChatOpenRouter('openai/gpt-5-mini'); + } + if (process.env.AZURE_OPENAI_API_KEY && process.env.AZURE_OPENAI_ENDPOINT) { + return new ChatAzure('gpt-4o'); + } + if (process.env.MISTRAL_API_KEY) { + return new ChatMistral({ + model: 'mistral-large-latest', + apiKey: process.env.MISTRAL_API_KEY, + baseURL: process.env.MISTRAL_BASE_URL, + }); + } + if (process.env.CEREBRAS_API_KEY) { + return new ChatCerebras({ + model: 'llama3.1-8b', + apiKey: process.env.CEREBRAS_API_KEY, + baseURL: process.env.CEREBRAS_BASE_URL, + }); + } + if (process.env.VERCEL_API_KEY) { + return new ChatVercel({ + model: 'openai/gpt-5-mini', + apiKey: process.env.VERCEL_API_KEY, + baseURL: process.env.VERCEL_BASE_URL, + }); + } + if (process.env.AWS_ACCESS_KEY_ID || process.env.AWS_PROFILE) { + return new ChatAnthropicBedrock({ + model: 'anthropic.claude-3-5-sonnet-20241022-v2:0', + region: process.env.AWS_REGION || 'us-east-1', + }); + } + return new ChatOllama(process.env.OLLAMA_MODEL || 'qwen2.5:latest', process.env.OLLAMA_HOST || 'http://localhost:11434'); +}; +const parseCommaSeparatedList = (value) => value + .split(',') + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); +export const buildBrowserProfileFromCliArgs = (args) => { + const profile = {}; + if (args.headless !== null) { + profile.headless = args.headless; + } + if (args.window_width !== null) { + profile.window_width = args.window_width; + } + if (args.window_height !== null) { + profile.window_height = args.window_height; + } + if (args.user_data_dir) { + profile.user_data_dir = args.user_data_dir; + } + if (args.profile_directory) { + profile.profile_directory = args.profile_directory; + } + if (args.allowed_domains && args.allowed_domains.length > 0) { + profile.allowed_domains = args.allowed_domains; + } + if (args.proxy_url || + args.no_proxy || + args.proxy_username || + args.proxy_password) { + const proxy = {}; + if (args.proxy_url) { + proxy.server = args.proxy_url; + } + if (args.no_proxy) { + proxy.bypass = parseCommaSeparatedList(args.no_proxy).join(','); + } + if (args.proxy_username) { + proxy.username = args.proxy_username; + } + if (args.proxy_password) { + proxy.password = args.proxy_password; + } + profile.proxy = proxy; + } + if (Object.keys(profile).length === 0) { + return null; + } + return new BrowserProfile(profile); +}; +const runAgentTask = async ({ task, llm, browserProfile, browserSession, sessionAttachmentMode, }) => { + const agent = new Agent({ + task, + llm, + ...(browserProfile ? { browser_profile: browserProfile } : {}), + ...(browserSession ? { browser_session: browserSession } : {}), + ...(sessionAttachmentMode + ? { session_attachment_mode: sessionAttachmentMode } + : {}), + source: 'cli', + }); + await agent.run(); +}; +const runInteractiveMode = async (args, llm) => { + const historyPath = getCliHistoryPath(); + const history = await loadCliHistory(historyPath); + const browserProfile = buildBrowserProfileFromCliArgs(args) ?? new BrowserProfile(); + browserProfile.keep_alive = true; + const browserSession = new BrowserSession({ + browser_profile: browserProfile, + ...(args.cdp_url ? { cdp_url: args.cdp_url } : {}), + }); + const rl = createInterface({ + input: stdin, + output: stdout, + terminal: true, + historySize: CLI_HISTORY_LIMIT, + }); + if (Array.isArray(rl.history) && history.length > 0) { + rl.history = [...history].reverse(); + } + console.log('Interactive mode started. Type a task and press Enter.'); + console.log('Commands: help, exit'); + try { + while (true) { + const line = await rl.question('browser-use> '); + const task = line.trim(); + if (!task) { + continue; + } + if (isInteractiveExitCommand(task)) { + break; + } + if (isInteractiveHelpCommand(task)) { + console.log('Type any task to run it. Use "exit" to quit.'); + continue; + } + history.push(task); + await saveCliHistory(history, historyPath); + console.log(`Starting task: ${task}`); + try { + await runAgentTask({ + task, + llm, + browserProfile, + browserSession, + sessionAttachmentMode: 'strict', + }); + } + catch (error) { + console.error('Error running agent:', error); + } + } + } + finally { + rl.close(); + await saveCliHistory(history, historyPath); + try { + if (browserSession._owns_browser_resources) { + await browserSession.kill(); + } + else { + await browserSession.stop(); + } + } + catch (error) { + console.error(`Warning: failed to close interactive browser session: ${error.message}`); + } + } +}; +export const getCliUsage = () => `Usage: + browser-use # interactive mode (TTY) + browser-use doctor + browser-use install + browser-use setup [--mode ] + browser-use tunnel + browser-use task + browser-use session + browser-use profile + browser-use run --remote + browser-use + browser-use -p "" + browser-use [options] + browser-use --mcp + +Options: + -h, --help Show this help message + --version Print version and exit + --mcp Run as MCP server + --json Output command results as JSON when supported + -y, --yes Skip optional setup prompts where supported + --provider Force provider (openai|anthropic|google|deepseek|groq|openrouter|azure|mistral|cerebras|vercel|oci|ollama|browser-use|aws|aws-anthropic) + --model Set model (e.g., gpt-5-mini, claude-4-sonnet, gemini-2.5-pro) + -p, --prompt Run a single task + --mode Setup mode for setup command (local|remote|full) + --api-key Browser Use API key for setup or cloud operations + --headless Run browser in headless mode + --allowed-domains Comma-separated allowlist (e.g., example.com,*.example.org) + --window-width Browser window width + --window-height Browser window height + --user-data-dir Chrome user data directory + --profile-directory Chrome profile directory (Default, Profile 1, ...) + --proxy-url Proxy server URL (e.g., http://proxy.example.com:8080) + --no-proxy Comma-separated proxy bypass list + --proxy-username Proxy username + --proxy-password Proxy password + --cdp-url Connect to an existing Chromium instance via CDP + --debug Enable debug logging`; +const resolvePlaywrightCliPath = () => require.resolve('playwright/cli'); +export const runInstallCommand = (options = {}) => { + const playwrightCliPath = options.playwright_cli_path ?? resolvePlaywrightCliPath(); + const spawnImpl = options.spawn_impl ?? spawnSync; + const result = spawnImpl(process.execPath, [playwrightCliPath, 'install', 'chromium'], { + stdio: 'inherit', + }); + if (result.error) { + throw result.error; + } + if (result.status !== 0) { + throw new Error(`Playwright browser install failed with exit code ${result.status ?? 1}`); + } +}; +const writeLine = (stream, value) => { + stream.write(`${value}\n`); +}; +const parseTunnelPort = (value) => { + const port = Number.parseInt(String(value ?? ''), 10); + if (!Number.isFinite(port) || port <= 0) { + throw new Error(`Invalid port: ${value ?? ''}`); + } + return port; +}; +export const runTunnelCommand = async (argv, options = {}) => { + const manager = options.manager ?? get_tunnel_manager(); + const output = options.stdout ?? process.stdout; + const errorOutput = options.stderr ?? process.stderr; + const json_output = Boolean(options.json_output); + const render = (value) => { + if (json_output) { + writeLine(output, JSON.stringify(value, null, 2)); + } + }; + try { + const first = argv[0] ?? null; + if (!first) { + writeLine(errorOutput, 'Usage: browser-use tunnel | list | stop | stop --all'); + return 1; + } + if (first === 'list') { + const result = manager.list_tunnels(); + if (json_output) { + render(result); + } + else if (result.tunnels.length > 0) { + for (const tunnel of result.tunnels) { + writeLine(output, `${tunnel.port}: ${tunnel.url}`); + } + } + else { + writeLine(output, 'No active tunnels'); + } + return 0; + } + if (first === 'stop' || first === 'stop-all') { + const stopAll = first === 'stop-all' || argv.includes('--all'); + if (stopAll) { + const result = await manager.stop_all_tunnels(); + if (json_output) { + render(result); + } + else if (result.count > 0) { + writeLine(output, `Stopped ${result.count} tunnel(s): ${result.stopped.join(', ')}`); + } + else { + writeLine(output, 'No tunnels to stop'); + } + return 0; + } + const port = parseTunnelPort(argv[1]); + const result = await manager.stop_tunnel(port); + if ('error' in result) { + writeLine(errorOutput, result.error); + return 1; + } + if (json_output) { + render(result); + } + else { + writeLine(output, `Stopped tunnel on port ${result.stopped}`); + } + return 0; + } + const port = parseTunnelPort(first); + const result = await manager.start_tunnel(port); + if ('error' in result) { + writeLine(errorOutput, result.error); + return 1; + } + if (json_output) { + render(result); + } + else if (result.existing) { + writeLine(output, `Tunnel already running: http://localhost:${result.port} -> ${result.url}`); + } + else { + writeLine(output, `Tunnel started: http://localhost:${result.port} -> ${result.url}`); + } + return 0; + } + catch (error) { + writeLine(errorOutput, error.message); + return 1; + } +}; +const validateSetupMode = (mode) => { + const normalized = (mode ?? 'local').trim().toLowerCase(); + if (normalized === 'local' || + normalized === 'remote' || + normalized === 'full') { + return normalized; + } + throw new Error(`Invalid setup mode "${mode ?? ''}". Expected local, remote, or full.`); +}; +const renderSetupChecks = (mode, report) => { + const checks = { + browser_use_package: report.checks.package, + }; + if (mode === 'local' || mode === 'full') { + checks.browser = report.checks.browser; + } + if (mode === 'remote' || mode === 'full') { + checks.api_key = report.checks.api_key; + checks.cloudflared = report.checks.cloudflared; + } + return checks; +}; +const planSetupActions = (mode, checks, yes, api_key) => { + const actions = []; + if ((mode === 'local' || mode === 'full') && + checks.browser?.status !== 'ok') { + actions.push({ + type: 'install_browser', + description: 'Install browser (Chromium)', + required: true, + }); + } + if ((mode === 'remote' || mode === 'full') && + checks.api_key?.status !== 'ok') { + if (api_key?.trim()) { + actions.push({ + type: 'configure_api_key', + description: 'Configure API key', + required: true, + api_key: api_key.trim(), + }); + } + else if (!yes) { + actions.push({ + type: 'prompt_api_key', + description: 'Prompt for API key', + required: false, + }); + } + } + if ((mode === 'remote' || mode === 'full') && + checks.cloudflared?.status !== 'ok') { + actions.push({ + type: 'install_cloudflared', + description: 'Install cloudflared (for tunneling)', + required: true, + }); + } + return actions; +}; +const logSetupChecks = (stream, checks) => { + writeLine(stream, ''); + writeLine(stream, 'Running checks...'); + writeLine(stream, ''); + for (const [name, check] of Object.entries(checks)) { + const icon = check.status === 'ok' ? '✓' : check.status === 'missing' ? '⚠' : '✗'; + writeLine(stream, ` ${icon} ${name.replace(/_/g, ' ')}: ${check.message}`); + } + writeLine(stream, ''); +}; +const logSetupActions = (stream, actions) => { + if (actions.length === 0) { + writeLine(stream, 'No additional setup needed.'); + writeLine(stream, ''); + return; + } + writeLine(stream, ''); + writeLine(stream, 'Setup actions:'); + writeLine(stream, ''); + actions.forEach((action, index) => { + writeLine(stream, ` ${index + 1}. ${action.description} ${action.required ? '(required)' : '(optional)'}`); + }); + writeLine(stream, ''); +}; +const logSetupValidation = (stream, validation) => { + writeLine(stream, ''); + writeLine(stream, 'Validation:'); + writeLine(stream, ''); + for (const [name, result] of Object.entries(validation)) { + const normalized = String(result); + const ok = normalized === 'ok' || normalized === 'true'; + writeLine(stream, ` ${ok ? '✓' : '✗'} ${name.replace(/_/g, ' ')}: ${normalized}`); + } + writeLine(stream, ''); +}; +export const runSetupCommand = async (params, options = {}) => { + const mode = validateSetupMode(params.mode); + const yes = Boolean(params.yes); + const api_key = params.api_key?.trim() || null; + const runDoctor = options.run_doctor_checks ?? + ((doctorOptions) => runDoctorChecks(doctorOptions)); + const installCommand = options.install_command ?? runInstallCommand; + const saveApiKey = options.save_api_key ?? save_cloud_api_token; + const output = options.stdout ?? process.stdout; + const json_output = Boolean(options.json_output); + const initialReport = await runDoctor(); + const checks = renderSetupChecks(mode, initialReport); + const actions = planSetupActions(mode, checks, yes, api_key); + if (!json_output) { + logSetupChecks(output, checks); + logSetupActions(output, actions); + } + for (const action of actions) { + if (action.type === 'install_browser') { + if (!json_output) { + writeLine(output, 'Installing Chromium browser...'); + } + await installCommand(); + continue; + } + if (action.type === 'configure_api_key') { + if (!json_output) { + writeLine(output, 'Configuring API key...'); + } + saveApiKey(action.api_key); + continue; + } + if (action.type === 'prompt_api_key' && !json_output) { + writeLine(output, 'API key not configured'); + writeLine(output, ' Set via: export BROWSER_USE_API_KEY=your_key'); + writeLine(output, ' Or: browser-use setup --api-key '); + continue; + } + if (action.type === 'install_cloudflared' && !json_output) { + writeLine(output, 'cloudflared not installed'); + writeLine(output, ' Install via:'); + writeLine(output, ' macOS: brew install cloudflared'); + writeLine(output, ' Linux: curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o ~/.local/bin/cloudflared && chmod +x ~/.local/bin/cloudflared'); + writeLine(output, ' Windows: winget install Cloudflare.cloudflared'); + writeLine(output, ''); + } + } + const validationReport = await runDoctor(); + const validation = { + browser_use_import: 'ok', + }; + if (mode === 'local' || mode === 'full') { + validation.browser_available = + validationReport.checks.browser.status === 'ok' + ? 'ok' + : `failed: ${validationReport.checks.browser.message}`; + } + if (mode === 'remote' || mode === 'full') { + validation.api_key_available = + validationReport.checks.api_key.status === 'ok'; + validation.cloudflared_available = + validationReport.checks.cloudflared.status === 'ok'; + } + const result = { + status: 'success', + mode, + checks, + validation, + }; + if (json_output) { + writeLine(output, JSON.stringify(result, null, 2)); + } + else { + logSetupValidation(output, validation); + } + return 0; +}; +const formatDuration = (startedAt, finishedAt) => { + if (!startedAt) { + return ''; + } + const start = new Date(startedAt).getTime(); + const end = finishedAt ? new Date(finishedAt).getTime() : Date.now(); + if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) { + return ''; + } + const totalSeconds = Math.floor((end - start) / 1000); + if (totalSeconds < 60) { + return `${totalSeconds}s`; + } + if (totalSeconds < 3600) { + return `${Math.floor(totalSeconds / 60)}m ${totalSeconds % 60}s`; + } + return `${Math.floor(totalSeconds / 3600)}h ${Math.floor((totalSeconds % 3600) / 60)}m`; +}; +const printTaskStep = (stream, step, verbose) => { + const stepNumber = step.number ?? '?'; + const memory = String(step.memory ?? ''); + if (verbose) { + const url = String(step.url ?? ''); + const shortUrl = url.length > 60 ? `${url.slice(0, 57)}...` : url; + writeLine(stream, ` [${stepNumber}] ${shortUrl}`); + if (memory) { + const shortMemory = memory.length > 100 ? `${memory.slice(0, 97)}...` : memory; + writeLine(stream, ` Reasoning: ${shortMemory}`); + } + const actions = Array.isArray(step.actions) ? step.actions : []; + actions.slice(0, 2).forEach((action) => { + const text = String(action); + const shortAction = text.length > 70 ? `${text.slice(0, 67)}...` : text; + writeLine(stream, ` Action: ${shortAction}`); + }); + if (actions.length > 2) { + writeLine(stream, ` ... and ${actions.length - 2} more actions`); + } + return; + } + const shortMemory = memory + ? memory.length > 80 + ? `${memory.slice(0, 77)}...` + : memory + : '(no reasoning)'; + writeLine(stream, ` ${stepNumber}. ${shortMemory}`); +}; +const requireCommandTarget = (value, usage) => { + const target = value?.trim(); + if (!target || target.startsWith('-')) { + throw new Error(usage); + } + return target; +}; +const rejectUnexpectedPositionals = (positionals, usage) => { + if (positionals.length > 0) { + throw new Error(usage); + } +}; +const markUsedOption = (used_options, option) => { + if (!used_options.includes(option)) { + used_options.push(option); + } +}; +const rejectUnsupportedFlags = (used_options, allowed_options, usage) => { + const allowed = new Set(allowed_options); + const unsupported = used_options.find((option) => !allowed.has(option)); + if (unsupported) { + throw new Error(usage); + } +}; +const parseTaskCommandFlags = (argv) => { + const flags = { + json: false, + limit: 10, + status: null, + session: null, + compact: false, + verbose: false, + last: null, + reverse: false, + step: null, + positionals: [], + used_options: [], + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] ?? ''; + if (arg === '--json') { + flags.json = true; + markUsedOption(flags.used_options, '--json'); + continue; + } + if (arg === '--compact' || arg === '-c') { + flags.compact = true; + markUsedOption(flags.used_options, '--compact'); + continue; + } + if (arg === '--verbose' || arg === '-v') { + flags.verbose = true; + markUsedOption(flags.used_options, '--verbose'); + continue; + } + if (arg === '--reverse' || arg === '-r') { + flags.reverse = true; + markUsedOption(flags.used_options, '--reverse'); + continue; + } + if (arg === '--limit' || arg.startsWith('--limit=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.limit = parsePositiveInt('--limit', value); + markUsedOption(flags.used_options, '--limit'); + index = nextIndex; + continue; + } + if (arg === '--status' || arg.startsWith('--status=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.status = value; + markUsedOption(flags.used_options, '--status'); + index = nextIndex; + continue; + } + if (arg === '--session' || arg.startsWith('--session=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.session = value; + markUsedOption(flags.used_options, '--session'); + index = nextIndex; + continue; + } + if (arg === '--last' || arg === '-n' || arg.startsWith('--last=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.last = parsePositiveInt('--last', value); + markUsedOption(flags.used_options, '--last'); + index = nextIndex; + continue; + } + if (arg === '--step' || arg === '-s' || arg.startsWith('--step=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.step = parsePositiveInt('--step', value); + markUsedOption(flags.used_options, '--step'); + index = nextIndex; + continue; + } + if (arg.startsWith('-')) { + throw new Error(`Unknown option: ${arg}`); + } + flags.positionals.push(arg); + } + return flags; +}; +export const runTaskCommand = async (argv, options = {}) => { + const client = options.client ?? new CloudManagementClient(); + const output = options.stdout ?? process.stdout; + const errorOutput = options.stderr ?? process.stderr; + const subcommand = argv[0] ?? ''; + try { + if (subcommand === 'list') { + const flags = parseTaskCommandFlags(argv.slice(1)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use task list [options]'); + rejectUnsupportedFlags(flags.used_options, ['--json', '--limit', '--status', '--session'], 'Usage: browser-use task list [options]'); + const result = await client.list_tasks({ + pageSize: flags.limit, + filterBy: flags.status, + sessionId: flags.session, + }); + if (flags.json) { + writeLine(output, JSON.stringify(result.items, null, 2)); + return 0; + } + if (result.items.length === 0) { + writeLine(output, 'No tasks found'); + return 0; + } + writeLine(output, `Tasks (${result.items.length}):`); + for (const task of result.items) { + const emoji = { + created: '🕒', + started: '🔄', + running: '🔄', + finished: '✅', + stopped: '⏹️', + failed: '❌', + }[task.status] ?? '❓'; + const text = task.task.length > 50 ? `${task.task.slice(0, 47)}...` : task.task; + writeLine(output, ` ${emoji} ${task.id.slice(0, 8)}... [${task.status}] ${text}`); + } + return 0; + } + if (subcommand === 'status') { + const taskId = requireCommandTarget(argv[1], 'Usage: browser-use task status '); + const flags = parseTaskCommandFlags(argv.slice(2)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use task status '); + rejectUnsupportedFlags(flags.used_options, ['--json', '--compact', '--verbose', '--reverse', '--last', '--step'], 'Usage: browser-use task status '); + const task = await client.get_task(taskId); + if (flags.json) { + writeLine(output, JSON.stringify(task, null, 2)); + return 0; + } + const parts = [ + `${{ + created: '🕒', + started: '🔄', + running: '🔄', + finished: '✅', + stopped: '⏹️', + failed: '❌', + }[task.status] ?? '❓'} ${task.id.slice(0, 8)}... [${task.status}]`, + ]; + const duration = formatDuration(task.startedAt, task.finishedAt); + if (duration) { + parts.push(duration); + } + writeLine(output, parts.join(' ')); + let steps = [...(task.steps ?? [])]; + const showAllSteps = flags.compact || flags.verbose; + if (flags.step !== null) { + steps = steps.filter((step) => Number(step.number) === flags.step); + } + else if (!showAllSteps && steps.length > 1) { + writeLine(output, ` ... ${steps.length - 1} earlier steps`); + steps = [steps[steps.length - 1]]; + } + else if (flags.last !== null && flags.last < steps.length) { + writeLine(output, ` ... ${steps.length - flags.last} earlier steps`); + steps = steps.slice(-flags.last); + } + if (flags.reverse) { + steps.reverse(); + } + steps.forEach((step) => printTaskStep(output, step, flags.verbose)); + if (task.output) { + writeLine(output, ''); + writeLine(output, `Output: ${task.output}`); + } + return 0; + } + if (subcommand === 'stop') { + const taskId = requireCommandTarget(argv[1], 'Usage: browser-use task stop '); + const flags = parseTaskCommandFlags(argv.slice(2)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use task stop '); + rejectUnsupportedFlags(flags.used_options, ['--json'], 'Usage: browser-use task stop '); + await client.update_task(taskId, 'stop'); + if (flags.json) { + writeLine(output, JSON.stringify({ stopped: taskId }, null, 2)); + } + else { + writeLine(output, `Stopped task: ${taskId}`); + } + return 0; + } + if (subcommand === 'logs') { + const taskId = requireCommandTarget(argv[1], 'Usage: browser-use task logs '); + const flags = parseTaskCommandFlags(argv.slice(2)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use task logs '); + rejectUnsupportedFlags(flags.used_options, ['--json'], 'Usage: browser-use task logs '); + const result = await client.get_task_logs(taskId); + if (flags.json) { + writeLine(output, JSON.stringify(result, null, 2)); + } + else if (result.downloadUrl) { + writeLine(output, `Download logs: ${result.downloadUrl}`); + } + else { + writeLine(output, 'No logs available for this task'); + } + return 0; + } + writeLine(errorOutput, 'Usage: browser-use task [options]'); + return 1; + } + catch (error) { + writeLine(errorOutput, `Error: ${error.message}`); + return 1; + } +}; +const parseSessionCommandFlags = (argv) => { + const flags = { + json: false, + limit: 10, + status: null, + all: false, + delete: false, + profile: null, + proxy_country: null, + start_url: null, + screen_size: null, + positionals: [], + used_options: [], + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] ?? ''; + if (arg === '--json') { + flags.json = true; + markUsedOption(flags.used_options, '--json'); + continue; + } + if (arg === '--all') { + flags.all = true; + markUsedOption(flags.used_options, '--all'); + continue; + } + if (arg === '--delete') { + flags.delete = true; + markUsedOption(flags.used_options, '--delete'); + continue; + } + if (arg === '--limit' || arg.startsWith('--limit=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.limit = parsePositiveInt('--limit', value); + markUsedOption(flags.used_options, '--limit'); + index = nextIndex; + continue; + } + if (arg === '--status' || arg.startsWith('--status=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.status = value; + markUsedOption(flags.used_options, '--status'); + index = nextIndex; + continue; + } + if (arg === '--profile' || arg.startsWith('--profile=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.profile = value; + markUsedOption(flags.used_options, '--profile'); + index = nextIndex; + continue; + } + if (arg === '--proxy-country' || arg.startsWith('--proxy-country=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.proxy_country = value; + markUsedOption(flags.used_options, '--proxy-country'); + index = nextIndex; + continue; + } + if (arg === '--start-url' || arg.startsWith('--start-url=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.start_url = value; + markUsedOption(flags.used_options, '--start-url'); + index = nextIndex; + continue; + } + if (arg === '--screen-size' || arg.startsWith('--screen-size=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.screen_size = value; + markUsedOption(flags.used_options, '--screen-size'); + index = nextIndex; + continue; + } + if (arg.startsWith('-')) { + throw new Error(`Unknown option: ${arg}`); + } + flags.positionals.push(arg); + } + return flags; +}; +export const runSessionCommand = async (argv, options = {}) => { + const client = options.client ?? new CloudManagementClient(); + const output = options.stdout ?? process.stdout; + const errorOutput = options.stderr ?? process.stderr; + const subcommand = argv[0] ?? ''; + try { + if (subcommand === 'list') { + const flags = parseSessionCommandFlags(argv.slice(1)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use session list [options]'); + rejectUnsupportedFlags(flags.used_options, ['--json', '--limit', '--status'], 'Usage: browser-use session list [options]'); + const result = await client.list_sessions({ + pageSize: flags.limit, + filterBy: flags.status, + }); + if (flags.json) { + writeLine(output, JSON.stringify(result.items, null, 2)); + return 0; + } + if (result.items.length === 0) { + writeLine(output, 'No sessions found'); + return 0; + } + writeLine(output, `Sessions (${result.items.length}):`); + for (const session of result.items) { + const emoji = session.status === 'active' ? '🟢' : '⏹️'; + const duration = formatDuration(session.startedAt, session.finishedAt); + const details = duration ? ` ${duration}` : ''; + writeLine(output, ` ${emoji} ${session.id.slice(0, 8)}... [${session.status}]${details}`); + } + return 0; + } + if (subcommand === 'get') { + const sessionId = requireCommandTarget(argv[1], 'Usage: browser-use session get '); + const flags = parseSessionCommandFlags(argv.slice(2)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use session get '); + rejectUnsupportedFlags(flags.used_options, ['--json'], 'Usage: browser-use session get '); + const session = await client.get_session(sessionId); + if (flags.json) { + writeLine(output, JSON.stringify(session, null, 2)); + } + else { + writeLine(output, `${session.id} [${session.status}]`); + if (session.liveUrl) { + writeLine(output, `Live URL: ${session.liveUrl}`); + } + const duration = formatDuration(session.startedAt, session.finishedAt); + if (duration) { + writeLine(output, `Duration: ${duration}`); + } + } + return 0; + } + if (subcommand === 'stop') { + const flags = parseSessionCommandFlags(argv.slice(1)); + rejectUnsupportedFlags(flags.used_options, ['--json', '--all'], 'Usage: browser-use session stop | --all'); + if (flags.all) { + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use session stop | --all'); + const sessions = await client.list_sessions({ + pageSize: 100, + filterBy: 'active', + }); + const stopped = []; + for (const session of sessions.items) { + await client.update_session(session.id, 'stop'); + stopped.push(session.id); + } + if (flags.json) { + writeLine(output, JSON.stringify({ stopped }, null, 2)); + } + else if (stopped.length > 0) { + writeLine(output, `Stopped ${stopped.length} session(s): ${stopped.join(', ')}`); + } + else { + writeLine(output, 'No sessions to stop'); + } + return 0; + } + const [sessionIdCandidate, ...unexpectedPositionals] = flags.positionals; + rejectUnexpectedPositionals(unexpectedPositionals, 'Usage: browser-use session stop | --all'); + const sessionId = requireCommandTarget(sessionIdCandidate ?? argv[1], 'Usage: browser-use session stop | --all'); + await client.update_session(sessionId, 'stop'); + if (flags.json) { + writeLine(output, JSON.stringify({ stopped: sessionId }, null, 2)); + } + else { + writeLine(output, `Stopped session: ${sessionId}`); + } + return 0; + } + if (subcommand === 'create') { + const flags = parseSessionCommandFlags(argv.slice(1)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use session create [options]'); + rejectUnsupportedFlags(flags.used_options, [ + '--json', + '--profile', + '--proxy-country', + '--start-url', + '--screen-size', + ], 'Usage: browser-use session create [options]'); + let browserScreenWidth = null; + let browserScreenHeight = null; + if (flags.screen_size) { + const match = flags.screen_size.match(/^(\d+)x(\d+)$/i); + if (!match) { + throw new Error('Expected --screen-size WIDTHxHEIGHT'); + } + browserScreenWidth = Number.parseInt(match[1], 10); + browserScreenHeight = Number.parseInt(match[2], 10); + } + const session = await client.create_session({ + profileId: flags.profile, + proxyCountryCode: flags.proxy_country, + startUrl: flags.start_url, + browserScreenWidth, + browserScreenHeight, + }); + if (flags.json) { + writeLine(output, JSON.stringify(session, null, 2)); + } + else { + writeLine(output, `Created session: ${session.id}`); + if (session.liveUrl) { + writeLine(output, `Live URL: ${session.liveUrl}`); + } + } + return 0; + } + if (subcommand === 'share') { + const sessionId = requireCommandTarget(argv[1], 'Usage: browser-use session share [--delete]'); + const flags = parseSessionCommandFlags(argv.slice(2)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use session share [--delete]'); + rejectUnsupportedFlags(flags.used_options, ['--json', '--delete'], 'Usage: browser-use session share [--delete]'); + if (flags.delete) { + await client.delete_session_public_share(sessionId); + if (flags.json) { + writeLine(output, JSON.stringify({ deleted: sessionId }, null, 2)); + } + else { + writeLine(output, `Deleted public share for session: ${sessionId}`); + } + } + else { + const share = await client.create_session_public_share(sessionId); + if (flags.json) { + writeLine(output, JSON.stringify(share, null, 2)); + } + else { + writeLine(output, `Public share URL: ${share.shareUrl}`); + } + } + return 0; + } + writeLine(errorOutput, 'Usage: browser-use session [options]'); + return 1; + } + catch (error) { + writeLine(errorOutput, `Error: ${error.message}`); + return 1; + } +}; +const parseProfileCommandFlags = (argv) => { + const flags = { + json: false, + remote: false, + limit: 20, + name: null, + domain: null, + from_profile: null, + positionals: [], + used_options: [], + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] ?? ''; + if (arg === '--json') { + flags.json = true; + markUsedOption(flags.used_options, '--json'); + continue; + } + if (arg === '--remote') { + flags.remote = true; + markUsedOption(flags.used_options, '--remote'); + continue; + } + if (arg === '--limit' || arg.startsWith('--limit=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.limit = parsePositiveInt('--limit', value); + markUsedOption(flags.used_options, '--limit'); + index = nextIndex; + continue; + } + if (arg === '--name' || arg.startsWith('--name=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.name = value; + markUsedOption(flags.used_options, '--name'); + index = nextIndex; + continue; + } + if (arg === '--domain' || arg.startsWith('--domain=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.domain = value; + markUsedOption(flags.used_options, '--domain'); + index = nextIndex; + continue; + } + if (arg === '--from' || arg.startsWith('--from=')) { + const { value, nextIndex } = takeOptionValue(arg, index, argv); + flags.from_profile = value; + markUsedOption(flags.used_options, '--from'); + index = nextIndex; + continue; + } + if (arg.startsWith('-')) { + throw new Error(`Unknown option: ${arg}`); + } + flags.positionals.push(arg); + } + return flags; +}; +const normalizeProfileCookieDomain = (value) => String(value ?? '') + .trim() + .replace(/^\./, '') + .toLowerCase(); +const cookieMatchesDomainFilter = (cookie, domainFilter) => { + if (!domainFilter) { + return true; + } + const cookieDomain = normalizeProfileCookieDomain(cookie.domain); + const normalizedFilter = normalizeProfileCookieDomain(domainFilter); + return Boolean(cookieDomain && + normalizedFilter && + (cookieDomain === normalizedFilter || + cookieDomain.endsWith(`.${normalizedFilter}`) || + normalizedFilter.endsWith(`.${cookieDomain}`))); +}; +export const runProfileCommand = async (argv, options = {}) => { + const client = options.client ?? new CloudManagementClient(); + const profileLister = options.profile_lister ?? (() => systemChrome.listProfiles()); + const localSessionFactory = options.local_session_factory ?? + ((profile_directory) => BrowserSession.from_system_chrome({ + profile_directory, + profile: { headless: true }, + })); + const remoteSessionFactory = options.remote_session_factory ?? + ((init) => new BrowserSession({ + cdp_url: init.cdp_url, + })); + const cloudBrowserClientFactory = options.cloud_browser_client_factory ?? (() => new CloudBrowserClient()); + const output = options.stdout ?? process.stdout; + const errorOutput = options.stderr ?? process.stderr; + const subcommand = argv[0] ?? ''; + try { + if (subcommand === 'list') { + const flags = parseProfileCommandFlags(argv.slice(1)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use profile list [--remote] [options]'); + rejectUnsupportedFlags(flags.used_options, ['--json', '--remote', '--limit'], 'Usage: browser-use profile list [--remote] [options]'); + if (flags.remote) { + const result = await client.list_profiles({ + pageSize: flags.limit, + }); + if (flags.json) { + writeLine(output, JSON.stringify(result.items, null, 2)); + } + else if (result.items.length === 0) { + writeLine(output, 'No cloud profiles found'); + } + else { + writeLine(output, `Cloud profiles (${result.items.length}):`); + result.items.forEach((profile) => { + writeLine(output, ` ${profile.id}: ${profile.name || 'Unnamed'}`); + }); + } + return 0; + } + const profiles = profileLister(); + if (flags.json) { + writeLine(output, JSON.stringify({ profiles }, null, 2)); + } + else if (profiles.length === 0) { + writeLine(output, 'No Chrome profiles found'); + } + else { + writeLine(output, 'Local Chrome profiles:'); + profiles.forEach((profile) => { + const emailSuffix = profile.email ? ` (${profile.email})` : ''; + writeLine(output, ` ${profile.directory}: ${profile.name}${emailSuffix}`); + }); + } + return 0; + } + if (subcommand === 'get') { + const profileId = requireCommandTarget(argv[1], 'Usage: browser-use profile get [--remote]'); + const flags = parseProfileCommandFlags(argv.slice(2)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use profile get [--remote]'); + rejectUnsupportedFlags(flags.used_options, ['--json', '--remote'], 'Usage: browser-use profile get [--remote]'); + if (flags.remote) { + const profile = await client.get_profile(profileId); + if (flags.json) { + writeLine(output, JSON.stringify(profile, null, 2)); + } + else { + writeLine(output, `Profile: ${profile.id}`); + if (profile.name) { + writeLine(output, ` Name: ${profile.name}`); + } + writeLine(output, ` Updated: ${profile.updatedAt}`); + } + return 0; + } + const profile = profileLister().find((entry) => entry.directory === profileId || entry.name === profileId); + if (!profile) { + throw new Error(`Profile "${profileId}" not found`); + } + if (flags.json) { + writeLine(output, JSON.stringify(profile, null, 2)); + } + else { + writeLine(output, `Profile: ${profile.directory}`); + writeLine(output, ` Name: ${profile.name}`); + if (profile.email) { + writeLine(output, ` Email: ${profile.email}`); + } + } + return 0; + } + if (subcommand === 'create') { + const flags = parseProfileCommandFlags(argv.slice(1)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use profile create --remote [--name ] [--json]'); + rejectUnsupportedFlags(flags.used_options, ['--json', '--remote', '--name'], 'Usage: browser-use profile create --remote [--name ] [--json]'); + if (!flags.remote) { + throw new Error('Profile create is only supported with --remote'); + } + const profile = await client.create_profile({ name: flags.name }); + if (flags.json) { + writeLine(output, JSON.stringify(profile, null, 2)); + } + else { + writeLine(output, `Created cloud profile: ${profile.id}`); + } + return 0; + } + if (subcommand === 'delete') { + const profileId = requireCommandTarget(argv[1], 'Usage: browser-use profile delete --remote'); + const flags = parseProfileCommandFlags(argv.slice(2)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use profile delete --remote'); + rejectUnsupportedFlags(flags.used_options, ['--json', '--remote'], 'Usage: browser-use profile delete --remote'); + if (!flags.remote) { + throw new Error('Profile delete is only supported with --remote'); + } + await client.delete_profile(profileId); + if (flags.json) { + writeLine(output, JSON.stringify({ deleted: profileId }, null, 2)); + } + else { + writeLine(output, `Deleted cloud profile: ${profileId}`); + } + return 0; + } + if (subcommand === 'cookies') { + const profileId = requireCommandTarget(argv[1], 'Usage: browser-use profile cookies '); + const flags = parseProfileCommandFlags(argv.slice(2)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use profile cookies '); + rejectUnsupportedFlags(flags.used_options, ['--json'], 'Usage: browser-use profile cookies '); + if (flags.remote) { + throw new Error('Profile cookies is only supported for local Chrome profiles'); + } + const profile = profileLister().find((entry) => entry.directory === profileId || entry.name === profileId); + if (!profile) { + throw new Error(`Profile "${profileId}" not found`); + } + const session = localSessionFactory(profile.directory); + await session.start(); + try { + const cookies = (await session.get_cookies?.()) ?? []; + const domains = new Map(); + for (const cookie of cookies) { + const domain = normalizeProfileCookieDomain(cookie.domain) || 'unknown'; + domains.set(domain, (domains.get(domain) ?? 0) + 1); + } + const sortedDomains = Array.from(domains.entries()).sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0])); + if (flags.json) { + writeLine(output, JSON.stringify({ + domains: Object.fromEntries(sortedDomains), + total_cookies: cookies.length, + }, null, 2)); + } + else { + const emailSuffix = profile.email ? ` (${profile.email})` : ''; + writeLine(output, `Loading cookies from: ${profile.name}${emailSuffix}`); + writeLine(output, ''); + writeLine(output, `Cookies by domain (${cookies.length} total):`); + sortedDomains.slice(0, 20).forEach(([domain, count]) => { + writeLine(output, ` ${domain}: ${count}`); + }); + if (sortedDomains.length > 20) { + writeLine(output, ` ... and ${sortedDomains.length - 20} more domains`); + } + writeLine(output, ''); + writeLine(output, 'To sync cookies to cloud:'); + writeLine(output, ` browser-use profile sync --from "${profile.directory}" --domain `); + } + } + finally { + await session.stop?.(); + } + return 0; + } + if (subcommand === 'sync') { + const flags = parseProfileCommandFlags(argv.slice(1)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use profile sync --from [--name ] [--domain ] [--json]'); + rejectUnsupportedFlags(flags.used_options, ['--json', '--from', '--name', '--domain'], 'Usage: browser-use profile sync --from [--name ] [--domain ] [--json]'); + const profiles = profileLister(); + const fromProfile = flags.from_profile?.trim(); + if (!fromProfile) { + writeLine(errorOutput, 'Usage: browser-use profile sync --from [--name ] [--domain ] [--json]'); + if (profiles.length > 0) { + writeLine(errorOutput, 'Available local profiles:'); + profiles.forEach((profile) => { + const emailSuffix = profile.email ? ` (${profile.email})` : ''; + writeLine(errorOutput, ` ${profile.directory}: ${profile.name}${emailSuffix}`); + }); + } + return 1; + } + const profile = profiles.find((entry) => entry.directory === fromProfile || entry.name === fromProfile); + if (!profile) { + throw new Error(`Profile "${fromProfile}" not found`); + } + const progress = flags.json ? errorOutput : output; + const logProgress = (message) => writeLine(progress, message); + const cloudName = flags.name ?? + (flags.domain + ? `Chrome - ${profile.name} (${flags.domain})` + : `Chrome - ${profile.name}`); + let cloudProfileId = null; + let cloudBrowserSessionId = null; + const cloudBrowserClient = cloudBrowserClientFactory(); + try { + logProgress(`Syncing: ${profile.name}${profile.email ? ` (${profile.email})` : ''}`); + logProgress(' Creating cloud profile...'); + const cloudProfile = await client.create_profile({ name: cloudName }); + cloudProfileId = cloudProfile.id; + logProgress(` Created: ${cloudProfileId}`); + logProgress(' Exporting cookies from local profile...'); + const localSession = localSessionFactory(profile.directory); + let filteredCookies = []; + await localSession.start(); + try { + const cookies = ((await localSession.get_cookies?.()) ?? + []); + filteredCookies = cookies.filter((cookie) => cookieMatchesDomainFilter(cookie, flags.domain)); + } + finally { + await localSession.stop?.(); + } + if (filteredCookies.length === 0) { + throw new Error(flags.domain + ? `No cookies found for domain: ${flags.domain}` + : 'No cookies found in local profile'); + } + logProgress(` Found ${filteredCookies.length} cookies`); + logProgress(' Importing cookies to cloud profile...'); + const cloudBrowser = await cloudBrowserClient.create_browser({ + profile_id: cloudProfileId, + }); + cloudBrowserSessionId = cloudBrowser.id; + if (!cloudBrowser.cdpUrl) { + throw new Error('Cloud browser did not return a CDP URL'); + } + const remoteSession = remoteSessionFactory({ + cdp_url: cloudBrowser.cdpUrl, + }); + await remoteSession.start(); + try { + if (!remoteSession.browser_context?.addCookies) { + throw new Error('Remote browser context does not support addCookies'); + } + await remoteSession.browser_context.addCookies(filteredCookies); + } + finally { + await remoteSession.stop?.(); + } + await cloudBrowserClient.stop_browser(cloudBrowserSessionId); + cloudBrowserSessionId = null; + if (flags.json) { + writeLine(output, JSON.stringify({ + success: true, + profile_id: cloudProfileId, + cookies_synced: filteredCookies.length, + }, null, 2)); + } + else { + logProgress('Profile synced successfully!'); + logProgress(` Cloud profile ID: ${cloudProfileId}`); + } + return 0; + } + catch (error) { + if (cloudBrowserSessionId) { + try { + await cloudBrowserClient.stop_browser(cloudBrowserSessionId); + } + catch { + // Ignore cleanup failures. + } + } + if (cloudProfileId) { + try { + await client.delete_profile(cloudProfileId); + } + catch { + // Ignore cleanup failures. + } + } + throw error; + } + } + if (subcommand === 'update') { + const profileId = requireCommandTarget(argv[1], 'Usage: browser-use profile update --remote --name '); + const flags = parseProfileCommandFlags(argv.slice(2)); + rejectUnexpectedPositionals(flags.positionals, 'Usage: browser-use profile update --remote --name '); + rejectUnsupportedFlags(flags.used_options, ['--json', '--remote', '--name'], 'Usage: browser-use profile update --remote --name '); + if (!flags.remote) { + throw new Error('Profile update is only supported with --remote'); + } + if (!flags.name) { + throw new Error('Usage: browser-use profile update --remote --name '); + } + const profile = await client.update_profile(profileId, { + name: flags.name, + }); + if (flags.json) { + writeLine(output, JSON.stringify(profile, null, 2)); + } + else { + writeLine(output, `Updated cloud profile: ${profile.id}`); + } + return 0; + } + writeLine(errorOutput, 'Usage: browser-use profile [--remote] [options]'); + return 1; + } + catch (error) { + writeLine(errorOutput, `Error: ${error.message}`); + return 1; + } +}; +const CLOUD_RUN_FLAGS = new Set([ + '--remote', + '--llm', + '--session-id', + '--proxy-country', + '--wait', + '--stream', + '--flash', + '--thinking', + '--vision', + '--no-vision', + '--start-url', + '--metadata', + '--secret', + '--allowed-domain', + '--skill-id', + '--structured-output', + '--judge', + '--judge-ground-truth', + '--max-steps', + '--profile', +]); +const CLOUD_RUN_VALUE_FLAGS = new Set([ + '--llm', + '--session-id', + '--proxy-country', + '--start-url', + '--structured-output', + '--judge-ground-truth', + '--max-steps', + '--profile', + '--metadata', + '--secret', + '--allowed-domain', + '--skill-id', +]); +export const hasCloudRunFlags = (argv) => { + for (const arg of argv) { + if (arg === '--') { + break; + } + if (CLOUD_RUN_FLAGS.has(arg)) { + return true; + } + const separator = arg.indexOf('='); + if (separator > 0 && CLOUD_RUN_VALUE_FLAGS.has(arg.slice(0, separator))) { + return true; + } + } + return false; +}; +const hasExplicitRemoteRunFlag = (argv) => { + for (const arg of argv) { + if (arg === '--') { + break; + } + if (arg === '--remote') { + return true; + } + } + return false; +}; +const PREFIXED_SUBCOMMAND_VALUE_FLAGS = [ + '--api-key', + '--provider', + '--model', + '--window-width', + '--window-height', + '--user-data-dir', + '--profile-directory', + '--allowed-domains', + '--proxy-url', + '--no-proxy', + '--proxy-username', + '--proxy-password', + '--cdp-url', +]; +const matchesOptionWithValue = (arg, option) => arg === option || arg.startsWith(`${option}=`); +export const extractPrefixedSubcommand = (argv) => { + const forwardedArgs = []; + let debug = false; + let index = 0; + while (index < argv.length) { + const arg = argv[index] ?? ''; + if (arg === '--debug') { + debug = true; + index += 1; + continue; + } + if (arg === '--json') { + forwardedArgs.push(arg); + index += 1; + continue; + } + if (arg === '--headless') { + index += 1; + continue; + } + const valueOption = PREFIXED_SUBCOMMAND_VALUE_FLAGS.find((option) => matchesOptionWithValue(arg, option)); + if (valueOption) { + if (arg === valueOption) { + if (index + 1 >= argv.length) { + break; + } + index += 2; + } + else { + index += 1; + } + continue; + } + break; + } + const command = argv[index]; + if (command !== 'run' && + command !== 'task' && + command !== 'session' && + command !== 'profile') { + return null; + } + return { + command, + argv: argv.slice(index + 1), + debug, + forwardedArgs, + }; +}; +const parseKeyValuePairs = (values, option) => { + const result = {}; + values.forEach((value) => { + const separator = value.indexOf('='); + if (separator <= 0) { + throw new Error(`Invalid value for ${option}: expected KEY=VALUE`); + } + const key = value.slice(0, separator).trim(); + if (!key) { + throw new Error(`Invalid value for ${option}: expected KEY=VALUE`); + } + result[key] = value.slice(separator + 1); + }); + return Object.keys(result).length > 0 ? result : null; +}; +const parseCloudRunArgs = (argv) => { + const flags = { + remote: false, + wait: false, + stream: false, + flash: false, + thinking: false, + vision: null, + llm: null, + session_id: null, + proxy_country: null, + start_url: null, + structured_output: null, + judge: false, + judge_ground_truth: null, + max_steps: null, + profile: null, + metadata: [], + secret: [], + allowed_domain: [], + skill_id: [], + task_parts: [], + }; + for (let index = 0; index < argv.length; index += 1) { + const rawArg = argv[index] ?? ''; + if (rawArg === '--') { + flags.task_parts.push(...argv.slice(index + 1)); + break; + } + const separator = rawArg.indexOf('='); + const hasInlineValue = separator > 0; + const arg = hasInlineValue ? rawArg.slice(0, separator) : rawArg; + const inlineValue = hasInlineValue + ? rawArg.slice(separator + 1).trim() + : ''; + if (arg === '--remote') { + flags.remote = true; + continue; + } + if (arg === '--wait') { + flags.wait = true; + continue; + } + if (arg === '--stream') { + flags.stream = true; + continue; + } + if (arg === '--flash') { + flags.flash = true; + continue; + } + if (arg === '--thinking') { + flags.thinking = true; + continue; + } + if (arg === '--vision') { + flags.vision = true; + continue; + } + if (arg === '--no-vision') { + flags.vision = false; + continue; + } + if (arg === '--judge') { + flags.judge = true; + continue; + } + if (arg === '--llm' || + arg === '--session-id' || + arg === '--proxy-country' || + arg === '--start-url' || + arg === '--structured-output' || + arg === '--judge-ground-truth' || + arg === '--max-steps' || + arg === '--profile' || + arg === '--metadata' || + arg === '--secret' || + arg === '--allowed-domain' || + arg === '--skill-id') { + const { value: next, nextIndex } = takeOptionValue(rawArg, index, argv); + if (arg === '--llm') { + flags.llm = next; + } + else if (arg === '--session-id') { + flags.session_id = next; + } + else if (arg === '--proxy-country') { + flags.proxy_country = next; + } + else if (arg === '--start-url') { + flags.start_url = next; + } + else if (arg === '--structured-output') { + flags.structured_output = next; + } + else if (arg === '--judge-ground-truth') { + flags.judge_ground_truth = next; + } + else if (arg === '--max-steps') { + flags.max_steps = parsePositiveInt('--max-steps', next); + } + else if (arg === '--profile') { + flags.profile = next; + } + else if (arg === '--metadata') { + flags.metadata.push(next); + } + else if (arg === '--secret') { + flags.secret.push(next); + } + else if (arg === '--allowed-domain') { + flags.allowed_domain.push(next); + } + else { + flags.skill_id.push(next); + } + index = nextIndex; + continue; + } + if (rawArg.startsWith('-')) { + throw new Error(`Unknown option: ${rawArg}`); + } + flags.task_parts.push(rawArg); + } + return flags; +}; +export const runCloudTaskCommand = async (argv, options = {}) => { + const client = options.client ?? new CloudManagementClient(); + const output = options.stdout ?? process.stdout; + const errorOutput = options.stderr ?? process.stderr; + const sleepImpl = options.sleep_impl ?? + ((ms) => new Promise((r) => setTimeout(r, ms))); + let autoCreatedSessionId = null; + let taskCreated = false; + try { + const flags = parseCloudRunArgs(argv); + if (!flags.remote) { + throw new Error('Usage: browser-use run --remote '); + } + const task = flags.task_parts.join(' ').trim(); + if (!task) { + throw new Error('Usage: browser-use run --remote '); + } + let sessionId = flags.session_id; + if (!sessionId && (flags.profile || flags.proxy_country)) { + const session = await client.create_session({ + profileId: flags.profile, + proxyCountryCode: flags.proxy_country, + startUrl: flags.start_url, + }); + sessionId = session.id; + autoCreatedSessionId = session.id; + writeLine(output, `Created cloud session: ${sessionId}`); + } + const created = await client.create_task({ + task, + llm: flags.llm, + sessionId, + startUrl: flags.start_url, + maxSteps: flags.max_steps, + structuredOutput: flags.structured_output, + metadata: parseKeyValuePairs(flags.metadata, '--metadata'), + secrets: parseKeyValuePairs(flags.secret, '--secret'), + allowedDomains: flags.allowed_domain.length > 0 ? flags.allowed_domain : null, + flashMode: flags.flash, + thinking: flags.thinking, + vision: flags.vision, + judge: flags.judge, + judgeGroundTruth: flags.judge_ground_truth, + skillIds: flags.skill_id.length > 0 ? flags.skill_id : null, + }); + taskCreated = true; + if (!flags.wait) { + writeLine(output, `Task started: ${created.id} (session: ${created.sessionId})`); + writeLine(output, `Use "browser-use task status ${created.id}" to check progress.`); + return 0; + } + let lastStatus = null; + while (true) { + const taskView = await client.get_task(created.id); + if (flags.stream && taskView.status !== lastStatus) { + writeLine(output, `Status: ${taskView.status}`); + lastStatus = taskView.status; + } + if (taskView.status === 'finished' || + taskView.status === 'stopped' || + taskView.status === 'failed') { + writeLine(output, `Task ${taskView.status}: ${taskView.id}`); + if (taskView.output) { + writeLine(output, taskView.output); + } + return taskView.status === 'finished' ? 0 : 1; + } + await sleepImpl(1000); + } + } + catch (error) { + if (autoCreatedSessionId && !taskCreated) { + try { + await client.update_session(autoCreatedSessionId, 'stop'); + } + catch { + // Ignore cleanup failures. + } + } + writeLine(errorOutput, `Error: ${error.message}`); + return 1; + } +}; +const enableDebugLogging = () => { + process.env.BROWSER_USE_LOGGING_LEVEL = 'debug'; + setupLogging({ logLevel: 'debug', forceSetup: true }); +}; +const summarizeDoctorChecks = (checks) => { + const values = Object.values(checks); + const total = values.length; + const ok = values.filter((check) => check.status === 'ok').length; + const warning = values.filter((check) => check.status === 'warning').length; + const missing = values.filter((check) => check.status === 'missing').length; + const error = values.filter((check) => check.status === 'error').length; + const parts = [`${ok}/${total} checks passed`]; + if (warning > 0) { + parts.push(`${warning} warnings`); + } + if (missing > 0) { + parts.push(`${missing} missing`); + } + if (error > 0) { + parts.push(`${error} errors`); + } + return parts.join(', '); +}; +const findSystemBinary = (binary) => { + const command = process.platform === 'win32' ? 'where' : 'which'; + const result = spawnSync(command, [binary], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + if (result.status !== 0) { + return null; + } + const firstLine = result.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .find(Boolean); + return firstLine ?? null; +}; +export const runDoctorChecks = async (options = {}) => { + const fetchImpl = options.fetch_impl ?? fetch; + const configuredApiKey = options.api_key !== undefined + ? options.api_key?.trim() || null + : process.env.BROWSER_USE_API_KEY?.trim() || + new DeviceAuthClient().api_token?.trim() || + null; + const apiKeySource = options.api_key !== undefined + ? 'argument' + : process.env.BROWSER_USE_API_KEY?.trim() + ? 'environment' + : new DeviceAuthClient().api_token?.trim() + ? 'cloud_auth' + : null; + const tunnelStatus = options.cloudflared_path !== undefined + ? options.cloudflared_path + ? { + available: true, + source: 'system', + path: options.cloudflared_path, + note: 'cloudflared installed', + } + : { + available: false, + source: null, + path: null, + note: 'cloudflared not installed - install it manually before using tunnel', + } + : get_tunnel_manager().get_status(); + const checks = { + package: { + status: 'ok', + message: `browser-use ${options.version ?? get_browser_use_version()}`, + }, + browser: (() => { + const executable = options.browser_executable !== undefined + ? options.browser_executable + : systemChrome.findExecutable(); + if (executable) { + return { + status: 'ok', + message: `Chrome detected at ${executable}`, + }; + } + return { + status: 'warning', + message: 'Chrome executable not detected', + note: 'Local browser launch may fail until Chrome or Chromium is installed.', + }; + })(), + api_key: (() => { + if (configuredApiKey) { + return { + status: 'ok', + message: apiKeySource === 'cloud_auth' + ? 'Browser Use API key is configured in cloud auth' + : 'BROWSER_USE_API_KEY is configured', + }; + } + return { + status: 'missing', + message: 'BROWSER_USE_API_KEY is not configured', + note: 'Required for browser-use cloud browser features.', + }; + })(), + cloudflared: (() => { + if (tunnelStatus.available && tunnelStatus.path) { + return { + status: 'ok', + message: `cloudflared detected at ${tunnelStatus.path}`, + }; + } + return { + status: 'missing', + message: 'cloudflared not found', + note: tunnelStatus.note || + 'Tunnel features remain unavailable until cloudflared is installed.', + }; + })(), + network: { + status: 'warning', + message: 'Network connectivity check inconclusive', + note: 'Some remote features may not work offline.', + }, + }; + try { + const response = await fetchImpl('https://api.github.com', { + method: 'HEAD', + }); + if (response.ok || response.status < 500) { + checks.network = { + status: 'ok', + message: 'Network connectivity OK', + }; + } + } + catch { + checks.network = { + status: 'warning', + message: 'Network connectivity check inconclusive', + note: 'Some remote features may not work offline.', + }; + } + const allOk = Object.values(checks).every((check) => check.status === 'ok'); + return { + status: allOk ? 'healthy' : 'issues_found', + checks, + summary: summarizeDoctorChecks(checks), + }; +}; +const printDoctorReport = (report) => { + console.log(`status: ${report.status}`); + console.log(`summary: ${report.summary}`); + for (const [name, check] of Object.entries(report.checks)) { + console.log(`${name}: ${check.status} - ${check.message}`); + if (check.note) { + console.log(` note: ${check.note}`); + } + if (check.fix) { + console.log(` fix: ${check.fix}`); + } + } +}; +async function runMcpServer() { + const server = new MCPServer('browser-use', get_browser_use_version()); + await server.start(); + const shutdown = async () => { + await server.stop(); + process.exit(0); + }; + process.once('SIGINT', () => void shutdown()); + process.once('SIGTERM', () => void shutdown()); + await new Promise(() => { }); +} +export async function main(argv = process.argv.slice(2)) { + const prefixedSubcommand = extractPrefixedSubcommand(argv); + if (prefixedSubcommand) { + if (prefixedSubcommand.debug) { + enableDebugLogging(); + } + if (prefixedSubcommand.command === 'run') { + if (!hasCloudRunFlags(prefixedSubcommand.argv) || + !hasExplicitRemoteRunFlag(prefixedSubcommand.argv)) { + await main([ + ...prefixedSubcommand.forwardedArgs, + ...prefixedSubcommand.argv, + ]); + return; + } + const exitCode = await runCloudTaskCommand(prefixedSubcommand.argv); + if (exitCode !== 0) { + process.exit(exitCode); + } + return; + } + const subcommandArgv = [ + ...prefixedSubcommand.argv, + ...prefixedSubcommand.forwardedArgs, + ]; + if (prefixedSubcommand.command === 'task') { + const exitCode = await runTaskCommand(subcommandArgv); + if (exitCode !== 0) { + process.exit(exitCode); + } + return; + } + if (prefixedSubcommand.command === 'session') { + const exitCode = await runSessionCommand(subcommandArgv); + if (exitCode !== 0) { + process.exit(exitCode); + } + return; + } + const exitCode = await runProfileCommand(subcommandArgv); + if (exitCode !== 0) { + process.exit(exitCode); + } + return; + } + let args; + try { + args = parseCliArgs(argv); + } + catch (error) { + console.error(error.message); + console.error(getCliUsage()); + process.exit(1); + return; + } + if (args.help) { + console.log(getCliUsage()); + return; + } + if (args.version) { + console.log(get_browser_use_version()); + return; + } + if (args.debug) { + enableDebugLogging(); + } + if (args.mcp) { + await runMcpServer(); + return; + } + if (args.prompt == null && args.positional[0] === 'doctor') { + const report = await runDoctorChecks(); + printDoctorReport(report); + return; + } + if (args.prompt == null && args.positional[0] === 'install') { + console.log('Installing Chromium browser...'); + runInstallCommand(); + console.log('Chromium browser installed.'); + return; + } + if (args.prompt == null && args.positional[0] === 'setup') { + const exitCode = await runSetupCommand({ + mode: args.setup_mode, + yes: args.yes, + api_key: args.api_key, + }, { + json_output: args.json, + }); + if (exitCode !== 0) { + process.exit(exitCode); + } + return; + } + if (args.prompt == null && args.positional[0] === 'tunnel') { + const exitCode = await runTunnelCommand(args.positional.slice(1), { + json_output: args.json, + }); + if (exitCode !== 0) { + process.exit(exitCode); + } + return; + } + const task = resolveTask(args); + const shouldStartInteractive = shouldStartInteractiveMode(task); + if (!task && !shouldStartInteractive) { + console.error(getCliUsage()); + process.exit(1); + return; + } + let llm; + try { + llm = getLlmFromCliArgs(args); + } + catch (error) { + console.error(`Error selecting LLM: ${error.message}`); + process.exit(1); + return; + } + if (shouldStartInteractive) { + await runInteractiveMode(args, llm); + return; + } + if (!task) { + console.error(getCliUsage()); + process.exit(1); + return; + } + console.log(`Starting task: ${task}`); + const browserProfile = buildBrowserProfileFromCliArgs(args); + const browserSession = args.cdp_url + ? new BrowserSession({ + browser_profile: browserProfile ?? undefined, + cdp_url: args.cdp_url, + }) + : null; + try { + await runAgentTask({ + task, + llm, + browserProfile, + browserSession, + }); + } + catch (error) { + console.error('Error running agent:', error); + process.exit(1); + } +} +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} diff --git a/dist/code-use/formatting.d.ts b/dist/code-use/formatting.d.ts new file mode 100644 index 00000000..d3d54598 --- /dev/null +++ b/dist/code-use/formatting.d.ts @@ -0,0 +1,3 @@ +import type { BrowserStateSummary } from '../browser/views.js'; +import type { BrowserSession } from '../browser/session.js'; +export declare const format_browser_state_for_llm: (state: BrowserStateSummary, namespace: Record, _browser_session: BrowserSession) => Promise; diff --git a/dist/code-use/formatting.js b/dist/code-use/formatting.js new file mode 100644 index 00000000..59079d54 --- /dev/null +++ b/dist/code-use/formatting.js @@ -0,0 +1,18 @@ +export const format_browser_state_for_llm = async (state, namespace, _browser_session) => { + const lines = []; + lines.push('## Browser State'); + lines.push(`**URL:** ${state.url}`); + lines.push(`**Title:** ${state.title}`); + lines.push(''); + const vars = Object.keys(namespace) + .filter((key) => !key.startsWith('_')) + .sort(); + lines.push(`**Available:** ${vars.length > 0 ? vars.join(', ') : '(none)'}`); + lines.push(''); + const dom = typeof state.llm_representation === 'function' + ? state.llm_representation() + : ''; + lines.push('**DOM Structure:**'); + lines.push(dom || 'Empty DOM tree (you might have to wait for page load)'); + return lines.join('\n'); +}; diff --git a/dist/code-use/index.d.ts b/dist/code-use/index.d.ts new file mode 100644 index 00000000..45a3cbbe --- /dev/null +++ b/dist/code-use/index.d.ts @@ -0,0 +1,6 @@ +export * from './views.js'; +export * from './utils.js'; +export * from './namespace.js'; +export * from './service.js'; +export * from './formatting.js'; +export * from './notebook-export.js'; diff --git a/dist/code-use/index.js b/dist/code-use/index.js new file mode 100644 index 00000000..45a3cbbe --- /dev/null +++ b/dist/code-use/index.js @@ -0,0 +1,6 @@ +export * from './views.js'; +export * from './utils.js'; +export * from './namespace.js'; +export * from './service.js'; +export * from './formatting.js'; +export * from './notebook-export.js'; diff --git a/dist/code-use/namespace.d.ts b/dist/code-use/namespace.d.ts new file mode 100644 index 00000000..76058bd4 --- /dev/null +++ b/dist/code-use/namespace.d.ts @@ -0,0 +1,5 @@ +import type { BrowserSession } from '../browser/session.js'; +export interface CreateNamespaceOptions { + namespace?: Record; +} +export declare const create_namespace: (browser_session: BrowserSession, options?: CreateNamespaceOptions) => Record; diff --git a/dist/code-use/namespace.js b/dist/code-use/namespace.js new file mode 100644 index 00000000..fa75f136 --- /dev/null +++ b/dist/code-use/namespace.js @@ -0,0 +1,81 @@ +const buildExpression = (source, args) => `(${source})(${args.map((arg) => JSON.stringify(arg)).join(',')})`; +export const create_namespace = (browser_session, options = {}) => { + const namespace = options.namespace ?? {}; + namespace.browser = browser_session; + namespace.navigate = async (url, init = {}) => { + await browser_session.navigate_to(url, init); + }; + namespace.go_back = async () => { + await browser_session.go_back(); + }; + namespace.go_forward = async () => { + await browser_session.go_forward(); + }; + namespace.refresh = async () => { + await browser_session.refresh(); + }; + namespace.wait = async (seconds) => { + await browser_session.wait(seconds); + }; + namespace.click = async (index) => { + const node = await browser_session.get_dom_element_by_index(index); + if (!node) { + throw new Error(`Element index ${index} not found`); + } + return browser_session._click_element_node(node); + }; + namespace.input = async (index, text, clear = true) => { + const node = await browser_session.get_dom_element_by_index(index); + if (!node) { + throw new Error(`Element index ${index} not found`); + } + return browser_session._input_text_element_node(node, text, { + clear, + }); + }; + namespace.select_dropdown = async (index, text) => { + const node = await browser_session.get_dom_element_by_index(index); + if (!node) { + throw new Error(`Element index ${index} not found`); + } + return browser_session.select_dropdown_option(node, text); + }; + namespace.upload_file = async (index, file_path) => { + const node = await browser_session.get_dom_element_by_index(index); + if (!node) { + throw new Error(`Element index ${index} not found`); + } + return browser_session.upload_file(node, file_path); + }; + namespace.screenshot = async (full_page = false) => { + return browser_session.take_screenshot(full_page); + }; + namespace.send_keys = async (keys) => { + await browser_session.send_keys(keys); + }; + namespace.evaluate = async (code, ...args) => { + const page = await browser_session.get_current_page(); + if (!page) { + throw new Error('No active page for evaluate'); + } + if (typeof code === 'function') { + return page.evaluate(code, ...args); + } + if (args.length === 0) { + return page.evaluate(code); + } + return page.evaluate(buildExpression(code, args)); + }; + namespace.done = (result = null, success = true) => { + namespace._task_done = true; + namespace._task_success = success; + namespace._task_result = + typeof result === 'string' + ? result + : result == null + ? null + : JSON.stringify(result); + return result; + }; + return namespace; +}; diff --git a/dist/code-use/notebook-export.d.ts b/dist/code-use/notebook-export.d.ts new file mode 100644 index 00000000..b9f91c82 --- /dev/null +++ b/dist/code-use/notebook-export.d.ts @@ -0,0 +1,3 @@ +import type { CodeAgent } from './service.js'; +export declare const export_to_ipynb: (agent: CodeAgent, output_path: string) => string; +export declare const session_to_python_script: (agent: CodeAgent) => string; diff --git a/dist/code-use/notebook-export.js b/dist/code-use/notebook-export.js new file mode 100644 index 00000000..fd1bd69c --- /dev/null +++ b/dist/code-use/notebook-export.js @@ -0,0 +1,56 @@ +import fs from 'node:fs'; +import path from 'node:path'; +export const export_to_ipynb = (agent, output_path) => { + const notebook = { + nbformat: 4, + nbformat_minor: 5, + metadata: { + kernelspec: { + display_name: 'Node.js', + language: 'javascript', + name: 'javascript', + }, + }, + cells: agent.session.cells.map((cell) => { + if (cell.cell_type === 'markdown') { + return { + cell_type: 'markdown', + metadata: {}, + source: cell.source.split('\n').map((entry) => `${entry}\n`), + }; + } + return { + cell_type: 'code', + metadata: {}, + source: cell.source.split('\n').map((entry) => `${entry}\n`), + execution_count: cell.execution_count, + outputs: cell.output + ? [ + { + output_type: 'stream', + name: 'stdout', + text: `${cell.output}\n`, + }, + ] + : [], + }; + }), + }; + const resolved = path.resolve(output_path); + fs.mkdirSync(path.dirname(resolved), { recursive: true }); + fs.writeFileSync(resolved, JSON.stringify(notebook, null, 2), 'utf-8'); + return resolved; +}; +export const session_to_python_script = (agent) => { + const lines = []; + lines.push('# Generated from browser-use code-use session'); + lines.push(''); + for (const cell of agent.session.cells) { + if (cell.cell_type !== 'code') { + continue; + } + lines.push(cell.source); + lines.push(''); + } + return lines.join('\n'); +}; diff --git a/dist/code-use/service.d.ts b/dist/code-use/service.d.ts new file mode 100644 index 00000000..ec5c9890 --- /dev/null +++ b/dist/code-use/service.d.ts @@ -0,0 +1,24 @@ +import { CodeAgentHistory, CodeAgentHistoryList, NotebookSession } from './views.js'; +import type { BrowserSession } from '../browser/session.js'; +type ExecutorFn = (source: string, namespace: Record) => Promise; +export interface CodeAgentOptions { + task: string; + browser_session: BrowserSession; + namespace?: Record; + executor?: ExecutorFn; +} +export declare class CodeAgent { + task: string; + browser_session: BrowserSession; + session: NotebookSession; + namespace: Record; + complete_history: CodeAgentHistory[]; + private readonly executor; + constructor(options: CodeAgentOptions); + add_cell(source: string): import("./views.js").CodeCell; + execute_cell(source: string): Promise; + run(max_steps?: number): Promise; + get history(): CodeAgentHistoryList; + close(): Promise; +} +export {}; diff --git a/dist/code-use/service.js b/dist/code-use/service.js new file mode 100644 index 00000000..649bdd88 --- /dev/null +++ b/dist/code-use/service.js @@ -0,0 +1,104 @@ +import { CodeAgentHistory, CodeAgentHistoryList, CodeAgentState, CodeAgentStepMetadata, NotebookSession, } from './views.js'; +import { create_namespace } from './namespace.js'; +const AsyncFunction = Object.getPrototypeOf(async function () { + return undefined; +}).constructor; +const default_executor = async (source, namespace) => { + const injectableNames = Object.keys(namespace).filter((name) => /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(name)); + const prelude = injectableNames + .map((name) => `const ${name} = namespace[${JSON.stringify(name)}];`) + .join('\n'); + const runner = new AsyncFunction('namespace', `${prelude}\n${source}\nreturn undefined;`); + return runner(namespace); +}; +export class CodeAgent { + task; + browser_session; + session; + namespace; + complete_history = []; + executor; + constructor(options) { + this.task = options.task; + this.browser_session = options.browser_session; + this.namespace = create_namespace(options.browser_session, options.namespace ? { namespace: options.namespace } : {}); + this.session = new NotebookSession({ namespace: this.namespace }); + this.executor = options.executor ?? default_executor; + } + add_cell(source) { + return this.session.add_cell(source); + } + async execute_cell(source) { + const cell = this.add_cell(source); + const startedAt = Date.now() / 1000; + cell.status = 'running'; + cell.execution_count = this.session.increment_execution_count(); + let resultItem; + try { + const output = await this.executor(cell.source, this.namespace); + cell.status = 'success'; + cell.output = + output == null + ? null + : typeof output === 'string' + ? output + : JSON.stringify(output); + cell.error = null; + resultItem = { + extracted_content: cell.output, + error: null, + is_done: Boolean(this.namespace._task_done), + success: typeof this.namespace._task_success === 'boolean' + ? this.namespace._task_success + : null, + }; + } + catch (error) { + cell.status = 'error'; + cell.output = null; + cell.error = String(error?.message ?? error); + resultItem = { + extracted_content: null, + error: cell.error, + is_done: false, + success: false, + }; + } + const page = await this.browser_session.get_current_page(); + const state = new CodeAgentState({ + url: typeof page?.url === 'function' ? page.url() : null, + title: typeof page?.title === 'function' ? await page.title() : null, + }); + const metadata = new CodeAgentStepMetadata({ + step_start_time: startedAt, + step_end_time: Date.now() / 1000, + }); + const modelOutput = { + model_output: cell.source, + full_response: cell.source, + }; + const historyItem = new CodeAgentHistory({ + model_output: modelOutput, + result: [resultItem], + state, + metadata, + }); + this.complete_history.push(historyItem); + this.session._complete_history = [...this.complete_history]; + return cell; + } + async run(max_steps = 100) { + const pending = this.session.cells.filter((cell) => cell.status === 'pending'); + const toRun = pending.slice(0, Math.max(max_steps, 0)); + for (const cell of toRun) { + await this.execute_cell(cell.source); + } + return this.history; + } + get history() { + return new CodeAgentHistoryList(this.complete_history, null); + } + async close() { + // Keep lifecycle explicit; caller controls browser shutdown. + } +} diff --git a/dist/code-use/utils.d.ts b/dist/code-use/utils.d.ts new file mode 100644 index 00000000..17abcc4f --- /dev/null +++ b/dist/code-use/utils.d.ts @@ -0,0 +1,4 @@ +export declare const truncate_message_content: (content: string, max_length?: number) => string; +export declare const detect_token_limit_issue: (completion: string, completion_tokens: number | null, max_tokens: number | null, stop_reason: string | null) => [boolean, string | null]; +export declare const extract_url_from_task: (task: string) => string | null; +export declare const extract_code_blocks: (text: string) => Record; diff --git a/dist/code-use/utils.js b/dist/code-use/utils.js new file mode 100644 index 00000000..5be3b6b3 --- /dev/null +++ b/dist/code-use/utils.js @@ -0,0 +1,98 @@ +export const truncate_message_content = (content, max_length = 10000) => { + if (content.length <= max_length) { + return content; + } + return `${content.slice(0, max_length)}\n\n[... truncated ${content.length - max_length} characters for history]`; +}; +export const detect_token_limit_issue = (completion, completion_tokens, max_tokens, stop_reason) => { + if (stop_reason === 'max_tokens') { + return [true, `Response terminated due to max_tokens (${stop_reason})`]; + } + if (completion_tokens != null && + max_tokens != null && + max_tokens > 0 && + completion_tokens / max_tokens >= 0.9) { + return [ + true, + `Response used ${(completion_tokens / max_tokens) * 100}% of max_tokens (${completion_tokens}/${max_tokens})`, + ]; + } + if (completion.length >= 6) { + const last6 = completion.slice(-6); + const repeated = completion.split(last6).length - 1; + if (repeated >= 40) { + return [ + true, + `Repetitive output detected: last 6 chars "${last6}" appears ${repeated} times`, + ]; + } + } + return [false, null]; +}; +export const extract_url_from_task = (task) => { + const withoutEmails = task.replace(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g, ''); + const matches = []; + const patterns = [ + /https?:\/\/[^\s<>"']+/g, + /(?:www\.)?[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*\.[a-zA-Z]{2,}(?:\/[^\s<>"']*)?/g, + ]; + for (const pattern of patterns) { + const found = withoutEmails.match(pattern) ?? []; + for (const entry of found) { + const trimmed = entry.replace(/[.,;:!?()[\]]+$/g, ''); + matches.push(trimmed.startsWith('http://') || trimmed.startsWith('https://') + ? trimmed + : `https://${trimmed}`); + } + } + const unique = [...new Set(matches)]; + if (unique.length !== 1) { + return null; + } + return unique[0]; +}; +export const extract_code_blocks = (text) => { + const blocks = {}; + const regex = /(`{3,})(\w+)(?:\s+(\w+))?\n([\s\S]*?)\1(?:\n|$)/g; + let match; + let pythonIndex = 0; + while ((match = regex.exec(text)) != null) { + const langRaw = match[2].toLowerCase(); + const name = match[3] ?? null; + const content = match[4].replace(/\s+$/, ''); + if (!content) { + continue; + } + const lang = langRaw === 'javascript' || langRaw === 'js' + ? 'js' + : langRaw === 'markdown' || langRaw === 'md' + ? 'markdown' + : langRaw === 'sh' || langRaw === 'shell' + ? 'bash' + : langRaw; + if (!['python', 'js', 'bash', 'markdown'].includes(lang)) { + continue; + } + if (name) { + blocks[name] = content; + continue; + } + if (lang === 'python') { + blocks[`python_${pythonIndex}`] = content; + pythonIndex += 1; + continue; + } + blocks[lang] = content; + } + if (pythonIndex > 0) { + blocks.python = blocks.python_0; + } + else { + const genericMatches = [...text.matchAll(/```\n([\s\S]*?)```/g)].map((entry) => entry[1].trim()); + const generic = genericMatches.filter(Boolean).join('\n\n'); + if (generic) { + blocks.python = generic; + } + } + return blocks; +}; diff --git a/dist/code-use/views.d.ts b/dist/code-use/views.d.ts new file mode 100644 index 00000000..4036cd08 --- /dev/null +++ b/dist/code-use/views.d.ts @@ -0,0 +1,108 @@ +import type { UsageSummary } from '../tokens/views.js'; +export type CellType = 'code' | 'markdown'; +export type ExecutionStatus = 'pending' | 'running' | 'success' | 'error'; +export interface CodeCellInit { + id?: string; + cell_type?: CellType; + source: string; + output?: string | null; + execution_count?: number | null; + status?: ExecutionStatus; + error?: string | null; + browser_state?: string | null; +} +export declare class CodeCell { + id: string; + cell_type: CellType; + source: string; + output: string | null; + execution_count: number | null; + status: ExecutionStatus; + error: string | null; + browser_state: string | null; + constructor(init: CodeCellInit); +} +export interface CodeAgentModelOutput { + model_output: string; + full_response: string; +} +export interface CodeAgentResult { + extracted_content?: string | null; + error?: string | null; + is_done: boolean; + success?: boolean | null; +} +export declare class CodeAgentState { + url: string | null; + title: string | null; + screenshot_path: string | null; + constructor(init: { + url?: string | null; + title?: string | null; + screenshot_path?: string | null; + }); + get_screenshot(): string | null; +} +export declare class CodeAgentStepMetadata { + input_tokens: number | null; + output_tokens: number | null; + step_start_time: number; + step_end_time: number; + constructor(init: { + input_tokens?: number | null; + output_tokens?: number | null; + step_start_time: number; + step_end_time: number; + }); + get duration_seconds(): number; +} +export declare class CodeAgentHistory { + model_output: CodeAgentModelOutput | null; + result: CodeAgentResult[]; + state: CodeAgentState; + metadata: CodeAgentStepMetadata | null; + screenshot_path: string | null; + constructor(init: { + model_output?: CodeAgentModelOutput | null; + result?: CodeAgentResult[]; + state: CodeAgentState; + metadata?: CodeAgentStepMetadata | null; + screenshot_path?: string | null; + }); +} +export declare class CodeAgentHistoryList { + private readonly complete_history; + private readonly usage_summary; + constructor(complete_history: CodeAgentHistory[], usage_summary?: UsageSummary | null); + get history(): CodeAgentHistory[]; + get usage(): UsageSummary | null; + final_result(): string | null; + is_done(): boolean; + is_successful(): boolean | null; + errors(): (string | null)[]; + has_errors(): boolean; + urls(): (string | null)[]; + action_results(): CodeAgentResult[]; + extracted_content(): string[]; + number_of_steps(): number; + total_duration_seconds(): number; +} +export declare class NotebookSession { + id: string; + cells: CodeCell[]; + current_execution_count: number; + namespace: Record; + _complete_history: CodeAgentHistory[]; + _usage_summary: UsageSummary | null; + constructor(init?: { + id?: string; + cells?: CodeCell[]; + current_execution_count?: number; + namespace?: Record; + }); + add_cell(source: string): CodeCell; + get_cell(cell_id: string): CodeCell | null; + get_latest_cell(): CodeCell; + increment_execution_count(): number; + get history(): CodeAgentHistoryList; +} diff --git a/dist/code-use/views.js b/dist/code-use/views.js new file mode 100644 index 00000000..cfbf269e --- /dev/null +++ b/dist/code-use/views.js @@ -0,0 +1,165 @@ +import fs from 'node:fs'; +import { uuid7str } from '../utils.js'; +export class CodeCell { + id; + cell_type; + source; + output; + execution_count; + status; + error; + browser_state; + constructor(init) { + this.id = init.id ?? uuid7str(); + this.cell_type = init.cell_type ?? 'code'; + this.source = init.source; + this.output = init.output ?? null; + this.execution_count = init.execution_count ?? null; + this.status = init.status ?? 'pending'; + this.error = init.error ?? null; + this.browser_state = init.browser_state ?? null; + } +} +export class CodeAgentState { + url; + title; + screenshot_path; + constructor(init) { + this.url = init.url ?? null; + this.title = init.title ?? null; + this.screenshot_path = init.screenshot_path ?? null; + } + get_screenshot() { + if (!this.screenshot_path || !fs.existsSync(this.screenshot_path)) { + return null; + } + return Buffer.from(fs.readFileSync(this.screenshot_path)).toString('base64'); + } +} +export class CodeAgentStepMetadata { + input_tokens; + output_tokens; + step_start_time; + step_end_time; + constructor(init) { + this.input_tokens = init.input_tokens ?? null; + this.output_tokens = init.output_tokens ?? null; + this.step_start_time = init.step_start_time; + this.step_end_time = init.step_end_time; + } + get duration_seconds() { + return this.step_end_time - this.step_start_time; + } +} +export class CodeAgentHistory { + model_output; + result; + state; + metadata; + screenshot_path; + constructor(init) { + this.model_output = init.model_output ?? null; + this.result = init.result ?? []; + this.state = init.state; + this.metadata = init.metadata ?? null; + this.screenshot_path = init.screenshot_path ?? null; + } +} +export class CodeAgentHistoryList { + complete_history; + usage_summary; + constructor(complete_history, usage_summary = null) { + this.complete_history = complete_history; + this.usage_summary = usage_summary; + } + get history() { + return this.complete_history; + } + get usage() { + return this.usage_summary; + } + final_result() { + const last = this.complete_history[this.complete_history.length - 1]; + if (!last?.result?.length) { + return null; + } + return last.result[last.result.length - 1].extracted_content ?? null; + } + is_done() { + const last = this.complete_history[this.complete_history.length - 1]; + if (!last?.result?.length) { + return false; + } + return Boolean(last.result[last.result.length - 1].is_done); + } + is_successful() { + const last = this.complete_history[this.complete_history.length - 1]; + if (!last?.result?.length) { + return null; + } + const final = last.result[last.result.length - 1]; + return final.is_done ? (final.success ?? null) : null; + } + errors() { + return this.complete_history.map((entry) => { + const withError = entry.result.find((result) => Boolean(result.error)); + return withError?.error ?? null; + }); + } + has_errors() { + return this.errors().some((error) => Boolean(error)); + } + urls() { + return this.complete_history.map((entry) => entry.state.url); + } + action_results() { + return this.complete_history.flatMap((entry) => entry.result); + } + extracted_content() { + return this.action_results() + .map((entry) => entry.extracted_content ?? null) + .filter((entry) => typeof entry === 'string'); + } + number_of_steps() { + return this.complete_history.length; + } + total_duration_seconds() { + return this.complete_history.reduce((sum, entry) => { + return sum + (entry.metadata?.duration_seconds ?? 0); + }, 0); + } +} +export class NotebookSession { + id; + cells; + current_execution_count; + namespace; + _complete_history; + _usage_summary; + constructor(init = {}) { + this.id = init.id ?? uuid7str(); + this.cells = init.cells ?? []; + this.current_execution_count = init.current_execution_count ?? 0; + this.namespace = init.namespace ?? {}; + this._complete_history = []; + this._usage_summary = null; + } + add_cell(source) { + const cell = new CodeCell({ source }); + this.cells.push(cell); + return cell; + } + get_cell(cell_id) { + return this.cells.find((cell) => cell.id === cell_id) ?? null; + } + get_latest_cell() { + return this.cells[this.cells.length - 1] ?? null; + } + increment_execution_count() { + this.current_execution_count += 1; + return this.current_execution_count; + } + get history() { + return new CodeAgentHistoryList(this._complete_history, this._usage_summary); + } +} diff --git a/dist/config.d.ts b/dist/config.d.ts new file mode 100644 index 00000000..4cb7cbd2 --- /dev/null +++ b/dist/config.d.ts @@ -0,0 +1,123 @@ +export declare const is_running_in_docker: () => boolean; +declare class OldConfig { + private _dirs_created; + get BROWSER_USE_LOGGING_LEVEL(): string; + get ANONYMIZED_TELEMETRY(): boolean; + get BROWSER_USE_CLOUD_SYNC(): boolean; + get BROWSER_USE_CLOUD_API_URL(): string; + get BROWSER_USE_CLOUD_UI_URL(): string; + get BROWSER_USE_DEBUG_LOG_FILE(): string | null; + get BROWSER_USE_INFO_LOG_FILE(): string | null; + get XDG_CACHE_HOME(): string; + get XDG_CONFIG_HOME(): string; + get BROWSER_USE_CONFIG_DIR(): string; + get BROWSER_USE_CONFIG_FILE(): string; + get BROWSER_USE_PROFILES_DIR(): string; + get BROWSER_USE_DEFAULT_USER_DATA_DIR(): string; + get BROWSER_USE_EXTENSIONS_DIR(): string; + get OPENAI_API_KEY(): string; + get ANTHROPIC_API_KEY(): string; + get GOOGLE_API_KEY(): string; + get DEEPSEEK_API_KEY(): string; + get GROQ_API_KEY(): string; + get GROK_API_KEY(): string; + get NOVITA_API_KEY(): string; + get AZURE_OPENAI_ENDPOINT(): string; + get AZURE_OPENAI_KEY(): string; + get SKIP_LLM_API_KEY_VERIFICATION(): boolean; + get DEFAULT_LLM(): string; + get IN_DOCKER(): boolean; + get IS_IN_EVALS(): boolean; + get BROWSER_USE_VERSION_CHECK(): boolean; + get WIN_FONT_DIR(): string; + _ensure_dirs(base_dir?: string): void; +} +declare class FlatEnvConfig { + get BROWSER_USE_LOGGING_LEVEL(): string; + get ANONYMIZED_TELEMETRY(): boolean; + get BROWSER_USE_CLOUD_SYNC(): boolean | null; + get BROWSER_USE_CLOUD_API_URL(): string; + get BROWSER_USE_CLOUD_UI_URL(): string; + get BROWSER_USE_DEBUG_LOG_FILE(): string | null; + get BROWSER_USE_INFO_LOG_FILE(): string | null; + get XDG_CACHE_HOME(): string; + get XDG_CONFIG_HOME(): string; + get BROWSER_USE_CONFIG_DIR(): string | null; + get OPENAI_API_KEY(): string; + get ANTHROPIC_API_KEY(): string; + get GOOGLE_API_KEY(): string; + get DEEPSEEK_API_KEY(): string; + get GROQ_API_KEY(): string; + get GROK_API_KEY(): string; + get NOVITA_API_KEY(): string; + get AZURE_OPENAI_ENDPOINT(): string; + get AZURE_OPENAI_KEY(): string; + get SKIP_LLM_API_KEY_VERIFICATION(): boolean; + get DEFAULT_LLM(): string; + get IN_DOCKER(): boolean | null; + get IS_IN_EVALS(): boolean; + get BROWSER_USE_VERSION_CHECK(): boolean; + get WIN_FONT_DIR(): string; + get BROWSER_USE_CONFIG_PATH(): string | null; + get BROWSER_USE_HEADLESS(): boolean | null; + get BROWSER_USE_ALLOWED_DOMAINS(): string | null; + get BROWSER_USE_LLM_MODEL(): string | null; + get BROWSER_USE_PROXY_URL(): string | null; + get BROWSER_USE_NO_PROXY(): string | null; + get BROWSER_USE_PROXY_USERNAME(): string | null; + get BROWSER_USE_PROXY_PASSWORD(): string | null; + get BROWSER_USE_DISABLE_EXTENSIONS(): boolean | null; +} +interface DBStyleEntry { + id: string; + default: boolean; + created_at: string; +} +export interface BrowserProfileEntry extends DBStyleEntry { + headless?: boolean | null; + user_data_dir?: string | null; + allowed_domains?: string[] | null; + downloads_path?: string | null; + [key: string]: unknown; +} +export interface LLMEntry extends DBStyleEntry { + api_key?: string | null; + model?: string | null; + temperature?: number | null; + max_tokens?: number | null; +} +export interface AgentEntry extends DBStyleEntry { + max_steps?: number | null; + use_vision?: boolean | null; + system_prompt?: string | null; +} +export interface DBStyleConfigJSON { + browser_profile: Record; + llm: Record; + agent: Record; +} +type RuntimeConfig = { + browser_profile: Record; + llm: Record; + agent: Record; +}; +declare class ConfigCore { + private _get_config_path; + private _get_db_config; + private _get_default_entry; + _get_default_profile(): Record; + _get_default_llm(): Record; + _get_default_agent(): Record; + _load_config(): RuntimeConfig; + _ensure_dirs(): void; + load_config(): RuntimeConfig; + get_default_profile(): Record; + get_default_llm(): Record; + get_default_agent(): Record; +} +type ConfigType = ConfigCore & OldConfig & FlatEnvConfig; +export declare const CONFIG: ConfigType; +export declare const load_browser_use_config: () => RuntimeConfig; +export declare const get_default_profile: (config: Record) => any; +export declare const get_default_llm: (config: Record) => any; +export {}; diff --git a/dist/config.js b/dist/config.js new file mode 100644 index 00000000..1ce8101c --- /dev/null +++ b/dist/config.js @@ -0,0 +1,532 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { config as loadEnv } from 'dotenv'; +import { createLogger } from './logging-config.js'; +loadEnv(); +const logger = createLogger('browser_use.config'); +const expand_user = (value) => value.replace(/^~(?=$|\/|\\)/, os.homedir()); +const resolve_path = (value) => path.resolve(expand_user(value)); +const string_to_bool = (value, defaultValue = false) => { + if (value === undefined || value === null || value === '') { + return defaultValue; + } + return ['true', '1', 't', 'y', 'yes'].includes(value.toLowerCase()); +}; +let docker_cache = null; +export const is_running_in_docker = () => { + if (docker_cache !== null) { + return docker_cache; + } + try { + if (fs.existsSync('/.dockerenv')) { + docker_cache = true; + return true; + } + } + catch { + /* no-op */ + } + try { + const cgroup = fs.readFileSync('/proc/1/cgroup', 'utf-8').toLowerCase(); + if (cgroup.includes('docker') || cgroup.includes('kubepods')) { + docker_cache = true; + return true; + } + } + catch { + /* no-op */ + } + try { + const cmdline = fs.readFileSync('/proc/1/cmdline', 'utf-8').toLowerCase(); + if (cmdline.includes('py') || + cmdline.includes('uv') || + cmdline.includes('app')) { + docker_cache = true; + return true; + } + } + catch { + /* no-op */ + } + try { + const processes = fs + .readdirSync('/proc') + .filter((entry) => /^\d+$/.test(entry)); + if (processes.length > 0 && processes.length < 10) { + docker_cache = true; + return true; + } + } + catch { + /* no-op */ + } + docker_cache = false; + return false; +}; +const ensure_dir = (target) => fs.mkdirSync(target, { recursive: true }); +class OldConfig { + _dirs_created = false; + get BROWSER_USE_LOGGING_LEVEL() { + return (process.env.BROWSER_USE_LOGGING_LEVEL ?? 'info').toLowerCase(); + } + get ANONYMIZED_TELEMETRY() { + return string_to_bool(process.env.ANONYMIZED_TELEMETRY, true); + } + get BROWSER_USE_CLOUD_SYNC() { + const value = process.env.BROWSER_USE_CLOUD_SYNC; + return value ? string_to_bool(value) : this.ANONYMIZED_TELEMETRY; + } + get BROWSER_USE_CLOUD_API_URL() { + const url = process.env.BROWSER_USE_CLOUD_API_URL ?? 'https://api.browser-use.com'; + if (!url.includes('://')) { + throw new Error('BROWSER_USE_CLOUD_API_URL must be a valid URL'); + } + return url; + } + get BROWSER_USE_CLOUD_UI_URL() { + const url = process.env.BROWSER_USE_CLOUD_UI_URL ?? ''; + if (url && !url.includes('://')) { + throw new Error('BROWSER_USE_CLOUD_UI_URL must be a valid URL if set'); + } + return url; + } + get BROWSER_USE_DEBUG_LOG_FILE() { + return process.env.BROWSER_USE_DEBUG_LOG_FILE ?? null; + } + get BROWSER_USE_INFO_LOG_FILE() { + return process.env.BROWSER_USE_INFO_LOG_FILE ?? null; + } + get XDG_CACHE_HOME() { + return resolve_path(process.env.XDG_CACHE_HOME ?? '~/.cache'); + } + get XDG_CONFIG_HOME() { + return resolve_path(process.env.XDG_CONFIG_HOME ?? '~/.config'); + } + get BROWSER_USE_CONFIG_DIR() { + const configured = process.env.BROWSER_USE_CONFIG_DIR; + const dir = configured + ? resolve_path(configured) + : path.join(this.XDG_CONFIG_HOME, 'browseruse'); + this._ensure_dirs(dir); + return dir; + } + get BROWSER_USE_CONFIG_FILE() { + return path.join(this.BROWSER_USE_CONFIG_DIR, 'config.json'); + } + get BROWSER_USE_PROFILES_DIR() { + const dir = path.join(this.BROWSER_USE_CONFIG_DIR, 'profiles'); + this._ensure_dirs(dir); + return dir; + } + get BROWSER_USE_DEFAULT_USER_DATA_DIR() { + return path.join(this.BROWSER_USE_PROFILES_DIR, 'default'); + } + get BROWSER_USE_EXTENSIONS_DIR() { + const dir = path.join(this.BROWSER_USE_CONFIG_DIR, 'extensions'); + this._ensure_dirs(dir); + return dir; + } + get OPENAI_API_KEY() { + return process.env.OPENAI_API_KEY ?? ''; + } + get ANTHROPIC_API_KEY() { + return process.env.ANTHROPIC_API_KEY ?? ''; + } + get GOOGLE_API_KEY() { + return process.env.GOOGLE_API_KEY ?? ''; + } + get DEEPSEEK_API_KEY() { + return process.env.DEEPSEEK_API_KEY ?? ''; + } + get GROQ_API_KEY() { + return process.env.GROQ_API_KEY ?? process.env.GROK_API_KEY ?? ''; + } + get GROK_API_KEY() { + return this.GROQ_API_KEY; + } + get NOVITA_API_KEY() { + return process.env.NOVITA_API_KEY ?? ''; + } + get AZURE_OPENAI_ENDPOINT() { + return process.env.AZURE_OPENAI_ENDPOINT ?? ''; + } + get AZURE_OPENAI_KEY() { + return process.env.AZURE_OPENAI_KEY ?? ''; + } + get SKIP_LLM_API_KEY_VERIFICATION() { + return string_to_bool(process.env.SKIP_LLM_API_KEY_VERIFICATION, false); + } + get DEFAULT_LLM() { + return process.env.DEFAULT_LLM ?? ''; + } + get IN_DOCKER() { + return (string_to_bool(process.env.IN_DOCKER, false) || is_running_in_docker()); + } + get IS_IN_EVALS() { + return string_to_bool(process.env.IS_IN_EVALS, false); + } + get BROWSER_USE_VERSION_CHECK() { + return string_to_bool(process.env.BROWSER_USE_VERSION_CHECK, true); + } + get WIN_FONT_DIR() { + return process.env.WIN_FONT_DIR ?? 'C:\\Windows\\Fonts'; + } + _ensure_dirs(base_dir) { + if (!this._dirs_created) { + const config_dir = base_dir ?? + (process.env.BROWSER_USE_CONFIG_DIR + ? resolve_path(process.env.BROWSER_USE_CONFIG_DIR) + : path.join(this.XDG_CONFIG_HOME, 'browseruse')); + ensure_dir(config_dir); + ensure_dir(path.join(config_dir, 'profiles')); + ensure_dir(path.join(config_dir, 'extensions')); + this._dirs_created = true; + } + } +} +class FlatEnvConfig { + get BROWSER_USE_LOGGING_LEVEL() { + return process.env.BROWSER_USE_LOGGING_LEVEL ?? 'info'; + } + get ANONYMIZED_TELEMETRY() { + return string_to_bool(process.env.ANONYMIZED_TELEMETRY, true); + } + get BROWSER_USE_CLOUD_SYNC() { + const value = process.env.BROWSER_USE_CLOUD_SYNC; + return value ? string_to_bool(value) : null; + } + get BROWSER_USE_CLOUD_API_URL() { + return (process.env.BROWSER_USE_CLOUD_API_URL ?? 'https://api.browser-use.com'); + } + get BROWSER_USE_CLOUD_UI_URL() { + return process.env.BROWSER_USE_CLOUD_UI_URL ?? ''; + } + get BROWSER_USE_DEBUG_LOG_FILE() { + return process.env.BROWSER_USE_DEBUG_LOG_FILE ?? null; + } + get BROWSER_USE_INFO_LOG_FILE() { + return process.env.BROWSER_USE_INFO_LOG_FILE ?? null; + } + get XDG_CACHE_HOME() { + return resolve_path(process.env.XDG_CACHE_HOME ?? '~/.cache'); + } + get XDG_CONFIG_HOME() { + return resolve_path(process.env.XDG_CONFIG_HOME ?? '~/.config'); + } + get BROWSER_USE_CONFIG_DIR() { + return process.env.BROWSER_USE_CONFIG_DIR + ? resolve_path(process.env.BROWSER_USE_CONFIG_DIR) + : null; + } + get OPENAI_API_KEY() { + return process.env.OPENAI_API_KEY ?? ''; + } + get ANTHROPIC_API_KEY() { + return process.env.ANTHROPIC_API_KEY ?? ''; + } + get GOOGLE_API_KEY() { + return process.env.GOOGLE_API_KEY ?? ''; + } + get DEEPSEEK_API_KEY() { + return process.env.DEEPSEEK_API_KEY ?? ''; + } + get GROQ_API_KEY() { + return process.env.GROQ_API_KEY ?? process.env.GROK_API_KEY ?? ''; + } + get GROK_API_KEY() { + return this.GROQ_API_KEY; + } + get NOVITA_API_KEY() { + return process.env.NOVITA_API_KEY ?? ''; + } + get AZURE_OPENAI_ENDPOINT() { + return process.env.AZURE_OPENAI_ENDPOINT ?? ''; + } + get AZURE_OPENAI_KEY() { + return process.env.AZURE_OPENAI_KEY ?? ''; + } + get SKIP_LLM_API_KEY_VERIFICATION() { + return string_to_bool(process.env.SKIP_LLM_API_KEY_VERIFICATION, false); + } + get DEFAULT_LLM() { + return process.env.DEFAULT_LLM ?? ''; + } + get IN_DOCKER() { + const value = process.env.IN_DOCKER; + return value === undefined ? null : string_to_bool(value); + } + get IS_IN_EVALS() { + return string_to_bool(process.env.IS_IN_EVALS, false); + } + get BROWSER_USE_VERSION_CHECK() { + return string_to_bool(process.env.BROWSER_USE_VERSION_CHECK, true); + } + get WIN_FONT_DIR() { + return process.env.WIN_FONT_DIR ?? 'C:\\Windows\\Fonts'; + } + get BROWSER_USE_CONFIG_PATH() { + return process.env.BROWSER_USE_CONFIG_PATH + ? resolve_path(process.env.BROWSER_USE_CONFIG_PATH) + : null; + } + get BROWSER_USE_HEADLESS() { + const value = process.env.BROWSER_USE_HEADLESS; + return value === undefined ? null : string_to_bool(value); + } + get BROWSER_USE_ALLOWED_DOMAINS() { + return process.env.BROWSER_USE_ALLOWED_DOMAINS ?? null; + } + get BROWSER_USE_LLM_MODEL() { + return process.env.BROWSER_USE_LLM_MODEL ?? null; + } + get BROWSER_USE_PROXY_URL() { + return process.env.BROWSER_USE_PROXY_URL ?? null; + } + get BROWSER_USE_NO_PROXY() { + return process.env.BROWSER_USE_NO_PROXY ?? null; + } + get BROWSER_USE_PROXY_USERNAME() { + return process.env.BROWSER_USE_PROXY_USERNAME ?? null; + } + get BROWSER_USE_PROXY_PASSWORD() { + return process.env.BROWSER_USE_PROXY_PASSWORD ?? null; + } + get BROWSER_USE_DISABLE_EXTENSIONS() { + const value = process.env.BROWSER_USE_DISABLE_EXTENSIONS; + return value === undefined ? null : string_to_bool(value); + } +} +const create_default_config = () => { + logger.debug('Creating fresh default config.json'); + const profile_id = randomUUID(); + const llm_id = randomUUID(); + const agent_id = randomUUID(); + return { + browser_profile: { + [profile_id]: { + id: profile_id, + default: true, + created_at: new Date().toISOString(), + headless: false, + user_data_dir: null, + allowed_domains: null, + downloads_path: null, + }, + }, + llm: { + [llm_id]: { + id: llm_id, + default: true, + created_at: new Date().toISOString(), + model: 'gpt-4.1-mini', + api_key: null, + temperature: null, + max_tokens: null, + }, + }, + agent: { + [agent_id]: { + id: agent_id, + default: true, + created_at: new Date().toISOString(), + max_steps: null, + use_vision: null, + system_prompt: null, + }, + }, + }; +}; +const OPENAI_API_KEY_PLACEHOLDERS = new Set([ + 'your-openai-api-key-here', + 'your-openai-api-key', +]); +const sanitize_llm_api_key = (apiKey) => { + if (typeof apiKey !== 'string') { + return null; + } + const trimmed = apiKey.trim(); + if (!trimmed) { + return null; + } + if (OPENAI_API_KEY_PLACEHOLDERS.has(trimmed.toLowerCase())) { + return null; + } + return trimmed; +}; +const sanitize_db_config = (config) => { + const sanitizedLlmEntries = Object.fromEntries(Object.entries(config.llm ?? {}).map(([id, entry]) => [ + id, + { + ...entry, + api_key: sanitize_llm_api_key(entry.api_key), + }, + ])); + return { + ...config, + llm: sanitizedLlmEntries, + }; +}; +const looks_like_new_format = (data) => data && + typeof data === 'object' && + ['browser_profile', 'llm', 'agent'].every((key) => typeof data[key] === 'object') && + Object.values(data.browser_profile || {}).every((entry) => typeof entry === 'object' && 'id' in entry); +const load_and_migrate_config = (config_path) => { + if (!fs.existsSync(config_path)) { + const parent = path.dirname(config_path); + ensure_dir(parent); + const fresh = create_default_config(); + fs.writeFileSync(config_path, JSON.stringify(fresh, null, 2), 'utf-8'); + return fresh; + } + try { + const raw = JSON.parse(fs.readFileSync(config_path, 'utf-8')); + if (looks_like_new_format(raw)) { + return sanitize_db_config(raw); + } + logger.debug(`Old config format detected at ${config_path}, creating fresh config`); + const fresh = create_default_config(); + fs.writeFileSync(config_path, JSON.stringify(fresh, null, 2), 'utf-8'); + return fresh; + } + catch (error) { + logger.error(`Failed to load config from ${config_path}: ${error.message}, creating fresh config`); + const fresh = create_default_config(); + try { + fs.writeFileSync(config_path, JSON.stringify(fresh, null, 2), 'utf-8'); + } + catch (write_error) { + logger.error(`Failed to write fresh config: ${write_error.message}`); + } + return fresh; + } +}; +class ConfigCore { + _get_config_path() { + const env = new FlatEnvConfig(); + if (env.BROWSER_USE_CONFIG_PATH) { + return env.BROWSER_USE_CONFIG_PATH; + } + if (env.BROWSER_USE_CONFIG_DIR) { + return path.join(env.BROWSER_USE_CONFIG_DIR, 'config.json'); + } + return path.join(env.XDG_CONFIG_HOME, 'browseruse', 'config.json'); + } + _get_db_config() { + return load_and_migrate_config(this._get_config_path()); + } + _get_default_entry(records) { + for (const entry of Object.values(records)) { + if (entry.default) { + return { ...entry }; + } + } + const [first] = Object.values(records); + return first ? { ...first } : {}; + } + _get_default_profile() { + return this._get_default_entry(this._get_db_config().browser_profile); + } + _get_default_llm() { + return this._get_default_entry(this._get_db_config().llm); + } + _get_default_agent() { + return this._get_default_entry(this._get_db_config().agent); + } + _load_config() { + const config = { + browser_profile: this._get_default_profile(), + llm: this._get_default_llm(), + agent: this._get_default_agent(), + }; + const env = new FlatEnvConfig(); + if (env.BROWSER_USE_HEADLESS !== null) { + config.browser_profile.headless = env.BROWSER_USE_HEADLESS; + } + if (env.BROWSER_USE_ALLOWED_DOMAINS) { + config.browser_profile.allowed_domains = + env.BROWSER_USE_ALLOWED_DOMAINS.split(',') + .map((domain) => domain.trim()) + .filter(Boolean); + } + const proxy = {}; + if (env.BROWSER_USE_PROXY_URL) { + proxy.server = env.BROWSER_USE_PROXY_URL; + } + if (env.BROWSER_USE_NO_PROXY) { + proxy.bypass = env.BROWSER_USE_NO_PROXY.split(',') + .map((domain) => domain.trim()) + .filter(Boolean) + .join(','); + } + if (env.BROWSER_USE_PROXY_USERNAME) { + proxy.username = env.BROWSER_USE_PROXY_USERNAME; + } + if (env.BROWSER_USE_PROXY_PASSWORD) { + proxy.password = env.BROWSER_USE_PROXY_PASSWORD; + } + if (Object.keys(proxy).length > 0) { + config.browser_profile.proxy = proxy; + } + if (env.OPENAI_API_KEY) { + config.llm.api_key = env.OPENAI_API_KEY; + } + if (env.DEFAULT_LLM) { + config.llm.model = env.DEFAULT_LLM; + } + if (env.BROWSER_USE_LLM_MODEL) { + config.llm.model = env.BROWSER_USE_LLM_MODEL; + } + if (env.BROWSER_USE_DISABLE_EXTENSIONS !== null) { + config.browser_profile.enable_default_extensions = + !env.BROWSER_USE_DISABLE_EXTENSIONS; + } + return config; + } + _ensure_dirs() { + new OldConfig()._ensure_dirs(); + } + load_config() { + return this._load_config(); + } + get_default_profile() { + return this._get_default_profile(); + } + get_default_llm() { + return this._get_default_llm(); + } + get_default_agent() { + return this._get_default_agent(); + } +} +const config_handler = { + get(target, prop, receiver) { + if (typeof prop !== 'string') { + return Reflect.get(target, prop, receiver); + } + const old = new OldConfig(); + if (prop in old) { + const value = old[prop]; + return typeof value === 'function' + ? value.bind(old) + : value; + } + const env = new FlatEnvConfig(); + if (prop in env) { + const value = env[prop]; + return typeof value === 'function' + ? value.bind(env) + : value; + } + const coreValue = target[prop]; + if (typeof coreValue === 'function') { + return coreValue.bind(target); + } + return coreValue; + }, +}; +export const CONFIG = new Proxy(new ConfigCore(), config_handler); +export const load_browser_use_config = () => CONFIG.load_config(); +export const get_default_profile = (config) => config.browser_profile ?? {}; +export const get_default_llm = (config) => config.llm ?? {}; diff --git a/dist/controller/index.d.ts b/dist/controller/index.d.ts new file mode 100644 index 00000000..4f56cb93 --- /dev/null +++ b/dist/controller/index.d.ts @@ -0,0 +1,3 @@ +export * from './service.js'; +export * from './views.js'; +export * from './registry/index.js'; diff --git a/dist/controller/index.js b/dist/controller/index.js new file mode 100644 index 00000000..4f56cb93 --- /dev/null +++ b/dist/controller/index.js @@ -0,0 +1,3 @@ +export * from './service.js'; +export * from './views.js'; +export * from './registry/index.js'; diff --git a/dist/controller/registry/index.d.ts b/dist/controller/registry/index.d.ts new file mode 100644 index 00000000..c6dd1f8b --- /dev/null +++ b/dist/controller/registry/index.d.ts @@ -0,0 +1,2 @@ +export * from './service.js'; +export * from './views.js'; diff --git a/dist/controller/registry/index.js b/dist/controller/registry/index.js new file mode 100644 index 00000000..c6dd1f8b --- /dev/null +++ b/dist/controller/registry/index.js @@ -0,0 +1,2 @@ +export * from './service.js'; +export * from './views.js'; diff --git a/dist/controller/registry/service.d.ts b/dist/controller/registry/service.d.ts new file mode 100644 index 00000000..40cbac60 --- /dev/null +++ b/dist/controller/registry/service.d.ts @@ -0,0 +1,54 @@ +import { type ZodTypeAny } from 'zod'; +import type { Page } from '../../browser/types.js'; +import { FileSystem } from '../../filesystem/file-system.js'; +import { ActionModel, RegisteredAction } from './views.js'; +import type { BrowserSession } from '../../browser/session.js'; +type BaseChatModel = unknown; +export interface SensitiveDataMap { + [key: string]: string | Record; +} +export interface ExecuteActionContext { + context?: Context; + browser_session?: BrowserSession | null; + browser?: BrowserSession | null; + browser_context?: BrowserSession | null; + page_url?: string | null; + cdp_client?: unknown; + page_extraction_llm?: BaseChatModel | null; + extraction_schema?: Record | null; + file_system?: FileSystem | null; + available_file_paths?: string[] | null; + sensitive_data?: SensitiveDataMap | null; + signal?: AbortSignal | null; +} +export type RegistryActionHandler = (params: Params, ctx: ExecuteActionContext & { + page?: Page | null; + has_sensitive_data?: boolean; +}) => Promise | unknown; +export interface ActionOptions { + param_model?: ZodTypeAny; + action_name?: string; + domains?: string[] | null; + allowed_domains?: string[] | null; + page_filter?: ((page: Page) => boolean) | null; + terminates_sequence?: boolean; +} +export declare class Registry { + private registry; + private excludeActions; + constructor(exclude_actions?: string[] | null); + action(description: string, options?: ActionOptions): (handler: RegistryActionHandler) => any; + get_action(action_name: string): RegisteredAction | null; + exclude_action(action_name: string): void; + remove_action(action_name: string): void; + get_all_actions(): Map; + execute_action: (...args: any[]) => any; + private replace_sensitive_data; + private log_sensitive_data_usage; + create_action_model(options?: { + include_actions?: string[] | null; + page?: Page | null; + }): typeof ActionModel; + get_prompt_description(page?: Page | null): string; +} +export {}; diff --git a/dist/controller/registry/service.js b/dist/controller/registry/service.js new file mode 100644 index 00000000..50608bde --- /dev/null +++ b/dist/controller/registry/service.js @@ -0,0 +1,440 @@ +import { z } from 'zod'; +import { createHmac } from 'node:crypto'; +import { createLogger } from '../../logging-config.js'; +import { observe_debug } from '../../observability.js'; +import { time_execution_async } from '../../utils.js'; +import { is_new_tab_page, match_url_with_domain_pattern } from '../../utils.js'; +import { BrowserError } from '../../browser/views.js'; +import { ActionModel, ActionRegistry, RegisteredAction } from './views.js'; +const logger = createLogger('browser_use.controller.registry'); +const SPECIAL_PARAM_NAMES = new Set([ + 'context', + 'browser_session', + 'browser', + 'browser_context', + 'page', + 'page_url', + 'cdp_client', + 'page_extraction_llm', + 'available_file_paths', + 'has_sensitive_data', + 'file_system', + 'extraction_schema', + 'sensitive_data', + 'signal', +]); +const splitTopLevelParameters = (paramsSource) => { + const segments = []; + let current = ''; + let depthParen = 0; + let depthBrace = 0; + let depthBracket = 0; + let quote = null; + let escaped = false; + const flush = () => { + const trimmed = current.trim(); + if (trimmed) { + segments.push(trimmed); + } + current = ''; + }; + for (const char of paramsSource) { + if (escaped) { + current += char; + escaped = false; + continue; + } + if (char === '\\') { + current += char; + escaped = true; + continue; + } + if (quote) { + current += char; + if (char === quote) { + quote = null; + } + continue; + } + if (char === "'" || char === '"' || char === '`') { + current += char; + quote = char; + continue; + } + if (char === '(') + depthParen += 1; + else if (char === ')') + depthParen -= 1; + else if (char === '{') + depthBrace += 1; + else if (char === '}') + depthBrace -= 1; + else if (char === '[') + depthBracket += 1; + else if (char === ']') + depthBracket -= 1; + if (char === ',' && + depthParen === 0 && + depthBrace === 0 && + depthBracket === 0) { + flush(); + continue; + } + current += char; + } + flush(); + return segments; +}; +const extractFunctionParameters = (fn) => { + const source = fn.toString().trim(); + if (!source) { + return null; + } + let paramsSource; + const arrowIndex = source.indexOf('=>'); + if (arrowIndex !== -1) { + let lhs = source.slice(0, arrowIndex).trim(); + if (lhs.startsWith('async ')) { + lhs = lhs.slice('async '.length).trim(); + } + if (lhs.startsWith('(') && lhs.endsWith(')')) { + paramsSource = lhs.slice(1, -1); + } + else { + paramsSource = lhs; + } + } + else { + const openIndex = source.indexOf('('); + const closeIndex = source.indexOf(')', openIndex + 1); + if (openIndex === -1 || closeIndex === -1) { + return null; + } + paramsSource = source.slice(openIndex + 1, closeIndex); + } + const tokens = splitTopLevelParameters(paramsSource); + if (!tokens.length) { + return []; + } + const parsed = []; + for (const token of tokens) { + if (token.startsWith('...')) { + const fnName = fn.name || ''; + throw new Error(`Action '${fnName}' has ${token} which is not allowed. Actions must have explicit positional parameters only.`); + } + if (token.includes('{') || + token.includes('}') || + token.includes('[') || + token.includes(']')) { + return null; + } + const eqIndex = token.indexOf('='); + const name = (eqIndex === -1 ? token : token.slice(0, eqIndex)).trim(); + if (!name) { + return null; + } + parsed.push({ + name, + hasDefault: eqIndex !== -1, + }); + } + return parsed; +}; +const isAbortError = (error) => error instanceof Error && error.name === 'AbortError'; +const createAbortError = (reason) => { + if (isAbortError(reason)) { + return reason; + } + const message = reason instanceof Error ? reason.message : 'Operation aborted'; + const error = new Error(message); + error.name = 'AbortError'; + if (reason !== undefined) { + error.cause = reason; + } + return error; +}; +const wrapActionExecutionError = (actionName, error) => { + if (error instanceof BrowserError) { + return error; + } + const message = error instanceof Error ? error.message : String(error); + const wrapped = new Error(`Error executing action ${actionName}: ${message}`); + if (error !== undefined) { + wrapped.cause = error; + } + return wrapped; +}; +const isSpecialContextMissingError = (error) => { + if (!(error instanceof Error)) { + return false; + } + return (error.message.includes('requires browser_session but none provided') || + error.message.includes('requires page_extraction_llm but none provided')); +}; +const safeJsonStringify = (value) => { + try { + return JSON.stringify(value); + } + catch { + return String(value); + } +}; +const isTimeoutError = (error) => error instanceof Error && error.name === 'TimeoutError'; +const BASE32_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'; +const decodeBase32Secret = (secret) => { + const sanitized = secret.toUpperCase().replace(/[^A-Z2-7]/g, ''); + if (!sanitized.length) { + throw new Error('Invalid TOTP secret: empty base32 payload'); + } + let bits = 0; + let bitBuffer = 0; + const bytes = []; + for (const char of sanitized) { + const value = BASE32_ALPHABET.indexOf(char); + if (value < 0) { + throw new Error(`Invalid base32 character in TOTP secret: ${char}`); + } + bitBuffer = (bitBuffer << 5) | value; + bits += 5; + while (bits >= 8) { + bytes.push((bitBuffer >>> (bits - 8)) & 0xff); + bits -= 8; + } + } + if (!bytes.length) { + throw new Error('Invalid TOTP secret: failed to decode base32 payload'); + } + return Buffer.from(bytes); +}; +const generateTotpCode = (secret) => { + const key = decodeBase32Secret(secret); + const counter = Math.floor(Date.now() / 1000 / 30); + const counterBuffer = Buffer.alloc(8); + counterBuffer.writeBigUInt64BE(BigInt(counter)); + const hmac = createHmac('sha1', key).update(counterBuffer).digest(); + const offset = hmac[hmac.length - 1] & 0x0f; + const binaryCode = ((hmac[offset] & 0x7f) << 24) | + ((hmac[offset + 1] & 0xff) << 16) | + ((hmac[offset + 2] & 0xff) << 8) | + (hmac[offset + 3] & 0xff); + return String(binaryCode % 1_000_000).padStart(6, '0'); +}; +export class Registry { + registry = new ActionRegistry(); + excludeActions; + constructor(exclude_actions = null) { + this.excludeActions = new Set(exclude_actions ?? []); + } + action(description, options = {}) { + if (options.allowed_domains && options.domains) { + throw new Error("Cannot specify both 'domains' and 'allowed_domains' - they are aliases for the same parameter"); + } + let schema = options.param_model ?? z.object({}).strict(); + const actionNameOverride = options.action_name ?? null; + const domains = options.allowed_domains ?? options.domains ?? null; + const pageFilter = options.page_filter ?? null; + const terminatesSequence = options.terminates_sequence ?? false; + return (handler) => { + const actionName = actionNameOverride ?? handler.name; + if (this.excludeActions.has(actionName)) { + return handler; + } + const parsedHandlerParams = extractFunctionParameters(handler); + let normalizedHandler = handler; + if (!options.param_model) { + const supportsCompatSignature = Boolean(parsedHandlerParams && + parsedHandlerParams.length > 0 && + !(parsedHandlerParams.length <= 2 && + parsedHandlerParams[0]?.name === 'params')); + if (supportsCompatSignature && parsedHandlerParams) { + const actionParams = parsedHandlerParams.filter((entry) => !SPECIAL_PARAM_NAMES.has(entry.name)); + const shape = Object.fromEntries(actionParams.map((entry) => [ + entry.name, + entry.hasDefault ? z.any().optional() : z.any(), + ])); + schema = z.object(shape).strict(); + normalizedHandler = ((params, ctx) => { + const args = parsedHandlerParams.map((entry) => { + if (SPECIAL_PARAM_NAMES.has(entry.name)) { + const value = ctx[entry.name]; + if ((value === null || value === undefined) && + !entry.hasDefault) { + throw new Error(`Action ${actionName} requires ${entry.name} but none provided.`); + } + return value; + } + const value = params[entry.name]; + if (value === undefined && !entry.hasDefault) { + throw new Error(`${actionName}() missing required parameter '${entry.name}'`); + } + return value; + }); + return handler(...args); + }); + } + } + const action = new RegisteredAction(actionName, description, normalizedHandler, schema, domains, pageFilter, terminatesSequence); + this.registry.register(action); + return normalizedHandler; + }; + } + get_action(action_name) { + return this.registry.get(action_name); + } + exclude_action(action_name) { + this.excludeActions.add(action_name); + this.registry.remove(action_name); + } + remove_action(action_name) { + this.registry.remove(action_name); + } + get_all_actions() { + return this.registry.actionsMap; + } + execute_action = observe_debug({ + name: 'execute_action', + ignore_input: true, + ignore_output: true, + })(time_execution_async('--execute_action')(async (action_name, params, { browser_session = null, page_extraction_llm = null, extraction_schema = null, file_system = null, sensitive_data = null, available_file_paths = null, signal = null, context = null, } = {}) => { + const action = this.registry.get(action_name); + if (!action) { + throw new Error(`Action ${action_name} not found`); + } + const parsed = action.paramSchema.safeParse(params); + if (!parsed.success) { + throw new Error(`Invalid parameters ${safeJsonStringify(params)} for action ${action_name}: ${parsed.error.message}`); + } + let validatedParams = parsed.data; + let currentUrl = null; + if (browser_session?.agent_current_page?.url) { + currentUrl = browser_session.agent_current_page.url(); + } + else if (browser_session?.get_current_page) { + const currentPage = await browser_session.get_current_page(); + currentUrl = currentPage?.url() ?? null; + } + if (sensitive_data) { + validatedParams = this.replace_sensitive_data(validatedParams, sensitive_data, currentUrl); + } + let page = null; + if (browser_session?.get_current_page) { + page = await browser_session.get_current_page(); + } + const ctx = { + context, + browser_session, + browser: browser_session, + browser_context: browser_session, + page, + page_url: currentUrl, + cdp_client: browser_session?.cdp_client ?? null, + page_extraction_llm, + extraction_schema, + file_system, + available_file_paths, + sensitive_data, + signal, + has_sensitive_data: (action_name === 'input_text' || action_name === 'input') && + Boolean(sensitive_data), + }; + if (signal?.aborted) { + throw createAbortError(signal.reason); + } + try { + return await action.handler(validatedParams, ctx); + } + catch (error) { + if (signal?.aborted || isAbortError(error)) { + throw createAbortError(signal?.reason ?? error); + } + if (isTimeoutError(error)) { + throw new Error(`Error executing action ${action_name} due to timeout.`, { cause: error }); + } + if (isSpecialContextMissingError(error)) { + throw error; + } + throw wrapActionExecutionError(action_name, error); + } + })); + replace_sensitive_data(params, sensitiveData, currentUrl) { + const secretPattern = /(.*?)<\/secret>/g; + const applicableSecrets = {}; + for (const [domainOrKey, content] of Object.entries(sensitiveData)) { + if (content && typeof content === 'object' && !Array.isArray(content)) { + if (currentUrl && + !is_new_tab_page(currentUrl) && + match_url_with_domain_pattern(currentUrl, domainOrKey)) { + Object.assign(applicableSecrets, content); + } + } + else if (typeof content === 'string') { + applicableSecrets[domainOrKey] = content; + } + } + const cloneValue = (value) => { + if (Array.isArray(value)) { + return value.map((item) => cloneValue(item)); + } + if (value && typeof value === 'object') { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, cloneValue(v)])); + } + return value; + }; + const processed = cloneValue(params); + const replaced = new Set(); + const missing = new Set(); + const traverse = (value) => { + if (typeof value === 'string') { + return value.replace(secretPattern, (_, placeholderValue) => { + const placeholder = String(placeholderValue); + if (placeholder in applicableSecrets) { + replaced.add(placeholder); + const replacement = applicableSecrets[placeholder]; + if (placeholder.endsWith('bu_2fa_code')) { + return generateTotpCode(replacement); + } + return replacement; + } + missing.add(placeholder); + return `${placeholder}`; + }); + } + if (Array.isArray(value)) { + return value.map(traverse); + } + if (value && typeof value === 'object') { + for (const [key, val] of Object.entries(value)) { + value[key] = traverse(val); + } + } + return value; + }; + traverse(processed); + this.log_sensitive_data_usage(replaced, currentUrl); + if (missing.size > 0) { + logger.warning(`Missing or empty keys in sensitive_data dictionary: ${Array.from(missing).join(', ')}`); + } + return processed; + } + log_sensitive_data_usage(placeholders, currentUrl) { + if (!placeholders.size) { + return; + } + const urlInfo = currentUrl && !is_new_tab_page(currentUrl) ? ` on ${currentUrl}` : ''; + logger.info(`🔒 Using sensitive data placeholders: ${Array.from(placeholders).sort().join(', ')}${urlInfo}`); + } + create_action_model(options = {}) { + const { include_actions = null, page = null } = options; + const availableActions = this.registry.getAvailableActions(page, include_actions); + class DynamicActionModel extends ActionModel { + static available_actions = availableActions.map((action) => action.name); + } + Object.defineProperty(DynamicActionModel, 'name', { + value: 'ActionModel', + }); + return DynamicActionModel; + } + get_prompt_description(page) { + return this.registry.get_prompt_description(page ?? undefined); + } +} diff --git a/dist/controller/registry/views.d.ts b/dist/controller/registry/views.d.ts new file mode 100644 index 00000000..ea7bf97e --- /dev/null +++ b/dist/controller/registry/views.d.ts @@ -0,0 +1,58 @@ +import { type ZodTypeAny } from 'zod'; +import type { Page } from '../../browser/types.js'; +export type ActionHandler = (...args: any[]) => Promise | unknown; +type BrowserSession = unknown; +type BaseChatModel = unknown; +type FileSystem = unknown; +export declare class RegisteredAction { + readonly name: string; + readonly description: string; + readonly handler: ActionHandler; + readonly paramSchema: ZodTypeAny; + readonly domains: string[] | null; + readonly pageFilter: ((page: Page) => boolean) | null; + readonly terminates_sequence: boolean; + constructor(name: string, description: string, handler: ActionHandler, paramSchema: ZodTypeAny, domains?: string[] | null, pageFilter?: ((page: Page) => boolean) | null, terminates_sequence?: boolean); + promptDescription(): string; +} +export declare class ActionModel { + constructor(initialData?: Record); + private data; + toJSON(): Record; + model_dump(options?: { + exclude_none?: boolean; + }): any; + model_dump_json(options?: { + exclude_none?: boolean; + }): string; + get_index(): number | null; + set_index(index: number): void; +} +export declare class ActionRegistry { + private actions; + register(action: RegisteredAction): void; + remove(name: string): void; + get(name: string): RegisteredAction | null; + getAll(): RegisteredAction[]; + get actionsMap(): Map; + get actionEntries(): RegisteredAction[]; + private _matchDomains; + private _matchPageFilter; + getAvailableActions(page?: Page | null, includeActions?: string[] | null): RegisteredAction[]; + get_prompt_description(page?: Page | null): string; +} +export declare class SpecialActionParameters { + context: any | null; + browser_session: BrowserSession | null; + browser: BrowserSession | null; + browser_context: BrowserSession | null; + page: Page | null; + page_extraction_llm: BaseChatModel | null; + extraction_schema: Record | null; + file_system: FileSystem | null; + available_file_paths: string[] | null; + signal: AbortSignal | null; + has_sensitive_data: boolean; + static get_browser_requiring_params(): Set; +} +export {}; diff --git a/dist/controller/registry/views.js b/dist/controller/registry/views.js new file mode 100644 index 00000000..5cfcf424 --- /dev/null +++ b/dist/controller/registry/views.js @@ -0,0 +1,211 @@ +import { z } from 'zod'; +import { match_url_with_domain_pattern } from '../../utils.js'; +const getPageUrl = (page) => { + if (!page) { + return ''; + } + const candidate = page.url; + if (typeof candidate === 'function') { + try { + return candidate.call(page); + } + catch { + return ''; + } + } + return candidate ?? ''; +}; +// Render an action's param schema as compact JSON Schema for the LLM prompt. +// Replaces a prior raw dump of zod's private `_def` AST, which leaked +// internal keys like `innerType`/`defaultValue` and confused the LLM into +// copying default booleans into numeric fields (see scroll.num_pages bug). +function renderParamsJsonSchema(schema, skipKeys) { + const raw = z.toJSONSchema(schema, { unrepresentable: 'any' }); + // Strip dialect noise the LLM doesn't need. + delete raw.$schema; + const properties = raw.properties ?? {}; + const filteredProps = {}; + for (const [key, value] of Object.entries(properties)) { + if (skipKeys.has(key)) { + continue; + } + filteredProps[key] = value; + } + raw.properties = filteredProps; + if (Array.isArray(raw.required)) { + raw.required = raw.required.filter((key) => typeof key === 'string' && !skipKeys.has(key)); + if (raw.required.length === 0) { + delete raw.required; + } + } + return raw; +} +export class RegisteredAction { + name; + description; + handler; + paramSchema; + domains; + pageFilter; + terminates_sequence; + constructor(name, description, handler, paramSchema, domains = null, pageFilter = null, terminates_sequence = false) { + this.name = name; + this.description = description; + this.handler = handler; + this.paramSchema = paramSchema; + this.domains = domains; + this.pageFilter = pageFilter; + this.terminates_sequence = terminates_sequence; + } + promptDescription() { + const skipKeys = new Set(['title']); + let description = `${this.description}: \n`; + description += `{${this.name}: `; + const schemaShape = (this.paramSchema instanceof z.ZodObject && this.paramSchema.shape) || + ('shape' in this.paramSchema ? this.paramSchema.shape : null); + const hideStructuredDoneSuccess = Boolean(this.name === 'done' && + schemaShape && + typeof schemaShape === 'object' && + Object.prototype.hasOwnProperty.call(schemaShape, 'data') && + Object.prototype.hasOwnProperty.call(schemaShape, 'success')); + if (hideStructuredDoneSuccess) { + skipKeys.add('success'); + } + const hideExtractOutputSchema = Boolean(this.name === 'extract_structured_data' && + schemaShape && + typeof schemaShape === 'object' && + Object.prototype.hasOwnProperty.call(schemaShape, 'output_schema')); + if (hideExtractOutputSchema) { + skipKeys.add('output_schema'); + } + const jsonSchema = renderParamsJsonSchema(this.paramSchema, skipKeys); + description += JSON.stringify(jsonSchema); + description += '}'; + return description; + } +} +export class ActionModel { + constructor(initialData = {}) { + this.data = initialData; + } + data; + toJSON() { + return this.data; + } + model_dump(options) { + const clone = JSON.parse(JSON.stringify(this.data)); + if (options?.exclude_none) { + for (const [key, value] of Object.entries(clone)) { + if (value === null || value === undefined) { + delete clone[key]; + } + } + } + return clone; + } + model_dump_json(options) { + return JSON.stringify(this.model_dump(options)); + } + get_index() { + for (const value of Object.values(this.data)) { + if (value && typeof value === 'object' && 'index' in value) { + return value.index ?? null; + } + } + return null; + } + set_index(index) { + const [actionName] = Object.keys(this.data); + if (!actionName) { + return; + } + const params = this.data[actionName]; + if (params && typeof params === 'object' && 'index' in params) { + params.index = index; + } + } +} +export class ActionRegistry { + actions = new Map(); + register(action) { + this.actions.set(action.name, action); + } + remove(name) { + this.actions.delete(name); + } + get(name) { + return this.actions.get(name) ?? null; + } + getAll() { + return Array.from(this.actions.values()); + } + get actionsMap() { + return new Map(this.actions); + } + get actionEntries() { + return Array.from(this.actions.values()); + } + _matchDomains(domains, pageUrl) { + if (!domains || domains.length === 0) { + return true; + } + if (!pageUrl) { + return false; + } + return domains.some((pattern) => { + try { + return match_url_with_domain_pattern(pageUrl, pattern); + } + catch { + return false; + } + }); + } + _matchPageFilter(pageFilter, page) { + if (!pageFilter) { + return true; + } + try { + return pageFilter(page); + } + catch { + return false; + } + } + getAvailableActions(page, includeActions) { + const include = includeActions ? new Set(includeActions) : null; + return this.actionEntries.filter((action) => { + if (include && !include.has(action.name)) { + return false; + } + if (!page) { + return !action.pageFilter && !action.domains; + } + const pageUrl = getPageUrl(page); + const domainAllowed = this._matchDomains(action.domains, pageUrl); + const pageAllowed = this._matchPageFilter(action.pageFilter, page); + return domainAllowed && pageAllowed; + }); + } + get_prompt_description(page) { + return this.getAvailableActions(page) + .map((action) => action.promptDescription()) + .join('\n'); + } +} +export class SpecialActionParameters { + context = null; + browser_session = null; + browser = null; + browser_context = null; + page = null; + page_extraction_llm = null; + extraction_schema = null; + file_system = null; + available_file_paths = null; + signal = null; + has_sensitive_data = false; + static get_browser_requiring_params() { + return new Set(['browser_session', 'browser', 'browser_context', 'page']); + } +} diff --git a/dist/controller/service.d.ts b/dist/controller/service.d.ts new file mode 100644 index 00000000..9dc624e9 --- /dev/null +++ b/dist/controller/service.d.ts @@ -0,0 +1,58 @@ +import { z } from 'zod'; +import { ActionResult } from '../agent/views.js'; +import { FileSystem } from '../filesystem/file-system.js'; +import { Registry } from './registry/service.js'; +type BrowserSession = any; +type BaseChatModel = { + ainvoke: (messages: any[], output_format?: undefined, options?: { + signal?: AbortSignal; + }) => Promise<{ + completion: string; + }>; +}; +export interface ControllerOptions { + exclude_actions?: string[]; + output_model?: z.ZodTypeAny | null; + display_files_in_done_text?: boolean; + context?: Context; +} +export interface ActParams { + browser_session: BrowserSession; + page_extraction_llm?: BaseChatModel | null; + sensitive_data?: Record> | null; + available_file_paths?: string[] | null; + file_system?: FileSystem | null; + context?: Context | null; + signal?: AbortSignal | null; +} +export declare class Controller { + registry: Registry; + private displayFilesInDoneText; + private outputModel; + private coordinateClickingEnabled; + private clickActionHandler; + private logger; + constructor(options?: ControllerOptions); + private registerDefaultActions; + private registerNavigationActions; + private registerElementActions; + private registerClickActions; + private registerTabActions; + private registerContentActions; + private registerExplorationActions; + private registerScrollActions; + private registerFileSystemActions; + private registerUtilityActions; + private registerKeyboardActions; + private registerDropdownActions; + private registerSheetsActions; + private gotoSheetsRange; + private registerDoneAction; + use_structured_output_action(outputModel: z.ZodTypeAny): void; + get_output_model(): z.ZodTypeAny | null; + exclude_action(actionName: string): void; + set_coordinate_clicking(enabled: boolean): void; + action(description: string, options?: {}): (handler: import("./index.js").RegistryActionHandler) => any; + act(action: Record, { browser_session, page_extraction_llm, sensitive_data, available_file_paths, file_system, context, signal, }: ActParams): Promise; +} +export {}; diff --git a/dist/controller/service.js b/dist/controller/service.js new file mode 100644 index 00000000..f6e8f3f9 --- /dev/null +++ b/dist/controller/service.js @@ -0,0 +1,2488 @@ +import fs, { promises as fsp } from 'node:fs'; +import path from 'node:path'; +import { validate as validateJsonSchema } from '@cfworker/json-schema'; +import { ActionResult } from '../agent/views.js'; +import { ClickCoordinateEvent, ClickElementEvent, CloseTabEvent, GetDropdownOptionsEvent, GoBackEvent, NavigateToUrlEvent, ScrollEvent, ScrollToTextEvent, ScreenshotEvent, SelectDropdownOptionEvent, SendKeysEvent, SwitchTabEvent, TypeTextEvent, UploadFileEvent, WaitEvent, } from '../browser/events.js'; +import { BrowserError } from '../browser/views.js'; +import { chunkMarkdownByStructure, extractCleanMarkdownFromHtml, } from '../dom/markdown-extractor.js'; +import { FileSystem } from '../filesystem/file-system.js'; +import { ClickElementActionIndexOnlySchema, ClickElementActionSchema, CloseTabActionSchema, DoneActionSchema, EvaluateActionSchema, ExtractStructuredDataActionSchema, FindElementsActionSchema, DropdownOptionsActionSchema, SelectDropdownActionSchema, GoToUrlActionSchema, InputTextActionSchema, NoParamsActionSchema, ReadFileActionSchema, ReplaceFileStrActionSchema, ScrollActionSchema, ScrollToTextActionSchema, SearchActionSchema, SearchPageActionSchema, SearchGoogleActionSchema, ScreenshotActionSchema, SaveAsPdfActionSchema, StructuredOutputActionSchema, SwitchTabActionSchema, UploadFileActionSchema, WaitActionSchema, WriteFileActionSchema, SendKeysActionSchema, SheetsRangeActionSchema, SheetsUpdateActionSchema, SheetsInputActionSchema, } from './views.js'; +import { Registry } from './registry/service.js'; +import { SystemMessage, UserMessage } from '../llm/messages.js'; +import { createLogger } from '../logging-config.js'; +import { sanitize_surrogates } from '../utils.js'; +import { findUnsupportedJsonSchemaKeyword, normalizeStructuredDataBySchema, } from '../tools/extraction/schema-utils.js'; +import { getClickDescription } from '../tools/utils.js'; +const DEFAULT_WAIT_OFFSET = 1; +const MAX_WAIT_SECONDS = 30; +const toActionEntries = (action) => { + if (!action) { + return []; + } + return Object.entries(action).filter(([, params]) => params != null); +}; +const createAbortError = (reason) => { + if (reason instanceof Error) { + return reason; + } + const error = new Error('Operation aborted'); + error.name = 'AbortError'; + return error; +}; +const isAbortError = (error) => { + return error instanceof Error && error.name === 'AbortError'; +}; +const resolveUniqueOutputPath = async (directory, fileName) => { + const parsed = path.parse(fileName); + let candidate = path.join(directory, fileName); + let counter = 1; + while (fs.existsSync(candidate)) { + candidate = path.join(directory, `${parsed.name} (${counter})${parsed.ext}`); + counter += 1; + } + return candidate; +}; +const throwIfAborted = (signal) => { + if (signal?.aborted) { + throw createAbortError(signal.reason); + } +}; +const waitWithSignal = async (timeoutMs, signal) => { + if (timeoutMs <= 0) { + throwIfAborted(signal); + return; + } + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + resolve(); + }, timeoutMs); + const onAbort = () => { + clearTimeout(timeout); + cleanup(); + reject(createAbortError(signal?.reason)); + }; + const cleanup = () => { + signal?.removeEventListener('abort', onAbort); + }; + if (signal) { + if (signal.aborted) { + onAbort(); + return; + } + signal.addEventListener('abort', onAbort, { once: true }); + } + }); +}; +const dispatchBrowserEventIfAvailable = async (browser_session, event, fallback) => { + if (typeof browser_session?.dispatch_browser_event === 'function') { + const dispatchResult = await browser_session.dispatch_browser_event(event); + return dispatchResult?.event?.event_result ?? null; + } + return fallback(); +}; +const runWithTimeoutAndSignal = async (operation, timeoutMs, signal, timeoutMessage = 'Operation timed out') => { + throwIfAborted(signal); + if (timeoutMs <= 0) { + return operation(); + } + return await new Promise((resolve, reject) => { + let settled = false; + const onAbort = () => { + if (settled) { + return; + } + settled = true; + cleanup(); + reject(createAbortError(signal?.reason)); + }; + const onTimeout = () => { + if (settled) { + return; + } + settled = true; + cleanup(); + reject(new Error(timeoutMessage)); + }; + const cleanup = () => { + if (timeout) { + clearTimeout(timeout); + } + signal?.removeEventListener('abort', onAbort); + }; + const timeout = setTimeout(onTimeout, timeoutMs); + if (signal) { + if (signal.aborted) { + onAbort(); + return; + } + signal.addEventListener('abort', onAbort, { once: true }); + } + void operation() + .then((value) => { + if (settled) { + return; + } + settled = true; + cleanup(); + resolve(value); + }) + .catch((error) => { + if (settled) { + return; + } + settled = true; + cleanup(); + reject(error); + }); + }); +}; +const validateAndFixJavaScript = (code) => { + let fixedCode = code; + // Fix double-escaped quotes often produced in tool-argument JSON. + fixedCode = fixedCode.replace(/\\"/g, '"'); + // Fix over-escaped regex tokens (e.g. \\d -> \d). + fixedCode = fixedCode.replace(/\\\\([dDsSwWbBnrtfv])/g, '\\$1'); + fixedCode = fixedCode.replace(/\\\\([.*+?^${}()|[\]])/g, '\\$1'); + // Convert brittle mixed-quote selectors/XPaths into template literals. + fixedCode = fixedCode.replace(/document\.evaluate\s*\(\s*"([^"]*)"\s*,/g, (_match, xpath) => `document.evaluate(\`${xpath}\`,`); + fixedCode = fixedCode.replace(/(querySelector(?:All)?)\s*\(\s*"([^"]*)"\s*\)/g, (_match, methodName, selector) => `${methodName}(\`${selector}\`)`); + fixedCode = fixedCode.replace(/\.closest\s*\(\s*"([^"]*)"\s*\)/g, (_match, selector) => `.closest(\`${selector}\`)`); + fixedCode = fixedCode.replace(/\.matches\s*\(\s*"([^"]*)"\s*\)/g, (_match, selector) => `.matches(\`${selector}\`)`); + return fixedCode; +}; +export class Controller { + registry; + displayFilesInDoneText; + outputModel; + coordinateClickingEnabled; + clickActionHandler = null; + logger; + constructor(options = {}) { + const { exclude_actions = [], output_model = null, display_files_in_done_text = true, } = options; + this.registry = new Registry(exclude_actions); + this.displayFilesInDoneText = display_files_in_done_text; + this.outputModel = output_model; + this.coordinateClickingEnabled = false; + this.logger = createLogger('browser_use.controller'); + this.registerDefaultActions(output_model); + } + registerDefaultActions(outputModel) { + this.registerDoneAction(outputModel); + this.registerNavigationActions(); + this.registerElementActions(); + this.registerTabActions(); + this.registerContentActions(); + this.registerExplorationActions(); + this.registerScrollActions(); + this.registerFileSystemActions(); + this.registerUtilityActions(); + this.registerKeyboardActions(); + this.registerDropdownActions(); + this.registerSheetsActions(); + } + registerNavigationActions() { + this.registry.action('Search the query on a web search engine (duckduckgo, google, or bing).', { param_model: SearchActionSchema, terminates_sequence: true })(async function search(params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const requestedEngine = String(params.engine ?? 'duckduckgo'); + const engine = requestedEngine.toLowerCase(); + const encodedQuery = encodeURIComponent(params.query).replace(/%20/g, '+'); + const searchUrlByEngine = { + duckduckgo: `https://duckduckgo.com/?q=${encodedQuery}`, + google: `https://www.google.com/search?q=${encodedQuery}&udm=14`, + bing: `https://www.bing.com/search?q=${encodedQuery}`, + }; + const searchUrl = searchUrlByEngine[engine]; + if (!searchUrl) { + return new ActionResult({ + error: `Unsupported search engine: ${requestedEngine}. Options: duckduckgo, google, bing`, + }); + } + try { + await browser_session.navigate_to(searchUrl, { signal }); + const memory = `Searched ${requestedEngine} for '${params.query}'`; + return new ActionResult({ + extracted_content: memory, + long_term_memory: memory, + }); + } + catch (error) { + return new ActionResult({ + error: `Failed to search ${requestedEngine} for "${params.query}": ${String(error?.message ?? error)}`, + }); + } + }); + this.registry.action('Search the query in Google...', { + param_model: SearchGoogleActionSchema, + terminates_sequence: true, + })(async function search_google(params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(params.query)}&udm=14`; + const page = await browser_session.get_current_page(); + const currentUrl = page?.url().replace(/\/+$/, ''); + if (currentUrl === 'https://www.google.com') { + await browser_session.navigate_to(searchUrl, { signal }); + } + else { + await browser_session.create_new_tab(searchUrl, { signal }); + } + const msg = `🔍 Searched for "${params.query}" in Google`; + return new ActionResult({ + extracted_content: msg, + include_in_memory: true, + long_term_memory: `Searched Google for '${params.query}'`, + }); + }); + const navigateImpl = async function (params, { browser_session, signal, }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + try { + if (params.new_tab) { + await browser_session.create_new_tab(params.url, { signal }); + const tabIdx = browser_session.active_tab_index; + const msg = `🔗 Opened new tab #${tabIdx} with url ${params.url}`; + return new ActionResult({ + extracted_content: msg, + include_in_memory: true, + long_term_memory: `Opened new tab with URL ${params.url}`, + }); + } + await dispatchBrowserEventIfAvailable(browser_session, new NavigateToUrlEvent({ url: params.url, new_tab: false }), () => browser_session.navigate_to(params.url, { signal })); + const msg = `🔗 Navigated to ${params.url}`; + return new ActionResult({ + extracted_content: msg, + include_in_memory: true, + long_term_memory: `Navigated to ${params.url}`, + }); + } + catch (error) { + const errorMsg = String(error?.message ?? error ?? ''); + const networkFailures = [ + 'ERR_NAME_NOT_RESOLVED', + 'ERR_INTERNET_DISCONNECTED', + 'ERR_CONNECTION_REFUSED', + 'ERR_TIMED_OUT', + 'net::', + ]; + if (networkFailures.some((needle) => errorMsg.includes(needle))) { + return new ActionResult({ + error: `Navigation failed - site unavailable: ${params.url}`, + }); + } + if (error instanceof Error && + error.name === 'RuntimeError' && + errorMsg.includes('CDP client not initialized')) { + return new ActionResult({ + error: `Browser connection error: ${errorMsg}`, + }); + } + return new ActionResult({ + error: `Navigation failed: ${errorMsg}`, + }); + } + }; + this.registry.action('Navigate to URL...', { + param_model: GoToUrlActionSchema, + terminates_sequence: true, + })(async function go_to_url(params, { browser_session, signal }) { + return navigateImpl(params, { browser_session, signal }); + }); + this.registry.action('Navigate to URL...', { + param_model: GoToUrlActionSchema, + terminates_sequence: true, + })(async function navigate(params, { browser_session, signal }) { + return navigateImpl(params, { browser_session, signal }); + }); + this.registry.action('Go back', { + param_model: NoParamsActionSchema, + terminates_sequence: true, + })(async function go_back(_params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + try { + await dispatchBrowserEventIfAvailable(browser_session, new GoBackEvent(), () => browser_session.go_back({ signal })); + const memory = 'Navigated back'; + return new ActionResult({ extracted_content: memory }); + } + catch (error) { + return new ActionResult({ + error: `Failed to go back: ${String(error?.message ?? error)}`, + }); + } + }); + this.registry.action('Wait for x seconds.', { + param_model: WaitActionSchema, + })(async function wait(params, { signal, browser_session }) { + const seconds = params.seconds ?? 3; + const actualSeconds = Math.min(Math.max(seconds - DEFAULT_WAIT_OFFSET, 0), MAX_WAIT_SECONDS); + const msg = `🕒 Waited for ${seconds} second${seconds === 1 ? '' : 's'}`; + if (actualSeconds > 0) { + if (browser_session) { + await dispatchBrowserEventIfAvailable(browser_session, new WaitEvent({ + seconds: actualSeconds, + max_seconds: MAX_WAIT_SECONDS, + }), () => waitWithSignal(actualSeconds * 1000, signal)); + } + else { + await waitWithSignal(actualSeconds * 1000, signal); + } + } + return new ActionResult({ + extracted_content: msg, + long_term_memory: `Waited for ${seconds} second${seconds === 1 ? '' : 's'}`, + }); + }); + } + registerElementActions() { + const logger = this.logger; + const convertLlmCoordinatesToViewport = (llmX, llmY, browserSession) => { + const llmSize = browserSession?.llm_screenshot_size; + const viewportSize = browserSession?._original_viewport_size; + if (!Array.isArray(llmSize) || + llmSize.length !== 2 || + !Array.isArray(viewportSize) || + viewportSize.length !== 2) { + return [llmX, llmY]; + } + const [llmWidth, llmHeight] = llmSize.map((value) => Number(value)); + const [viewportWidth, viewportHeight] = viewportSize.map((value) => Number(value)); + if (!Number.isFinite(llmWidth) || + !Number.isFinite(llmHeight) || + !Number.isFinite(viewportWidth) || + !Number.isFinite(viewportHeight) || + llmWidth <= 0 || + llmHeight <= 0 || + viewportWidth <= 0 || + viewportHeight <= 0) { + return [llmX, llmY]; + } + const actualX = Math.floor((llmX / llmWidth) * viewportWidth); + const actualY = Math.floor((llmY / llmHeight) * viewportHeight); + logger.info(`🔄 Converting coordinates: LLM (${llmX}, ${llmY}) @ ${llmWidth}x${llmHeight} -> Viewport (${actualX}, ${actualY}) @ ${viewportWidth}x${viewportHeight}`); + return [actualX, actualY]; + }; + const clickImpl = async (params, { browser_session, signal }) => { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const collectTabIds = () => { + if (!Array.isArray(browser_session.tabs)) { + return new Set(); + } + return new Set(browser_session.tabs + .map((tab) => tab?.page_id) + .filter((pageId) => typeof pageId === 'number' && Number.isFinite(pageId))); + }; + const detectNewTabNote = async (tabsBefore) => { + try { + await waitWithSignal(50, signal); + const tabsAfter = Array.isArray(browser_session.tabs) + ? browser_session.tabs + : []; + const newTab = tabsAfter.find((tab) => { + const pageId = tab?.page_id; + return typeof pageId === 'number' && !tabsBefore.has(pageId); + }); + if (!newTab) { + return ''; + } + const tabId = typeof newTab?.tab_id === 'string' && newTab.tab_id.trim() + ? newTab.tab_id.trim() + : String(newTab.page_id).padStart(4, '0').slice(-4); + return `. Note: This opened a new tab (tab_id: ${tabId}) - switch to it if you need to interact with the new page.`; + } + catch { + return ''; + } + }; + if (params.coordinate_x != null && + params.coordinate_y != null && + params.index == null) { + if (!this.coordinateClickingEnabled) { + throw new BrowserError('Coordinate clicking is disabled for the current model. Provide an element index.'); + } + const tabsBefore = collectTabIds(); + const page = await browser_session.get_current_page(); + if (!page?.mouse?.click) { + throw new BrowserError('Unable to perform coordinate click on the current page.'); + } + const [actualX, actualY] = convertLlmCoordinatesToViewport(params.coordinate_x, params.coordinate_y, browser_session); + await dispatchBrowserEventIfAvailable(browser_session, new ClickCoordinateEvent({ + coordinate_x: actualX, + coordinate_y: actualY, + }), () => page.mouse.click(actualX, actualY)); + const coordinateMessage = `🖱️ Clicked at coordinates (${params.coordinate_x}, ${params.coordinate_y})` + + (await detectNewTabNote(tabsBefore)); + return new ActionResult({ + extracted_content: coordinateMessage, + include_in_memory: true, + long_term_memory: coordinateMessage, + metadata: { + click_x: actualX, + click_y: actualY, + }, + }); + } + if (params.index == null) { + return new ActionResult({ + error: 'Must provide either index or both coordinate_x and coordinate_y', + }); + } + const element = await browser_session.get_dom_element_by_index(params.index, { + signal, + }); + if (!element) { + const msg = `Element index ${params.index} not available - page may have changed. Try refreshing browser state.`; + logger.warning(`⚠️ ${msg}`); + return new ActionResult({ + extracted_content: msg, + }); + } + const tabsBefore = collectTabIds(); + if (browser_session.is_file_input?.(element)) { + const msg = `Index ${params.index} - has an element which opens file upload dialog.`; + return new ActionResult({ + extracted_content: msg, + include_in_memory: true, + success: false, + long_term_memory: msg, + }); + } + const downloadPath = await dispatchBrowserEventIfAvailable(browser_session, new ClickElementEvent({ + node: element, + button: 'left', + }), () => browser_session._click_element_node(element, { + signal, + })); + let msg; + if (downloadPath) { + msg = `💾 Downloaded file to ${downloadPath}`; + } + else { + let elementDescription = ''; + if (typeof element?.tag_name === 'string' && + typeof element?.get_all_text_till_next_clickable_element === + 'function') { + try { + elementDescription = getClickDescription(element); + } + catch { + elementDescription = ''; + } + } + if (elementDescription) { + msg = `🖱️ Clicked ${elementDescription}`; + } + else { + const snippet = element.get_all_text_till_next_clickable_element?.(2) ?? ''; + msg = `🖱️ Clicked button with index ${params.index}: ${snippet}`; + } + } + msg += await detectNewTabNote(tabsBefore); + return new ActionResult({ + extracted_content: msg, + include_in_memory: true, + long_term_memory: msg, + }); + }; + this.clickActionHandler = clickImpl; + this.registerClickActions(); + const detectSensitiveKeyName = (value, sensitiveData) => { + if (!value || !sensitiveData) { + return null; + } + for (const [domainOrKey, content] of Object.entries(sensitiveData)) { + if (typeof content === 'string') { + if (content === value) { + return domainOrKey; + } + continue; + } + if (!content || typeof content !== 'object') { + continue; + } + for (const [key, nestedValue] of Object.entries(content)) { + if (nestedValue === value) { + return key; + } + } + } + return null; + }; + const inputImpl = async function (params, { browser_session, has_sensitive_data, sensitive_data, signal, }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const element = await browser_session.get_dom_element_by_index(params.index, { signal }); + if (!element) { + const msg = `Element index ${params.index} not available - page may have changed. Try refreshing browser state.`; + logger.warning(`⚠️ ${msg}`); + return new ActionResult({ + extracted_content: msg, + }); + } + const isAutocompleteField = (node) => { + const attrs = node?.attributes ?? {}; + const role = String(attrs.role ?? '').toLowerCase(); + const ariaAutocomplete = String(attrs['aria-autocomplete'] ?? '').toLowerCase(); + const hasDatalist = String(attrs.list ?? '').trim().length > 0; + return (role === 'combobox' || + (ariaAutocomplete !== '' && ariaAutocomplete !== 'none') || + hasDatalist); + }; + const needsAutocompleteDelay = (node) => { + const attrs = node?.attributes ?? {}; + const role = String(attrs.role ?? '').toLowerCase(); + const ariaAutocomplete = String(attrs['aria-autocomplete'] ?? '').toLowerCase(); + return (role === 'combobox' || + (ariaAutocomplete !== '' && ariaAutocomplete !== 'none')); + }; + await dispatchBrowserEventIfAvailable(browser_session, new TypeTextEvent({ + node: element, + text: params.text, + clear: params.clear ?? true, + }), () => browser_session._input_text_element_node(element, params.text, { + clear: params.clear, + signal, + })); + let actualValue = null; + try { + const locator = await browser_session.get_locate_element?.(element); + if (locator && typeof locator.inputValue === 'function') { + const value = await locator.inputValue(); + actualValue = typeof value === 'string' ? value : null; + } + } + catch { + actualValue = null; + } + let msg = `⌨️ Input ${params.text} into index ${params.index}`; + if (has_sensitive_data) { + const sensitiveKeyName = detectSensitiveKeyName(params.text, sensitive_data ?? null); + msg = sensitiveKeyName + ? `Typed ${sensitiveKeyName}` + : 'Typed sensitive data'; + } + if (!has_sensitive_data && + actualValue != null && + actualValue !== params.text) { + msg += + `\n⚠️ Note: the field's actual value '${actualValue}' differs from typed text '${params.text}'. ` + + 'The page may have reformatted or autocompleted your input.'; + } + if (isAutocompleteField(element)) { + msg += + '\n💡 This is an autocomplete field. Wait for suggestions to appear, then click the correct suggestion instead of pressing Enter.'; + if (needsAutocompleteDelay(element)) { + await waitWithSignal(400, signal); + } + } + return new ActionResult({ + extracted_content: msg, + include_in_memory: true, + long_term_memory: msg, + }); + }; + this.registry.action('Click and input text into an input interactive element', { param_model: InputTextActionSchema })(async function input_text(params, { browser_session, has_sensitive_data, sensitive_data, signal }) { + return inputImpl(params, { + browser_session, + has_sensitive_data, + sensitive_data, + signal, + }); + }); + this.registry.action('Click and input text into an input interactive element', { param_model: InputTextActionSchema })(async function input(params, { browser_session, has_sensitive_data, sensitive_data, signal }) { + return inputImpl(params, { + browser_session, + has_sensitive_data, + sensitive_data, + signal, + }); + }); + this.registry.action('Upload file to interactive element with file path', { + param_model: UploadFileActionSchema, + })(async function upload_file(params, { browser_session, available_file_paths, file_system, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + let uploadPath = params.path; + const isLocalBrowser = browser_session?.is_local !== false; + const allowedPaths = new Set(available_file_paths ?? []); + const downloadedFiles = Array.isArray(browser_session?.downloaded_files) + ? browser_session.downloaded_files + : []; + for (const downloadedPath of downloadedFiles) { + allowedPaths.add(downloadedPath); + } + if (!allowedPaths.has(uploadPath)) { + const fsInstance = file_system ?? null; + const managedFile = fsInstance && typeof fsInstance.get_file === 'function' + ? fsInstance.get_file(uploadPath) + : null; + if (managedFile && fsInstance?.get_dir) { + uploadPath = path.join(fsInstance.get_dir(), uploadPath); + } + else if (!isLocalBrowser) { + // Remote browser paths may only exist on the remote runtime. + } + else { + return new ActionResult({ + error: `File path ${params.path} is not available. To fix: add this file path to available_file_paths when creating the Agent.`, + }); + } + } + if (isLocalBrowser) { + if (!fs.existsSync(uploadPath)) { + return new ActionResult({ + error: `File ${uploadPath} does not exist`, + }); + } + if (fs.statSync(uploadPath).size === 0) { + return new ActionResult({ + error: `File ${uploadPath} is empty (0 bytes). The file may not have been saved correctly.`, + }); + } + } + let selectorMap = null; + if (typeof browser_session.get_selector_map === 'function') { + selectorMap = await browser_session.get_selector_map({ signal }); + if (!(params.index in (selectorMap ?? {}))) { + return new ActionResult({ + error: `Element with index ${params.index} does not exist.`, + }); + } + } + let node = await browser_session.find_file_upload_element_by_index(params.index, 3, 3, { signal }); + if (!node && + selectorMap && + typeof browser_session.is_file_input === 'function') { + let currentScrollY = 0; + try { + const page = await browser_session.get_current_page?.(); + if (page?.evaluate) { + const evaluated = await page.evaluate(() => window.scrollY || window.pageYOffset || 0); + const numeric = typeof evaluated === 'number' ? evaluated : Number(evaluated); + if (Number.isFinite(numeric)) { + currentScrollY = numeric; + } + } + } + catch { + currentScrollY = 0; + } + let closest = null; + let minDistance = Number.POSITIVE_INFINITY; + for (const element of Object.values(selectorMap)) { + if (!browser_session.is_file_input(element)) { + continue; + } + const y = Number(element?.absolute_position?.y ?? 0); + const distance = Number.isFinite(y) + ? Math.abs(y - currentScrollY) + : 0; + if (!closest || distance < minDistance) { + closest = element; + minDistance = distance; + } + } + if (closest) { + node = closest; + } + } + if (!node) { + throw new BrowserError('No file upload element found on the page'); + } + await dispatchBrowserEventIfAvailable(browser_session, new UploadFileEvent({ + node, + file_path: uploadPath, + }), async () => { + const locator = await browser_session.get_locate_element(node); + if (!locator) { + throw new BrowserError('No file upload element found on the page'); + } + await locator.setInputFiles(uploadPath); + return null; + }); + const msg = `📁 Successfully uploaded file to index ${params.index}`; + return new ActionResult({ + extracted_content: msg, + include_in_memory: true, + long_term_memory: `Uploaded file ${uploadPath} to element ${params.index}`, + }); + }); + } + registerClickActions() { + const clickActionHandler = this.clickActionHandler; + if (!clickActionHandler) { + return; + } + const removeAction = this.registry?.remove_action; + if (typeof removeAction === 'function') { + removeAction.call(this.registry, 'click'); + removeAction.call(this.registry, 'click_element_by_index'); + } + const registerIndexAlias = () => { + this.registry.action('Click element by index.', { + param_model: ClickElementActionIndexOnlySchema, + action_name: 'click_element_by_index', + })(async (params, ctx) => { + return await clickActionHandler(params, ctx); + }); + }; + if (this.coordinateClickingEnabled) { + this.registry.action('Click element by index or coordinates. Use coordinates only if the index is not available. Either provide coordinates or index.', { + param_model: ClickElementActionSchema, + action_name: 'click', + })(async (params, ctx) => { + return await clickActionHandler(params, ctx); + }); + registerIndexAlias(); + return; + } + this.registry.action('Click element by index.', { + param_model: ClickElementActionIndexOnlySchema, + action_name: 'click', + })(async (params, ctx) => { + return await clickActionHandler(params, ctx); + }); + registerIndexAlias(); + } + registerTabActions() { + const tabLogger = this.logger; + const resolveTabIdentifier = (params) => { + if (typeof params.tab_id === 'string' && params.tab_id.trim()) { + return params.tab_id.trim(); + } + if (typeof params.page_id === 'number' && + Number.isFinite(params.page_id)) { + return params.page_id; + } + return -1; + }; + const formatTabId = (identifier, browser_session) => { + if (typeof identifier === 'string' && identifier.trim()) { + return identifier.trim(); + } + const numericIdentifier = typeof identifier === 'number' && Number.isFinite(identifier) + ? Math.floor(identifier) + : -1; + if (numericIdentifier >= 0) { + const matchedTab = Array.isArray(browser_session?.tabs) + ? browser_session.tabs.find((tab) => tab?.page_id === numericIdentifier) + : null; + const matchedTabId = typeof matchedTab?.tab_id === 'string' && matchedTab.tab_id.trim() + ? matchedTab.tab_id.trim() + : null; + return (matchedTabId ?? String(numericIdentifier).padStart(4, '0').slice(-4)); + } + return 'unknown'; + }; + const switchImpl = async function (params, { browser_session, signal, }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const identifier = resolveTabIdentifier(params); + const tabId = formatTabId(identifier, browser_session); + try { + const switchTargetId = identifier === -1 ? null : String(identifier).trim(); + await dispatchBrowserEventIfAvailable(browser_session, new SwitchTabEvent({ target_id: switchTargetId }), () => browser_session.switch_to_tab(identifier, { signal })); + const page = await browser_session.get_current_page(); + try { + await page?.wait_for_load_state?.('domcontentloaded', { + timeout: 5000, + }); + } + catch { + /* ignore */ + } + const memory = `Switched to tab #${tabId}`; + return new ActionResult({ + extracted_content: memory, + long_term_memory: memory, + }); + } + catch (error) { + tabLogger.warning(`Tab switch may have failed: ${error.message}`); + const memory = `Attempted to switch to tab #${tabId}`; + return new ActionResult({ + extracted_content: memory, + long_term_memory: memory, + }); + } + }; + this.registry.action('Switch tab', { + param_model: SwitchTabActionSchema, + terminates_sequence: true, + })(async function switch_tab(params, { browser_session, signal }) { + return switchImpl(params, { browser_session, signal }); + }); + this.registry.action('Switch tab', { + param_model: SwitchTabActionSchema, + terminates_sequence: true, + action_name: 'switch', + })(async function switch_alias(params, { browser_session, signal }) { + return switchImpl(params, { browser_session, signal }); + }); + const closeImpl = async function (params, { browser_session, signal, }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const identifier = resolveTabIdentifier(params); + const closedTabId = formatTabId(identifier, browser_session); + try { + const resolvedCloseTargetId = identifier === -1 + ? (browser_session?.active_tab?.target_id ?? + browser_session?.active_tab?.tab_id ?? + null) + : String(identifier).trim(); + if (!resolvedCloseTargetId) { + throw new Error('Could not resolve target tab to close'); + } + await dispatchBrowserEventIfAvailable(browser_session, new CloseTabEvent({ target_id: resolvedCloseTargetId }), () => browser_session.close_tab(identifier)); + const memory = `Closed tab #${closedTabId}`; + return new ActionResult({ + extracted_content: memory, + long_term_memory: memory, + }); + } + catch (error) { + tabLogger.warning(`Tab ${closedTabId} may already be closed: ${error.message}`); + const memory = `Tab #${closedTabId} closed (was already closed or invalid)`; + return new ActionResult({ + extracted_content: memory, + long_term_memory: memory, + }); + } + }; + this.registry.action('Close an existing tab', { + param_model: CloseTabActionSchema, + })(async function close_tab(params, { browser_session, signal }) { + return closeImpl(params, { browser_session, signal }); + }); + this.registry.action('Close an existing tab', { + param_model: CloseTabActionSchema, + })(async function close(params, { browser_session, signal }) { + return closeImpl(params, { browser_session, signal }); + }); + } + registerContentActions() { + const registry = this.registry; + const contentLogger = this.logger; + const extractStructuredDescription = "LLM extracts structured data from page markdown. Use when: on right page, know what to extract, haven't called before on same page+query. Can't get interactive elements. Set extract_links=True for URLs. Use start_from_char if previous extraction was truncated to extract data further down the page. When paginating across pages, pass already_collected with item identifiers (names/URLs) from prior pages to avoid duplicates."; + this.registry.action(extractStructuredDescription, { + param_model: ExtractStructuredDataActionSchema, + })(async function extract_structured_data(params, { page, page_extraction_llm, extraction_schema, file_system, signal }) { + throwIfAborted(signal); + if (!page) { + throw new BrowserError('No active page available for extraction.'); + } + if (!page_extraction_llm) { + throw new BrowserError('page_extraction_llm is not configured.'); + } + const fsInstance = file_system ?? new FileSystem(process.cwd(), false); + const pageHtml = await runWithTimeoutAndSignal(async () => { + const value = await page.content?.(); + return typeof value === 'string' ? value : ''; + }, 10000, signal, 'Page content extraction timed out'); + if (!pageHtml) { + throw new BrowserError('Unable to extract page content.'); + } + let combinedHtml = pageHtml; + const frames = typeof page.frames === 'function' + ? page.frames() + : Array.isArray(page.frames) + ? page.frames + : []; + const currentUrl = (() => { + const pageUrlValue = page.url; + if (typeof pageUrlValue === 'function') { + return String(pageUrlValue.call(page) ?? ''); + } + return typeof pageUrlValue === 'string' ? pageUrlValue : ''; + })(); + for (const iframe of frames) { + throwIfAborted(signal); + try { + await runWithTimeoutAndSignal(async () => { + await iframe.waitForLoadState?.('load'); + }, 1000, signal, 'Iframe load timeout'); + } + catch (error) { + if (isAbortError(error)) { + throw error; + } + } + const iframeUrl = typeof iframe.url === 'function' + ? iframe.url() + : typeof iframe.url === 'string' + ? iframe.url + : ''; + if (!iframeUrl || + iframeUrl === currentUrl || + iframeUrl.startsWith('data:') || + iframeUrl.startsWith('about:')) { + continue; + } + try { + const iframeHtml = await runWithTimeoutAndSignal(async () => { + const value = await iframe.content?.(); + return typeof value === 'string' ? value : ''; + }, 2000, signal, 'Iframe content extraction timeout'); + if (!iframeHtml) { + continue; + } + combinedHtml += `\n

IFRAME ${iframeUrl}

${iframeHtml}
`; + } + catch (error) { + if (isAbortError(error)) { + throw error; + } + } + } + const extracted = extractCleanMarkdownFromHtml(combinedHtml, { + extract_links: params.extract_links, + method: 'page_content', + url: currentUrl || undefined, + }); + let content = extracted.content; + const contentStats = extracted.stats; + const finalFilteredLength = contentStats.final_filtered_chars; + const startFromChar = Math.max(0, params.start_from_char ?? 0); + const maxChars = 100000; + const chunks = chunkMarkdownByStructure(content, maxChars, 5, startFromChar); + if (!chunks.length) { + return new ActionResult({ + error: `start_from_char (${startFromChar}) exceeds content length ${finalFilteredLength} characters.`, + }); + } + const chunk = chunks[0]; + content = chunk.content; + const wasTruncated = chunk.has_more; + if (chunk.overlap_prefix) { + content = `${chunk.overlap_prefix}\n${content}`; + } + if (startFromChar > 0) { + contentStats.started_from_char = startFromChar; + } + if (wasTruncated) { + contentStats.truncated_at_char = chunk.char_offset_end; + contentStats.next_start_char = chunk.char_offset_end; + contentStats.chunk_index = chunk.chunk_index; + contentStats.total_chunks = chunk.total_chunks; + } + const originalHtmlLength = contentStats.original_html_chars; + const initialMarkdownLength = contentStats.initial_markdown_chars; + const charsFiltered = contentStats.filtered_chars_removed; + let statsSummary = `Content processed: ${originalHtmlLength.toLocaleString()} HTML chars ` + + `→ ${initialMarkdownLength.toLocaleString()} initial markdown ` + + `→ ${finalFilteredLength.toLocaleString()} filtered markdown`; + if (startFromChar > 0) { + statsSummary += ` (started from char ${startFromChar.toLocaleString()})`; + } + if (wasTruncated && + contentStats.next_start_char != null && + contentStats.chunk_index != null && + contentStats.total_chunks != null) { + const chunkInfo = `chunk ${contentStats.chunk_index + 1} of ${contentStats.total_chunks}, `; + statsSummary += + ` → ${content.length.toLocaleString()} final chars ` + + `(${chunkInfo}use start_from_char=${contentStats.next_start_char} to continue)`; + } + else if (charsFiltered > 0) { + statsSummary += ` (filtered ${charsFiltered.toLocaleString()} chars of noise)`; + } + content = sanitize_surrogates(content); + const sanitizedQuery = sanitize_surrogates(params.query); + const alreadyCollected = Array.isArray(params.already_collected) + ? params.already_collected + .map((item) => sanitize_surrogates(String(item)).trim()) + .filter(Boolean) + : []; + const parseJsonFromCompletion = (completion) => { + const trimmed = completion.trim(); + const fencedMatch = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/i); + const candidate = fencedMatch?.[1]?.trim() || trimmed; + return JSON.parse(candidate); + }; + let effectiveOutputSchema = params.output_schema ?? extraction_schema; + if (effectiveOutputSchema != null) { + const unsupportedKeyword = findUnsupportedJsonSchemaKeyword(effectiveOutputSchema); + if (unsupportedKeyword) { + contentLogger.warning(`Invalid output_schema, falling back to free-text extraction: unsupported keyword '${unsupportedKeyword}'`); + effectiveOutputSchema = null; + } + } + const pageUrl = currentUrl || ''; + const maxMemoryLength = 10000; + if (effectiveOutputSchema != null) { + const systemPrompt = ` +You are an expert at extracting structured data from the markdown of a webpage. + + +You will be given a query, a JSON Schema, and the markdown of a webpage that has been filtered to remove noise and advertising content. + + + +- Extract ONLY information present in the webpage. Do not guess or fabricate values. +- Your response MUST conform to the provided JSON Schema exactly. +- If a required field's value cannot be found on the page, use null (if the schema allows it) or an empty string / empty array as appropriate. +- If the content was truncated, extract what is available from the visible portion. +- If items are provided, skip any items whose name/title/URL matches those listed. Do not include duplicates. +`.trim(); + const schemaJson = JSON.stringify(effectiveOutputSchema, null, 2); + const alreadyCollectedSection = alreadyCollected.length > 0 + ? `\n\n\nSkip items whose name/title/URL matches any of these already-collected identifiers:\n${alreadyCollected + .slice(0, 100) + .map((item) => `- ${item}`) + .join('\n')}\n` + : ''; + const prompt = `\n${sanitizedQuery}\n\n\n` + + `\n${schemaJson}\n\n\n` + + `\n${statsSummary}\n\n\n` + + `\n${content}\n` + + alreadyCollectedSection; + const response = await page_extraction_llm.ainvoke([new SystemMessage(systemPrompt), new UserMessage(prompt)], undefined, { signal: signal ?? undefined }); + throwIfAborted(signal); + const completion = response?.completion; + const completionText = typeof completion === 'string' + ? completion + : JSON.stringify(completion ?? {}); + let parsedResult; + try { + parsedResult = parseJsonFromCompletion(completionText); + } + catch (error) { + throw new BrowserError(`Structured extraction returned invalid JSON: ${error.message}`); + } + const schemaValidation = validateJsonSchema(parsedResult, effectiveOutputSchema); + if (!schemaValidation.valid) { + const details = (schemaValidation.errors ?? []) + .slice(0, 3) + .map((item) => String(item?.error ?? '').trim()) + .filter(Boolean) + .join('; '); + const suffix = details ? `: ${details}` : ''; + throw new BrowserError(`Structured extraction result does not match output_schema${suffix}`); + } + const normalizedResult = normalizeStructuredDataBySchema(parsedResult, effectiveOutputSchema); + const resultJson = JSON.stringify(normalizedResult); + const extractedContent = `\n${pageUrl}\n\n` + + `\n${sanitizedQuery}\n\n` + + `\n${resultJson}\n`; + const extractionMeta = { + data: normalizedResult, + schema_used: effectiveOutputSchema, + is_partial: wasTruncated, + source_url: pageUrl, + content_stats: contentStats, + }; + const includeOnce = extractedContent.length >= maxMemoryLength; + const memory = includeOnce + ? `Query: ${sanitizedQuery}\nContent in ${await fsInstance.save_extracted_content(extractedContent)} and once in .` + : extractedContent; + return new ActionResult({ + extracted_content: extractedContent, + include_extracted_content_only_once: includeOnce, + long_term_memory: memory, + metadata: { + structured_extraction: true, + extraction_result: extractionMeta, + }, + }); + } + const systemPrompt = ` +You are an expert at extracting data from the markdown of a webpage. + + +You will be given a query and the markdown of a webpage that has been filtered to remove noise and advertising content. + + + +- You are tasked to extract information from the webpage that is relevant to the query. +- You should ONLY use the information available in the webpage to answer the query. Do not make up information or provide guess from your own knowledge. +- If the information relevant to the query is not available in the page, your response should mention that. +- If the query asks for all items, products, etc., make sure to directly list all of them. +- If the content was truncated and you need more information, note that the user can use start_from_char parameter to continue from where truncation occurred. +- If items are provided, exclude any results whose name/title/URL matches those already collected. Do not include duplicates. + + + +- Your output should present ALL the information relevant to the query in a concise way. +- Do not answer in conversational format - directly output the relevant information or that the information is unavailable. +`.trim(); + const prompt = `\n${sanitizedQuery}\n\n\n` + + `\n${statsSummary}\n\n\n` + + `\n${content}\n` + + (alreadyCollected.length > 0 + ? `\n\n\nSkip items whose name/title/URL matches any of these already-collected identifiers:\n${alreadyCollected + .slice(0, 100) + .map((item) => `- ${item}`) + .join('\n')}\n` + : ''); + const response = await page_extraction_llm.ainvoke([new SystemMessage(systemPrompt), new UserMessage(prompt)], undefined, { signal: signal ?? undefined }); + throwIfAborted(signal); + const completion = response?.completion; + const completionText = typeof completion === 'string' + ? completion + : JSON.stringify(completion ?? {}); + const extractedContent = `\n${pageUrl}\n\n` + + `\n${sanitizedQuery}\n\n` + + `\n${completionText}\n`; + const includeOnce = extractedContent.length >= maxMemoryLength; + const memory = includeOnce + ? `Query: ${sanitizedQuery}\nContent in ${await fsInstance.save_extracted_content(extractedContent)} and once in .` + : extractedContent; + return new ActionResult({ + extracted_content: extractedContent, + include_extracted_content_only_once: includeOnce, + long_term_memory: memory, + }); + }); + this.registry.action(extractStructuredDescription, { + param_model: ExtractStructuredDataActionSchema, + action_name: 'extract', + })(async function extract(params, { browser_session, page_extraction_llm, extraction_schema, file_system, available_file_paths, sensitive_data, signal, }) { + return registry.execute_action('extract_structured_data', params, { + browser_session, + page_extraction_llm, + extraction_schema, + file_system, + available_file_paths, + sensitive_data, + signal, + }); + }); + } + registerExplorationActions() { + this.registry.action('Search page text for a pattern (like grep). Zero LLM cost and instant.', { param_model: SearchPageActionSchema })(async function search_page(params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const page = await browser_session.get_current_page(); + if (!page?.evaluate) { + throw new BrowserError('No active page for search_page.'); + } + const searchResult = (await page.evaluate(({ pattern, regex, caseSensitive, contextChars, cssScope, maxResults, }) => { + const sourceNode = cssScope + ? document.querySelector(cssScope) + : document.body; + if (!sourceNode) { + return { + error: `CSS scope not found: ${cssScope}`, + matches: [], + total: 0, + }; + } + const sourceText = sourceNode.innerText || + sourceNode.textContent || + ''; + if (!sourceText.trim()) { + return { + matches: [], + total: 0, + }; + } + const safePattern = regex + ? pattern + : pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const flags = caseSensitive ? 'g' : 'gi'; + let matcher; + try { + matcher = new RegExp(safePattern, flags); + } + catch (error) { + return { + error: `Invalid regex pattern: ${String(error)}`, + matches: [], + total: 0, + }; + } + const matches = []; + let foundTotal = 0; + let m; + while ((m = matcher.exec(sourceText)) !== null) { + foundTotal += 1; + if (matches.length < Math.max(1, maxResults)) { + const start = Math.max(0, m.index - Math.max(0, contextChars)); + const end = Math.min(sourceText.length, m.index + m[0].length + Math.max(0, contextChars)); + matches.push({ + position: m.index, + match: m[0], + snippet: sourceText.slice(start, end), + }); + } + if (m[0].length === 0) { + matcher.lastIndex += 1; + } + } + return { + matches, + total: foundTotal, + truncated: foundTotal > matches.length, + }; + }, { + pattern: params.pattern, + regex: params.regex, + caseSensitive: params.case_sensitive, + contextChars: params.context_chars, + cssScope: params.css_scope ?? null, + maxResults: params.max_results, + })); + if (!searchResult) { + return new ActionResult({ error: 'search_page returned no result' }); + } + if (searchResult.error) { + return new ActionResult({ + error: `search_page: ${searchResult.error}`, + }); + } + const total = searchResult.total ?? 0; + const matches = searchResult.matches ?? []; + if (total === 0 || !matches.length) { + const noMatchMessage = `No matches found for "${params.pattern}".`; + return new ActionResult({ + extracted_content: noMatchMessage, + long_term_memory: `Searched page for "${params.pattern}": 0 matches found.`, + }); + } + const lines = [ + `Found ${total} matches for "${params.pattern}" in page text:`, + ]; + for (let i = 0; i < matches.length; i += 1) { + const match = matches[i]; + const compactSnippet = match.snippet.replace(/\s+/g, ' ').trim(); + lines.push(`${i + 1}. [pos ${match.position}] "${match.match}" -> ${compactSnippet}`); + } + if (searchResult.truncated) { + lines.push(`... showing first ${matches.length} matches (increase max_results to see more).`); + } + const memory = `Searched page for "${params.pattern}": ${total} match${total === 1 ? '' : 'es'} found.`; + return new ActionResult({ + extracted_content: lines.join('\n'), + long_term_memory: memory, + }); + }); + this.registry.action('Query DOM elements by CSS selector (like find). Zero LLM cost and instant.', { param_model: FindElementsActionSchema })(async function find_elements(params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const page = await browser_session.get_current_page(); + if (!page?.evaluate) { + throw new BrowserError('No active page for find_elements.'); + } + const result = (await page.evaluate(({ selector, attributes, maxResults, includeText, }) => { + let elements; + try { + elements = Array.from(document.querySelectorAll(selector)); + } + catch (error) { + return { + error: `Invalid selector: ${String(error)}`, + elements: [], + total: 0, + }; + } + const selected = elements.slice(0, Math.max(1, maxResults)); + const payload = selected.map((el, idx) => { + const attrs = {}; + if (attributes?.length) { + for (const attr of attributes) { + const value = el.getAttribute(attr); + if (value != null) { + attrs[attr] = value; + } + } + } + return { + index: idx + 1, + tag: el.tagName.toLowerCase(), + text: includeText + ? (el.textContent || '').replace(/\s+/g, ' ').trim() + : '', + attributes: attrs, + }; + }); + return { + elements: payload, + total: elements.length, + truncated: elements.length > selected.length, + }; + }, { + selector: params.selector, + attributes: params.attributes ?? null, + maxResults: params.max_results, + includeText: params.include_text, + })); + if (!result) { + return new ActionResult({ error: 'find_elements returned no result' }); + } + if (result.error) { + return new ActionResult({ error: `find_elements: ${result.error}` }); + } + const elements = result.elements ?? []; + const total = result.total ?? 0; + if (!elements.length) { + const msg = `No elements found for selector "${params.selector}".`; + return new ActionResult({ + extracted_content: msg, + long_term_memory: msg, + }); + } + const lines = [ + `Found ${total} element${total === 1 ? '' : 's'} for selector "${params.selector}":`, + ]; + for (const el of elements) { + const attrs = Object.entries(el.attributes || {}) + .map(([k, v]) => `${k}=${JSON.stringify(v)}`) + .join(' '); + const text = params.include_text && el.text + ? ` text=${JSON.stringify(el.text)}` + : ''; + lines.push(`${el.index}. <${el.tag}>${text}${attrs ? ` ${attrs}` : ''}`.trim()); + } + if (result.truncated) { + lines.push(`... showing first ${elements.length} elements (increase max_results to see more).`); + } + return new ActionResult({ + extracted_content: lines.join('\n'), + long_term_memory: `Queried selector "${params.selector}" and found ${total} element${total === 1 ? '' : 's'}.`, + }); + }); + } + registerScrollActions() { + const registry = this.registry; + const scrollLogger = this.logger; // Capture logger reference for use in named function + // Define the scroll handler implementation (shared by multiple action names for LLM compatibility) + const scrollImpl = async (params, { browser_session, signal }) => { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const page = await browser_session.get_current_page(); + if (!page || !page.evaluate) { + throw new BrowserError('Unable to access current page for scrolling.'); + } + // Helper function to get window height with retries + const getWindowHeight = async (retries = 3) => { + for (let i = 0; i < retries; i++) { + throwIfAborted(signal); + try { + const height = await page.evaluate(() => window.innerHeight); + return height || 0; + } + catch (error) { + if (i === retries - 1) { + throw new Error(`Scroll failed due to an error: ${error}`, { + cause: error, + }); + } + await waitWithSignal(1000, signal); + } + } + return 0; + }; + const windowHeight = await getWindowHeight(); + const pagesScrolled = params.pages ?? params.num_pages ?? 1; + const scrollAmount = Math.floor(windowHeight * pagesScrolled); + const dy = params.down ? scrollAmount : -scrollAmount; + const direction = params.down ? 'down' : 'up'; + let scrollTarget = 'the page'; + // Element-specific scrolling if index is provided + if (params.index !== undefined && params.index !== null) { + try { + const elementNode = await browser_session.get_dom_element_by_index(params.index, { signal }); + if (!elementNode) { + return new ActionResult({ + error: `Element index ${params.index} not found in browser state`, + }); + } + // Try direct container scrolling (no events that might close dropdowns) + const containerScrollJs = ` + (params) => { + const { dy, elementXPath } = params; + + // Get the target element by XPath + const targetElement = document.evaluate(elementXPath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + if (!targetElement) { + return { success: false, reason: 'Element not found by XPath' }; + } + + // Try to find scrollable containers in the hierarchy (starting from element itself) + let currentElement = targetElement; + let scrollSuccess = false; + let scrolledElement = null; + let scrollDelta = 0; + let attempts = 0; + + // Check up to 10 elements in hierarchy (including the target element itself) + while (currentElement && attempts < 10) { + const computedStyle = window.getComputedStyle(currentElement); + const hasScrollableY = /(auto|scroll|overlay)/.test(computedStyle.overflowY); + const canScrollVertically = currentElement.scrollHeight > currentElement.clientHeight; + + if (hasScrollableY && canScrollVertically) { + const beforeScroll = currentElement.scrollTop; + const maxScroll = currentElement.scrollHeight - currentElement.clientHeight; + + // Calculate scroll amount (1/3 of provided dy for gentler scrolling) + let scrollAmount = dy / 3; + + // Ensure we don't scroll beyond bounds + if (scrollAmount > 0) { + scrollAmount = Math.min(scrollAmount, maxScroll - beforeScroll); + } else { + scrollAmount = Math.max(scrollAmount, -beforeScroll); + } + + // Try direct scrollTop manipulation (most reliable) + currentElement.scrollTop = beforeScroll + scrollAmount; + + const afterScroll = currentElement.scrollTop; + const actualScrollDelta = afterScroll - beforeScroll; + + if (Math.abs(actualScrollDelta) > 0.5) { + scrollSuccess = true; + scrolledElement = currentElement; + scrollDelta = actualScrollDelta; + break; + } + } + + // Move to parent (but don't go beyond body for dropdown case) + if (currentElement === document.body || currentElement === document.documentElement) { + break; + } + currentElement = currentElement.parentElement; + attempts++; + } + + if (scrollSuccess) { + // Successfully scrolled a container + return { + success: true, + method: 'direct_container_scroll', + containerType: 'element', + containerTag: scrolledElement.tagName.toLowerCase(), + containerClass: scrolledElement.className || '', + containerId: scrolledElement.id || '', + scrollDelta: scrollDelta + }; + } else { + // No container found or could scroll + return { + success: false, + reason: 'No scrollable container found', + needsPageScroll: true + }; + } + } + `; + const scrollParams = { dy, elementXPath: elementNode.xpath }; + const result = (await page.evaluate(containerScrollJs, scrollParams)); + if (result.success) { + if (result.containerType === 'element') { + let containerInfo = result.containerTag; + if (result.containerId) { + containerInfo += `#${result.containerId}`; + } + else if (result.containerClass) { + containerInfo += `.${result.containerClass.split(' ')[0]}`; + } + scrollTarget = `element ${params.index}'s scroll container (${containerInfo})`; + // Don't do additional page scrolling since we successfully scrolled the container + } + else { + scrollTarget = `the page (fallback from element ${params.index})`; + } + } + else { + // Container scroll failed, need page-level scrolling + scrollLogger.debug(`Container scroll failed for element ${params.index}: ${result.reason || 'Unknown'}`); + scrollTarget = `the page (no container found for element ${params.index})`; + // This will trigger page-level scrolling below + } + } + catch (error) { + scrollLogger.debug(`Element-specific scrolling failed for index ${params.index}: ${error}`); + scrollTarget = `the page (fallback from element ${params.index})`; + // Fall through to page-level scrolling + } + } + // Page-level scrolling (default or fallback) + if (scrollTarget === 'the page' || + scrollTarget.includes('fallback') || + scrollTarget.includes('no container found') || + scrollTarget.includes('mouse wheel failed')) { + scrollLogger.debug(`🔄 Performing page-level scrolling. Reason: ${scrollTarget}`); + try { + await dispatchBrowserEventIfAvailable(browser_session, new ScrollEvent({ + direction, + amount: Math.abs(dy), + }), () => browser_session._scrollContainer(dy)); + } + catch (error) { + // Hard fallback: always works on root scroller + await page.evaluate((y) => window.scrollBy(0, y), dy); + scrollLogger.debug('Smart scroll failed; used window.scrollBy fallback', error); + } + } + // Create descriptive message + let longTermMemory; + if (pagesScrolled === 1.0) { + longTermMemory = `Scrolled ${direction} ${scrollTarget} by one page`; + } + else { + longTermMemory = `Scrolled ${direction} ${scrollTarget} by ${pagesScrolled} pages`; + } + const msg = `🔍 ${longTermMemory}`; + scrollLogger.info(msg); + return new ActionResult({ + extracted_content: msg, + include_in_memory: true, + long_term_memory: longTermMemory, + }); + }; + // Register scroll action with multiple names for LLM compatibility + // Different LLMs may use different names: scroll, scroll_page, scroll_down + const scrollDescription = 'Scroll the page by specified number of pages (set down=True to scroll down, down=False to scroll up, num_pages=number of pages to scroll like 0.5 for half page, 1.0 for one page, etc.). Optional index parameter to scroll within a specific element or its scroll container (works well for dropdowns and custom UI components).'; + // Create named functions that wrap the implementation + // Different LLMs may use different names: scroll, scroll_page, scroll_down, scroll_by, scroll_page_by, scroll_up + const scrollAction = async function scroll(p, ctx) { + return scrollImpl(p, ctx); + }; + const scrollPageAction = async function scroll_page(p, ctx) { + return scrollImpl(p, ctx); + }; + const scrollDownAction = async function scroll_down(p, ctx) { + return scrollImpl(p, ctx); + }; + const scrollByAction = async function scroll_by(p, ctx) { + return scrollImpl(p, ctx); + }; + const scrollPageByAction = async function scroll_page_by(p, ctx) { + return scrollImpl(p, ctx); + }; + const scrollUpAction = async function scroll_up(p, ctx) { + return scrollImpl(p, ctx); + }; + this.registry.action(scrollDescription, { + param_model: ScrollActionSchema, + })(scrollAction); + this.registry.action(scrollDescription, { + param_model: ScrollActionSchema, + })(scrollPageAction); + this.registry.action(scrollDescription, { + param_model: ScrollActionSchema, + })(scrollDownAction); + this.registry.action(scrollDescription, { + param_model: ScrollActionSchema, + })(scrollByAction); + this.registry.action(scrollDescription, { + param_model: ScrollActionSchema, + })(scrollPageByAction); + this.registry.action(scrollDescription, { + param_model: ScrollActionSchema, + })(scrollUpAction); + this.registry.action('Scroll to a text in the current page', { + param_model: ScrollToTextActionSchema, + })(async function scroll_to_text(params, { browser_session }) { + if (!browser_session) + throw new Error('Browser session missing'); + await dispatchBrowserEventIfAvailable(browser_session, new ScrollToTextEvent({ + text: params.text, + direction: 'down', + }), async () => { + const page = await browser_session.get_current_page(); + if (!page?.evaluate) { + throw new BrowserError('Unable to access page for scrolling.'); + } + const success = await page.evaluate(({ text }) => { + const iterator = document.createNodeIterator(document.body, NodeFilter.SHOW_ELEMENT); + let node; + while ((node = iterator.nextNode())) { + const el = node; + if (!el || !el.textContent) + continue; + if (el.textContent.toLowerCase().includes(text.toLowerCase())) { + el.scrollIntoView({ behavior: 'smooth', block: 'center' }); + return true; + } + } + return false; + }, { text: params.text }); + if (!success) { + throw new BrowserError(`Text '${params.text}' not found on page`); + } + return null; + }); + const msg = `🔍 Scrolled to text: ${params.text}`; + return new ActionResult({ + extracted_content: msg, + include_in_memory: true, + long_term_memory: msg, + }); + }); + this.registry.action('Scroll to text.', { + param_model: ScrollToTextActionSchema, + action_name: 'find_text', + })(async function find_text(params, ctx) { + try { + return await registry.execute_action('scroll_to_text', params, ctx); + } + catch (error) { + if (isAbortError(error)) { + throw error; + } + const msg = `Text '${params.text}' not found or not visible on page`; + return new ActionResult({ + extracted_content: msg, + long_term_memory: `Tried scrolling to text '${params.text}' but it was not found`, + }); + } + }); + } + registerFileSystemActions() { + const registry = this.registry; + this.registry.action('Read the complete content of a file. Use this to view file contents before editing or to retrieve data from files. Supports text files (txt, md, json, csv, jsonl), documents (pdf, docx), and images (jpg, png).', { + param_model: ReadFileActionSchema, + })(async function read_file(params, { file_system, available_file_paths }) { + const fsInstance = file_system ?? new FileSystem(process.cwd(), false); + const allowed = Array.isArray(available_file_paths) && + available_file_paths.includes(params.file_name); + const structuredResult = typeof fsInstance.read_file_structured === 'function' + ? await fsInstance.read_file_structured(params.file_name, allowed) + : { + message: await fsInstance.read_file(params.file_name, allowed), + images: null, + }; + const result = String(structuredResult?.message ?? ''); + const images = Array.isArray(structuredResult?.images) + ? structuredResult.images + : null; + const MAX_MEMORY_SIZE = 1000; + let memory = result; + if (images && images.length > 0) { + memory = `Read image file ${params.file_name}`; + } + else if (result.length > MAX_MEMORY_SIZE) { + const lines = result.split('\n'); + let preview = ''; + let used = 0; + for (const line of lines) { + if (preview.length + line.length > MAX_MEMORY_SIZE) + break; + preview += `${line}\n`; + used += 1; + } + const remaining = lines.length - used; + memory = + remaining > 0 ? `${preview}${remaining} more lines...` : preview; + } + return new ActionResult({ + extracted_content: result, + long_term_memory: memory, + images, + include_extracted_content_only_once: true, + }); + }); + this.registry.action('Write content to a file. By default this OVERWRITES the entire file - use append=true to add to an existing file, or use replace_file for targeted edits within a file. ' + + 'FILENAME RULES: Use only letters, numbers, underscores, hyphens, dots, parentheses. Spaces are auto-converted to hyphens. ' + + 'SUPPORTED EXTENSIONS: .txt, .md, .json, .jsonl, .csv, .html, .xml, .pdf, .docx. ' + + 'CANNOT write binary/image files (.png, .jpg, .mp4, etc.) - do not attempt to save screenshots as files. ' + + 'For PDF files, write content in markdown format and it will be auto-converted to PDF.', { + param_model: WriteFileActionSchema, + })(async function write_file(params, { file_system }) { + const fsInstance = file_system ?? new FileSystem(process.cwd(), false); + let content = params.content; + const trailing = params.trailing_newline ?? true; + const leading = params.leading_newline ?? false; + if (trailing) { + content = `${content}\n`; + } + if (leading) { + content = `\n${content}`; + } + const append = params.append ?? false; + const result = append + ? await fsInstance.append_file(params.file_name, content) + : await fsInstance.write_file(params.file_name, content); + return new ActionResult({ + extracted_content: result, + long_term_memory: result, + }); + }); + this.registry.action('Replace specific text within a file by searching for old_str and replacing with new_str. Use this for targeted edits like updating todo checkboxes or modifying specific lines without rewriting the entire file.', { + param_model: ReplaceFileStrActionSchema, + })(async function replace_file_str(params, { file_system }) { + const fsInstance = file_system ?? new FileSystem(process.cwd(), false); + const result = await fsInstance.replace_file_str(params.file_name, params.old_str, params.new_str); + return new ActionResult({ + extracted_content: result, + long_term_memory: result, + }); + }); + this.registry.action('Replace specific text within a file by searching for old_str and replacing with new_str. Use this for targeted edits like updating todo checkboxes or modifying specific lines without rewriting the entire file.', { + param_model: ReplaceFileStrActionSchema, + action_name: 'replace_file', + })(async function replace_file(params, ctx) { + return registry.execute_action('replace_file_str', params, ctx); + }); + } + registerUtilityActions() { + this.registry.action('Take a screenshot of the current viewport. If file_name is provided, saves to that file and returns the path. Otherwise, screenshot is included in the next browser_state observation.', { param_model: ScreenshotActionSchema })(async function screenshot(params, { browser_session, file_system, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + if (params.file_name) { + const screenshotB64 = await dispatchBrowserEventIfAvailable(browser_session, new ScreenshotEvent({ + full_page: false, + }), async () => (await browser_session.take_screenshot?.(false)) ?? null); + if (!screenshotB64) { + return new ActionResult({ + error: 'Failed to capture screenshot.', + }); + } + const fsInstance = file_system ?? new FileSystem(process.cwd(), false); + let fileName = params.file_name; + if (!fileName.toLowerCase().endsWith('.png')) { + fileName = `${fileName}.png`; + } + fileName = FileSystem.sanitize_filename(fileName); + const filePath = path.join(fsInstance.get_dir(), fileName); + await fsp.writeFile(filePath, Buffer.from(screenshotB64, 'base64')); + const msg = `📸 Saved screenshot to ${filePath}`; + return new ActionResult({ + extracted_content: msg, + long_term_memory: msg, + attachments: [filePath], + }); + } + return new ActionResult({ + extracted_content: 'Requested screenshot for next observation', + metadata: { + include_screenshot: true, + }, + }); + }); + this.registry.action('Save the current page as a PDF file. Returns the file path of the saved PDF. Use this to capture the full page content as a printable document.', { param_model: SaveAsPdfActionSchema })(async function save_as_pdf(params, { browser_session, file_system, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const paperSizes = { + letter: { width: 8.5, height: 11 }, + legal: { width: 8.5, height: 14 }, + a4: { width: 8.27, height: 11.69 }, + a3: { width: 11.69, height: 16.54 }, + tabloid: { width: 11, height: 17 }, + }; + const page = await browser_session.get_current_page?.(); + if (!page) { + throw new BrowserError('No active page available for save_as_pdf.'); + } + const paperKey = String(params.paper_format ?? 'Letter').toLowerCase(); + const paperSize = paperSizes[paperKey] ?? paperSizes.letter; + const cdpSession = await browser_session.get_or_create_cdp_session?.(page); + if (!cdpSession?.send) { + throw new BrowserError('CDP session unavailable for save_as_pdf.'); + } + const result = await cdpSession.send('Page.printToPDF', { + printBackground: params.print_background, + landscape: params.landscape, + scale: params.scale, + paperWidth: paperSize.width, + paperHeight: paperSize.height, + preferCSSPageSize: true, + }); + const pdfData = result && typeof result.data === 'string' ? result.data : null; + if (!pdfData) { + throw new BrowserError('CDP Page.printToPDF returned no data.'); + } + const fsInstance = file_system ?? new FileSystem(process.cwd(), false); + let fileName = params.file_name?.trim(); + if (!fileName) { + try { + const titlePromise = typeof page.title === 'function' + ? page.title() + : Promise.resolve(''); + const pageTitle = await Promise.race([ + titlePromise, + new Promise((_, reject) => { + setTimeout(() => reject(new Error('timeout')), 2000); + }), + ]); + const safeTitle = String(pageTitle) + .replace(/[^\w\s-]+/g, '') + .trim() + .slice(0, 50); + fileName = safeTitle || 'page'; + } + catch { + fileName = 'page'; + } + } + if (!fileName.toLowerCase().endsWith('.pdf')) { + fileName = `${fileName}.pdf`; + } + fileName = FileSystem.sanitize_filename(fileName); + const filePath = await resolveUniqueOutputPath(fsInstance.get_dir(), fileName); + await fsp.writeFile(filePath, Buffer.from(pdfData, 'base64')); + const fileSize = (await fsp.stat(filePath)).size; + const baseName = path.basename(filePath); + const msg = `Saved page as PDF: ${baseName} (${fileSize.toLocaleString()} bytes)`; + return new ActionResult({ + extracted_content: msg, + long_term_memory: `${msg}. Full path: ${filePath}`, + attachments: [filePath], + }); + }); + this.registry.action('Execute browser JavaScript on the current page and return the result.', { param_model: EvaluateActionSchema })(async function evaluate(params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const page = await browser_session.get_current_page(); + if (!page?.evaluate) { + throw new BrowserError('No active page available for evaluate.'); + } + const validatedCode = validateAndFixJavaScript(params.code); + const payload = (await page.evaluate(async ({ code }) => { + try { + const raw = await Promise.resolve((0, eval)(code)); + let serializedResult; + if (raw === undefined) { + serializedResult = null; + } + else { + try { + serializedResult = JSON.parse(JSON.stringify(raw)); + } + catch { + serializedResult = String(raw); + } + } + return { ok: true, result: serializedResult }; + } + catch (error) { + return { + ok: false, + error: error instanceof Error + ? error.message + : String(error ?? 'Unknown evaluate error'), + }; + } + }, { code: validatedCode })); + if (!payload) { + return new ActionResult({ error: 'evaluate returned no result' }); + } + if (!payload.ok) { + const codePreview = validatedCode.length > 500 + ? `${validatedCode.slice(0, 500)}...` + : validatedCode; + return new ActionResult({ + error: `JavaScript Execution Failed:\n` + + `JavaScript execution error: ${payload.error ?? 'Unknown error'}\n\n` + + `Validated Code (after quote fixing):\n${codePreview}`, + }); + } + let rendered = typeof payload.result === 'string' + ? payload.result + : JSON.stringify(payload.result); + const imagePattern = /(data:image\/[^;]+;base64,[A-Za-z0-9+/=]+)/g; + const foundImages = rendered.match(imagePattern) ?? []; + let metadata = null; + if (foundImages.length > 0) { + metadata = { images: foundImages }; + for (const imageData of foundImages) { + rendered = rendered.split(imageData).join('[Image]'); + } + } + const maxChars = 20000; + if (rendered.length > maxChars) { + rendered = `${rendered.slice(0, maxChars - 50)}\n... [Truncated after 20000 characters]`; + } + const maxMemoryChars = 10000; + const includeExtractedContentOnlyOnce = rendered.length >= maxMemoryChars; + const longTermMemory = includeExtractedContentOnlyOnce + ? `JavaScript executed successfully, result length: ${rendered.length} characters.` + : rendered; + return new ActionResult({ + extracted_content: rendered, + long_term_memory: longTermMemory, + include_extracted_content_only_once: includeExtractedContentOnlyOnce, + metadata, + }); + }); + } + registerKeyboardActions() { + this.registry.action('Send keys to the active page', { + param_model: SendKeysActionSchema, + })(async function send_keys(params, { browser_session }) { + if (!browser_session) + throw new Error('Browser session missing'); + await dispatchBrowserEventIfAvailable(browser_session, new SendKeysEvent({ keys: params.keys }), async () => { + const page = await browser_session.get_current_page(); + const keyboard = page?.keyboard; + if (!keyboard) { + throw new BrowserError('Keyboard input is not available on the current page.'); + } + try { + await keyboard.press(params.keys); + } + catch (error) { + if (error instanceof Error && + error.message.includes('Unknown key')) { + for (const char of params.keys) { + await keyboard.press(char); + } + } + else { + throw error; + } + } + return null; + }); + const msg = `⌨️ Sent keys: ${params.keys}`; + return new ActionResult({ + extracted_content: msg, + include_in_memory: true, + long_term_memory: msg, + }); + }); + } + registerDropdownActions() { + const registry = this.registry; + const dropdownLogger = this.logger; + const formatAvailableOptions = (options) => options + .map((opt) => ` - [${opt.index}] text=${JSON.stringify(opt.text)} value=${JSON.stringify(opt.value)}`) + .join('\n'); + this.registry.action('Get all options from a native dropdown or ARIA menu', { param_model: DropdownOptionsActionSchema })(async function get_dropdown_options(params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const domElement = await browser_session.get_dom_element_by_index(params.index, { signal }); + if (!domElement) { + const msg = `Element index ${params.index} not available - page may have changed. Try refreshing browser state.`; + dropdownLogger.warning(`⚠️ ${msg}`); + return new ActionResult({ + extracted_content: msg, + }); + } + if (typeof browser_session.dispatch_browser_event === 'function') { + const dispatchResult = await browser_session.dispatch_browser_event(new GetDropdownOptionsEvent({ node: domElement })); + const eventResult = dispatchResult?.event?.event_result; + const eventMessage = eventResult?.message ?? + eventResult?.short_term_memory ?? + eventResult?.formatted_options ?? + null; + if (eventMessage) { + const memory = eventResult?.long_term_memory ?? + `Found dropdown options for index ${params.index}.`; + return new ActionResult({ + extracted_content: eventMessage, + include_in_memory: true, + include_extracted_content_only_once: true, + long_term_memory: memory, + }); + } + } + const page = await browser_session.get_current_page(); + if (!page?.evaluate) { + throw new BrowserError('Unable to evaluate dropdown options on current page.'); + } + if (!domElement.xpath) { + throw new BrowserError('DOM element does not include an XPath selector.'); + } + const payload = await page.evaluate(({ xpath }) => { + const element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + if (!element) + return null; + if (element.tagName?.toLowerCase() === 'select') { + const options = Array.from(element.options).map((opt, index) => ({ + text: opt.textContent?.trim() ?? '', + value: (opt.value ?? '').trim(), + index, + })); + return { type: 'select', options }; + } + const ariaRoles = new Set(['menu', 'listbox', 'combobox']); + const role = element.getAttribute('role'); + if (role && ariaRoles.has(role)) { + const nodes = element.querySelectorAll('[role="menuitem"],[role="option"]'); + const options = Array.from(nodes).map((node, index) => ({ + text: node.textContent?.trim() ?? '', + value: node.textContent?.trim() ?? '', + index, + })); + return { type: 'aria', options }; + } + return null; + }, { xpath: domElement.xpath }); + if (!payload || !payload.options?.length) { + throw new BrowserError('No options found for the specified dropdown.'); + } + const formatted = payload.options.map((opt) => `${opt.index}: text=${JSON.stringify(opt.text ?? '')}, value=${JSON.stringify(opt.value ?? '')}`); + formatted.push('Prefer exact text first; if needed select_dropdown_option also supports case-insensitive text/value matching.'); + const message = formatted.join('\n'); + return new ActionResult({ + extracted_content: message, + include_in_memory: true, + include_extracted_content_only_once: true, + long_term_memory: `Found dropdown options for index ${params.index}.`, + }); + }); + this.registry.action('Get all options from a native dropdown or ARIA menu', { + param_model: DropdownOptionsActionSchema, + action_name: 'dropdown_options', + })(async function dropdown_options(params, ctx) { + return registry.execute_action('get_dropdown_options', params, ctx); + }); + this.registry.action('Select dropdown option or ARIA menu item by text', { + param_model: SelectDropdownActionSchema, + })(async function select_dropdown_option(params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const domElement = await browser_session.get_dom_element_by_index(params.index, { signal }); + if (!domElement) { + const msg = `Element index ${params.index} not available - page may have changed. Try refreshing browser state.`; + dropdownLogger.warning(`⚠️ ${msg}`); + return new ActionResult({ + extracted_content: msg, + }); + } + if (!domElement?.xpath) { + throw new BrowserError('DOM element does not include an XPath selector.'); + } + if (typeof browser_session.dispatch_browser_event === 'function') { + const dispatchResult = await browser_session.dispatch_browser_event(new SelectDropdownOptionEvent({ + node: domElement, + text: params.text, + })); + const eventResult = dispatchResult?.event?.event_result; + const eventMessage = eventResult?.message ?? + eventResult?.short_term_memory ?? + eventResult?.matched_text ?? + null; + if (eventMessage) { + const memory = eventResult?.long_term_memory ?? eventMessage; + return new ActionResult({ + extracted_content: eventMessage, + include_in_memory: true, + long_term_memory: memory, + }); + } + } + const page = await browser_session.get_current_page(); + if (!page) { + throw new BrowserError('No active page for selection.'); + } + for (const frame of page.frames ?? []) { + try { + const typeInfo = await frame.evaluate((xpath) => { + const element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + if (!element) + return { found: false }; + const tagName = element.tagName?.toLowerCase(); + const role = element.getAttribute?.('role'); + if (tagName === 'select') + return { found: true, type: 'select' }; + if (role && ['menu', 'listbox', 'combobox'].includes(role)) + return { found: true, type: 'aria' }; + return { found: false }; + }, domElement.xpath); + if (!typeInfo?.found) + continue; + if (typeInfo.type === 'select') { + const selection = await frame.evaluate(({ xpath, text }) => { + const root = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + if (!root || root.tagName?.toLowerCase() !== 'select') { + return { found: false }; + } + const options = Array.from(root.options).map((opt, index) => ({ + index, + text: opt.textContent?.trim() ?? '', + value: (opt.value ?? '').trim(), + })); + const targetRaw = text.trim(); + const targetLower = text.trim().toLowerCase(); + let matchedIndex = options.findIndex((opt) => opt.text === targetRaw || opt.value === targetRaw); + if (matchedIndex < 0) { + matchedIndex = options.findIndex((opt) => opt.text.trim().toLowerCase() === targetLower || + opt.value.trim().toLowerCase() === targetLower); + } + if (matchedIndex < 0) { + return { found: true, success: false, options }; + } + const matched = options[matchedIndex]; + root.value = matched.value; + root.dispatchEvent(new Event('input', { bubbles: true })); + root.dispatchEvent(new Event('change', { bubbles: true })); + const selectedOption = root.selectedIndex >= 0 + ? root.options[root.selectedIndex] + : null; + const selectedText = selectedOption?.textContent?.trim() ?? ''; + const selectedValue = (root.value ?? '').trim(); + const selectedValueLower = selectedValue.trim().toLowerCase(); + const selectedTextLower = selectedText.trim().toLowerCase(); + const matchedValueLower = String(matched.value ?? '') + .trim() + .toLowerCase(); + const matchedTextLower = String(matched.text ?? '') + .trim() + .toLowerCase(); + const verified = selectedValueLower === matchedValueLower || + selectedTextLower === matchedTextLower; + return { + found: true, + success: verified, + options, + selectedText, + selectedValue, + matched, + }; + }, { xpath: domElement.xpath, text: params.text }); + if (selection?.found && selection.success) { + const matchedText = selection.matched?.text ?? params.text; + const matchedValue = selection.matched?.value ?? ''; + const msg = `Selected option ${matchedText} (${matchedValue})`; + return new ActionResult({ + extracted_content: msg, + include_in_memory: true, + long_term_memory: msg, + }); + } + if (selection?.found) { + const details = formatAvailableOptions(selection.options ?? []); + throw new BrowserError(`Could not select option '${params.text}' for index ${params.index}.\nAvailable options:\n${details}`); + } + continue; + } + const clicked = await frame.evaluate(({ xpath, text }) => { + const root = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + if (!root) + return false; + const nodes = root.querySelectorAll('[role="menuitem"],[role="option"]'); + const options = Array.from(nodes).map((node, index) => ({ + index, + text: node.textContent?.trim() ?? '', + value: node.textContent?.trim() ?? '', + })); + const targetRaw = text.trim(); + const targetLower = text.trim().toLowerCase(); + let matchedIndex = options.findIndex((opt) => opt.text === targetRaw || opt.value === targetRaw); + if (matchedIndex < 0) { + matchedIndex = options.findIndex((opt) => opt.text.trim().toLowerCase() === targetLower || + opt.value.trim().toLowerCase() === targetLower); + } + if (matchedIndex < 0) { + return { found: true, success: false, options }; + } + nodes[matchedIndex].click(); + return { + found: true, + success: true, + options, + matched: options[matchedIndex], + }; + }, { xpath: domElement.xpath, text: params.text }); + if (clicked?.found && clicked.success) { + const matchedText = clicked.matched?.text ?? params.text; + const msg = `Selected menu item ${matchedText}`; + return new ActionResult({ + extracted_content: msg, + include_in_memory: true, + long_term_memory: msg, + }); + } + if (clicked?.found) { + const details = formatAvailableOptions(clicked.options ?? []); + throw new BrowserError(`Could not select option '${params.text}' for index ${params.index}.\nAvailable options:\n${details}`); + } + } + catch (error) { + if (error instanceof BrowserError) { + throw error; + } + continue; + } + } + throw new BrowserError(`Could not select option '${params.text}' for index ${params.index}`); + }); + this.registry.action('Select dropdown option or ARIA menu item by text', { + param_model: SelectDropdownActionSchema, + action_name: 'select_dropdown', + })(async function select_dropdown(params, ctx) { + return registry.execute_action('select_dropdown_option', params, ctx); + }); + } + registerSheetsActions() { + const gotoSheetsRange = this.gotoSheetsRange.bind(this); + this.registry.action('Google Sheets: Get the contents of the entire sheet', { + domains: ['https://docs.google.com'], + })(async function sheets_get_contents(_params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const page = await browser_session.get_current_page(); + await page?.keyboard?.press('Enter'); + await page?.keyboard?.press('Escape'); + await page?.keyboard?.press('ControlOrMeta+A'); + await page?.keyboard?.press('ControlOrMeta+C'); + const content = await page?.evaluate?.(() => navigator.clipboard.readText()); + return new ActionResult({ + extracted_content: content ?? '', + include_in_memory: true, + long_term_memory: 'Retrieved sheet contents', + include_extracted_content_only_once: true, + }); + }); + this.registry.action('Google Sheets: Get the contents of a cell or range of cells', { + domains: ['https://docs.google.com'], + param_model: SheetsRangeActionSchema, + })(async function sheets_get_range(params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const page = await browser_session.get_current_page(); + await gotoSheetsRange(page, params.cell_or_range, signal); + await page?.keyboard?.press('ControlOrMeta+C'); + await waitWithSignal(100, signal); + const content = await page?.evaluate?.(() => navigator.clipboard.readText()); + return new ActionResult({ + extracted_content: content ?? '', + include_in_memory: true, + long_term_memory: `Retrieved contents from ${params.cell_or_range}`, + include_extracted_content_only_once: true, + }); + }); + this.registry.action('Google Sheets: Update the content of a cell or range of cells', { + domains: ['https://docs.google.com'], + param_model: SheetsUpdateActionSchema, + })(async function sheets_update(params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const page = await browser_session.get_current_page(); + await gotoSheetsRange(page, params.cell_or_range, signal); + await page?.evaluate?.((value) => { + const clipboardData = new DataTransfer(); + clipboardData.setData('text/plain', value); + document.activeElement?.dispatchEvent(new ClipboardEvent('paste', { clipboardData })); + }, params.value); + return new ActionResult({ + extracted_content: `Updated cells: ${params.cell_or_range} = ${params.value}`, + long_term_memory: `Updated cells ${params.cell_or_range} with ${params.value}`, + }); + }); + this.registry.action('Google Sheets: Clear whatever cells are currently selected', { + domains: ['https://docs.google.com'], + param_model: SheetsRangeActionSchema, + })(async function sheets_clear(params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const page = await browser_session.get_current_page(); + await gotoSheetsRange(page, params.cell_or_range, signal); + await page?.keyboard?.press('Backspace'); + return new ActionResult({ + extracted_content: `Cleared cells: ${params.cell_or_range}`, + long_term_memory: `Cleared cells ${params.cell_or_range}`, + }); + }); + this.registry.action('Google Sheets: Select a specific cell or range of cells', { + domains: ['https://docs.google.com'], + param_model: SheetsRangeActionSchema, + })(async function sheets_select(params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const page = await browser_session.get_current_page(); + await gotoSheetsRange(page, params.cell_or_range, signal); + return new ActionResult({ + extracted_content: `Selected cells: ${params.cell_or_range}`, + long_term_memory: `Selected cells ${params.cell_or_range}`, + }); + }); + this.registry.action('Google Sheets: Fallback method to type text into the currently selected cell', { + domains: ['https://docs.google.com'], + param_model: SheetsInputActionSchema, + })(async function sheets_input(params, { browser_session, signal }) { + if (!browser_session) + throw new Error('Browser session missing'); + throwIfAborted(signal); + const page = await browser_session.get_current_page(); + await page?.keyboard?.type(params.text, { delay: 100 }); + await page?.keyboard?.press('Enter'); + await page?.keyboard?.press('ArrowUp'); + return new ActionResult({ + extracted_content: `Inputted text ${params.text}`, + long_term_memory: `Inputted text '${params.text}' into cell`, + }); + }); + } + async gotoSheetsRange(page, cell_or_range, signal = null) { + if (!page?.keyboard) { + throw new BrowserError('No keyboard available for Google Sheets actions.'); + } + throwIfAborted(signal); + await page.keyboard.press('Enter'); + await page.keyboard.press('Escape'); + await waitWithSignal(100, signal); + await page.keyboard.press('Home'); + await page.keyboard.press('ArrowUp'); + await waitWithSignal(100, signal); + await page.keyboard.press('Control+G'); + await waitWithSignal(200, signal); + await page.keyboard.type(cell_or_range, { delay: 50 }); + await page.keyboard.press('Enter'); + await waitWithSignal(200, signal); + await page.keyboard.press('Escape'); + } + registerDoneAction(outputModel) { + const displayFilesInDoneText = this.displayFilesInDoneText; + if (outputModel) { + const structuredSchema = StructuredOutputActionSchema(outputModel); + this.registry.action('Complete task - with return text and success flag.', { param_model: structuredSchema })(async function done(params) { + const payload = params.data && + typeof params.data === 'object' && + !Array.isArray(params.data) + ? params.data + : {}; + return new ActionResult({ + is_done: true, + success: params.success, + extracted_content: JSON.stringify(payload), + long_term_memory: `Task completed. Success Status: ${params.success}`, + }); + }); + return; + } + this.registry.action('Complete task - provide a summary to the user.', { + param_model: DoneActionSchema, + })(async function done(params, { file_system }) { + const fsInstance = file_system ?? new FileSystem(process.cwd(), false); + let userMessage = params.text; + const lenMaxMemory = 100; + let memory = `Task completed: ${params.success} - ${params.text.slice(0, lenMaxMemory)}`; + if (params.text.length > lenMaxMemory) { + memory += ` - ${params.text.length - lenMaxMemory} more characters`; + } + const attachments = []; + if (params.files_to_display) { + if (displayFilesInDoneText) { + let attachmentText = ''; + for (const fileName of params.files_to_display) { + const content = fsInstance.display_file(fileName); + if (content) { + attachmentText += `\n\n${fileName}:\n${content}`; + attachments.push(fileName); + } + } + if (attachmentText) { + userMessage += '\n\nAttachments:'; + userMessage += attachmentText; + } + } + else { + for (const fileName of params.files_to_display) { + const content = fsInstance.display_file(fileName); + if (content) { + attachments.push(fileName); + } + } + } + } + const attachmentPaths = attachments.map((name) => `${fsInstance.get_dir()}/${name}`); + return new ActionResult({ + is_done: true, + success: params.success, + extracted_content: userMessage, + long_term_memory: memory, + attachments: attachmentPaths, + }); + }); + } + use_structured_output_action(outputModel) { + this.outputModel = outputModel; + this.registerDoneAction(outputModel); + } + get_output_model() { + return this.outputModel; + } + exclude_action(actionName) { + this.registry.exclude_action(actionName); + } + set_coordinate_clicking(enabled) { + const resolved = Boolean(enabled); + if (resolved === this.coordinateClickingEnabled) { + return; + } + this.coordinateClickingEnabled = resolved; + this.registerClickActions(); + this.logger.debug(`Coordinate clicking ${resolved ? 'enabled' : 'disabled'}`); + } + action(description, options = {}) { + return this.registry.action(description, options); + } + async act(action, { browser_session, page_extraction_llm = null, sensitive_data = null, available_file_paths = null, file_system = null, context = null, signal = null, }) { + const entries = toActionEntries(action); + for (const [actionName, params] of entries) { + try { + const result = await this.registry.execute_action(actionName, params, { + browser_session, + page_extraction_llm, + sensitive_data, + available_file_paths, + file_system, + context, + signal, + }); + if (typeof result === 'string') { + return new ActionResult({ extracted_content: result }); + } + if (result instanceof ActionResult) { + return result; + } + if (result == null) { + return new ActionResult(); + } + const resultType = result && typeof result === 'object' + ? (result.constructor?.name ?? typeof result) + : typeof result; + throw new Error(`Invalid action result type: ${resultType} of ${String(result)}`); + } + catch (error) { + if (error instanceof BrowserError) { + if (error.long_term_memory != null) { + if (error.short_term_memory != null) { + return new ActionResult({ + extracted_content: error.short_term_memory, + error: error.long_term_memory, + include_extracted_content_only_once: true, + }); + } + return new ActionResult({ + error: error.long_term_memory, + }); + } + throw error; + } + const message = String(error?.message ?? error ?? ''); + if (error instanceof Error && + message === `Error executing action ${actionName} due to timeout.`) { + return new ActionResult({ + error: `${actionName} was not executed due to timeout.`, + }); + } + return new ActionResult({ + error: message, + }); + } + } + return new ActionResult(); + } +} diff --git a/dist/controller/views.d.ts b/dist/controller/views.d.ts new file mode 100644 index 00000000..b624fb28 --- /dev/null +++ b/dist/controller/views.d.ts @@ -0,0 +1,167 @@ +import { z } from 'zod'; +export declare const SearchGoogleActionSchema: z.ZodObject<{ + query: z.ZodString; +}, z.core.$strip>; +export type SearchGoogleAction = z.infer; +export declare const SearchActionSchema: z.ZodObject<{ + query: z.ZodString; + engine: z.ZodDefault; +}, z.core.$strip>; +export type SearchAction = z.infer; +export declare const GoToUrlActionSchema: z.ZodObject<{ + url: z.ZodString; + new_tab: z.ZodDefault; +}, z.core.$strip>; +export type GoToUrlAction = z.infer; +export declare const WaitActionSchema: z.ZodObject<{ + seconds: z.ZodDefault; +}, z.core.$strip>; +export type WaitAction = z.infer; +export declare const ClickElementActionSchema: z.ZodObject<{ + index: z.ZodOptional; + coordinate_x: z.ZodOptional; + coordinate_y: z.ZodOptional; +}, z.core.$strip>; +export type ClickElementAction = z.infer; +export declare const ClickElementActionIndexOnlySchema: z.ZodObject<{ + index: z.ZodNumber; +}, z.core.$strip>; +export type ClickElementActionIndexOnly = z.infer; +export declare const InputTextActionSchema: z.ZodObject<{ + index: z.ZodNumber; + text: z.ZodString; + clear: z.ZodDefault; +}, z.core.$strip>; +export type InputTextAction = z.infer; +export declare const DoneActionSchema: z.ZodObject<{ + text: z.ZodString; + success: z.ZodDefault; + files_to_display: z.ZodDefault>; +}, z.core.$strip>; +export type DoneAction = z.infer; +export declare const StructuredOutputActionSchema: (dataSchema: T) => z.ZodObject<{ + success: z.ZodDefault; + data: T; +}, z.core.$strip>; +export type StructuredOutputAction = { + success: boolean; + data: T; +}; +export declare const SwitchTabActionSchema: z.ZodObject<{ + page_id: z.ZodOptional; + tab_id: z.ZodOptional; +}, z.core.$strip>; +export type SwitchTabAction = z.infer; +export declare const CloseTabActionSchema: z.ZodObject<{ + page_id: z.ZodOptional; + tab_id: z.ZodOptional; +}, z.core.$strip>; +export type CloseTabAction = z.infer; +export declare const ScrollActionSchema: z.ZodObject<{ + down: z.ZodDefault; + num_pages: z.ZodDefault; + pages: z.ZodOptional; + index: z.ZodOptional; +}, z.core.$strip>; +export type ScrollAction = z.infer; +export declare const SendKeysActionSchema: z.ZodObject<{ + keys: z.ZodString; +}, z.core.$strip>; +export type SendKeysAction = z.infer; +export declare const UploadFileActionSchema: z.ZodObject<{ + index: z.ZodNumber; + path: z.ZodString; +}, z.core.$strip>; +export type UploadFileAction = z.infer; +export declare const ScreenshotActionSchema: z.ZodObject<{ + file_name: z.ZodOptional; +}, z.core.$strip>; +export type ScreenshotAction = z.infer; +export declare const SaveAsPdfActionSchema: z.ZodObject<{ + file_name: z.ZodOptional; + print_background: z.ZodDefault; + landscape: z.ZodDefault; + scale: z.ZodDefault; + paper_format: z.ZodDefault; +}, z.core.$strip>; +export type SaveAsPdfAction = z.infer; +export declare const EvaluateActionSchema: z.ZodObject<{ + code: z.ZodString; +}, z.core.$strip>; +export type EvaluateAction = z.infer; +export declare const ExtractPageContentActionSchema: z.ZodObject<{ + value: z.ZodString; +}, z.core.$strip>; +export type ExtractPageContentAction = z.infer; +export declare const ExtractStructuredDataActionSchema: z.ZodObject<{ + query: z.ZodString; + extract_links: z.ZodDefault; + start_from_char: z.ZodDefault; + output_schema: z.ZodOptional>>; + already_collected: z.ZodDefault>; +}, z.core.$strip>; +export type ExtractStructuredDataAction = z.infer; +export declare const SearchPageActionSchema: z.ZodObject<{ + pattern: z.ZodString; + regex: z.ZodDefault; + case_sensitive: z.ZodDefault; + context_chars: z.ZodDefault; + css_scope: z.ZodOptional; + max_results: z.ZodDefault; +}, z.core.$strip>; +export type SearchPageAction = z.infer; +export declare const FindElementsActionSchema: z.ZodObject<{ + selector: z.ZodString; + attributes: z.ZodOptional>; + max_results: z.ZodDefault; + include_text: z.ZodDefault; +}, z.core.$strip>; +export type FindElementsAction = z.infer; +export declare const ReadFileActionSchema: z.ZodObject<{ + file_name: z.ZodString; +}, z.core.$strip>; +export type ReadFileAction = z.infer; +export declare const WriteFileActionSchema: z.ZodObject<{ + file_name: z.ZodString; + content: z.ZodString; + append: z.ZodOptional; + trailing_newline: z.ZodOptional; + leading_newline: z.ZodOptional; +}, z.core.$strip>; +export type WriteFileAction = z.infer; +export declare const ReplaceFileStrActionSchema: z.ZodObject<{ + file_name: z.ZodString; + old_str: z.ZodString; + new_str: z.ZodString; +}, z.core.$strip>; +export type ReplaceFileStrAction = z.infer; +export declare const ScrollToTextActionSchema: z.ZodObject<{ + text: z.ZodString; +}, z.core.$strip>; +export type ScrollToTextAction = z.infer; +export declare const DropdownOptionsActionSchema: z.ZodObject<{ + index: z.ZodNumber; +}, z.core.$strip>; +export type DropdownOptionsAction = z.infer; +export declare const SelectDropdownActionSchema: z.ZodObject<{ + index: z.ZodNumber; + text: z.ZodString; +}, z.core.$strip>; +export type SelectDropdownAction = z.infer; +export declare const SheetsRangeActionSchema: z.ZodObject<{ + cell_or_range: z.ZodString; +}, z.core.$strip>; +export type SheetsRangeAction = z.infer; +export declare const SheetsUpdateActionSchema: z.ZodObject<{ + cell_or_range: z.ZodString; + value: z.ZodString; +}, z.core.$strip>; +export type SheetsUpdateAction = z.infer; +export declare const SheetsInputActionSchema: z.ZodObject<{ + text: z.ZodString; +}, z.core.$strip>; +export type SheetsInputAction = z.infer; +export declare const NoParamsActionSchema: z.ZodObject<{ + description: z.ZodOptional; +}, z.core.$loose>; +export type NoParamsAction = z.infer; diff --git a/dist/controller/views.js b/dist/controller/views.js new file mode 100644 index 00000000..c64bb2a6 --- /dev/null +++ b/dist/controller/views.js @@ -0,0 +1,140 @@ +import { z } from 'zod'; +export const SearchGoogleActionSchema = z.object({ + query: z.string(), +}); +export const SearchActionSchema = z.object({ + query: z.string(), + engine: z.string().default('duckduckgo'), +}); +export const GoToUrlActionSchema = z.object({ + url: z.string(), + new_tab: z.boolean().default(false), +}); +export const WaitActionSchema = z.object({ + seconds: z.number().default(3), +}); +export const ClickElementActionSchema = z.object({ + index: z.number().int().min(1).optional(), + coordinate_x: z.number().int().optional(), + coordinate_y: z.number().int().optional(), +}); +export const ClickElementActionIndexOnlySchema = z.object({ + index: z.number().int().min(1), +}); +export const InputTextActionSchema = z.object({ + index: z.number().int().min(0), + text: z.string(), + clear: z.boolean().default(true), +}); +export const DoneActionSchema = z.object({ + text: z.string(), + success: z.boolean().default(true), + files_to_display: z.array(z.string()).default([]), +}); +export const StructuredOutputActionSchema = (dataSchema) => z.object({ + success: z + .boolean() + .default(true) + .describe('True if user_request completed successfully'), + data: dataSchema, +}); +const TabIdentifierActionSchema = z + .object({ + page_id: z.number().int().optional(), + tab_id: z.string().trim().length(4).optional(), +}) + .refine((value) => value.page_id != null || value.tab_id != null, { + message: 'Provide tab_id or page_id', +}); +export const SwitchTabActionSchema = TabIdentifierActionSchema; +export const CloseTabActionSchema = TabIdentifierActionSchema; +export const ScrollActionSchema = z.object({ + down: z.boolean().default(true), // Default to scroll down + num_pages: z.number().default(1), // Default to 1 page + pages: z.number().optional(), // Alias for num_pages + index: z.number().int().optional(), +}); +export const SendKeysActionSchema = z.object({ + keys: z.string(), +}); +export const UploadFileActionSchema = z.object({ + index: z.number().int(), + path: z.string(), +}); +export const ScreenshotActionSchema = z.object({ + file_name: z.string().optional(), +}); +export const SaveAsPdfActionSchema = z.object({ + file_name: z.string().optional(), + print_background: z.boolean().default(true), + landscape: z.boolean().default(false), + scale: z.number().min(0.1).max(2.0).default(1.0), + paper_format: z.string().default('Letter'), +}); +export const EvaluateActionSchema = z.object({ + code: z.string(), +}); +export const ExtractPageContentActionSchema = z.object({ + value: z.string(), +}); +export const ExtractStructuredDataActionSchema = z.object({ + query: z.string(), + extract_links: z.boolean().default(false), + start_from_char: z.number().int().default(0), + output_schema: z.record(z.string(), z.unknown()).nullable().optional(), + already_collected: z.array(z.string()).default([]), +}); +export const SearchPageActionSchema = z.object({ + pattern: z.string(), + regex: z.boolean().default(false), + case_sensitive: z.boolean().default(false), + context_chars: z.number().int().default(150), + css_scope: z.string().optional(), + max_results: z.number().int().default(25), +}); +export const FindElementsActionSchema = z.object({ + selector: z.string(), + attributes: z.array(z.string()).optional(), + max_results: z.number().int().default(50), + include_text: z.boolean().default(true), +}); +export const ReadFileActionSchema = z.object({ + file_name: z.string(), +}); +export const WriteFileActionSchema = z.object({ + file_name: z.string(), + content: z.string(), + append: z.boolean().optional(), + trailing_newline: z.boolean().optional(), + leading_newline: z.boolean().optional(), +}); +export const ReplaceFileStrActionSchema = z.object({ + file_name: z.string(), + old_str: z.string(), + new_str: z.string(), +}); +export const ScrollToTextActionSchema = z.object({ + text: z.string(), +}); +export const DropdownOptionsActionSchema = z.object({ + index: z.number().int().min(1), +}); +export const SelectDropdownActionSchema = z.object({ + index: z.number().int().min(1), + text: z.string(), +}); +export const SheetsRangeActionSchema = z.object({ + cell_or_range: z.string(), +}); +export const SheetsUpdateActionSchema = z.object({ + cell_or_range: z.string(), + value: z.string(), +}); +export const SheetsInputActionSchema = z.object({ + text: z.string(), +}); +export const NoParamsActionSchema = z + .object({ + description: z.string().optional(), +}) + .passthrough(); diff --git a/dist/dom/clickable-element-processor/service.d.ts b/dist/dom/clickable-element-processor/service.d.ts new file mode 100644 index 00000000..2d61f37b --- /dev/null +++ b/dist/dom/clickable-element-processor/service.d.ts @@ -0,0 +1,11 @@ +import { DOMElementNode } from '../views.js'; +export declare class ClickableElementProcessor { + static get_clickable_elements_hashes(dom_element: DOMElementNode): Set; + static get_clickable_elements(dom_element: DOMElementNode): DOMElementNode[]; + static hash_dom_element(dom_element: DOMElementNode): string; + private static _get_parent_branch_path; + private static _parent_branch_path_hash; + private static _attributes_hash; + private static _xpath_hash; + private static _hash_string; +} diff --git a/dist/dom/clickable-element-processor/service.js b/dist/dom/clickable-element-processor/service.js new file mode 100644 index 00000000..025e3cc3 --- /dev/null +++ b/dist/dom/clickable-element-processor/service.js @@ -0,0 +1,60 @@ +import crypto from 'node:crypto'; +import { DOMElementNode } from '../views.js'; +const sha256 = (value) => crypto.createHash('sha256').update(value).digest('hex'); +export class ClickableElementProcessor { + static get_clickable_elements_hashes(dom_element) { + const hashes = new Set(); + for (const element of this.get_clickable_elements(dom_element)) { + hashes.add(this.hash_dom_element(element)); + } + return hashes; + } + static get_clickable_elements(dom_element) { + const elements = []; + const traverse = (node) => { + for (const child of node.children) { + if (child instanceof DOMElementNode) { + if (child.highlight_index !== null && + child.highlight_index !== undefined) { + elements.push(child); + } + traverse(child); + } + } + }; + traverse(dom_element); + return elements; + } + static hash_dom_element(dom_element) { + const parent_branch_path = this._get_parent_branch_path(dom_element); + const branch_path_hash = this._parent_branch_path_hash(parent_branch_path); + const attributes_hash = this._attributes_hash(dom_element.attributes); + const xpath_hash = this._xpath_hash(dom_element.xpath); + return this._hash_string(`${branch_path_hash}-${attributes_hash}-${xpath_hash}`); + } + static _get_parent_branch_path(dom_element) { + const parents = []; + let current = dom_element; + while (current && current.parent) { + parents.push(current); + current = current.parent; + } + parents.reverse(); + return parents.map((parent) => parent.tag_name); + } + static _parent_branch_path_hash(parent_branch_path) { + return sha256(parent_branch_path.join('/')); + } + static _attributes_hash(attributes) { + const attributes_string = Object.entries(attributes) + .map(([key, value]) => `${key}=${value}`) + .join(''); + return this._hash_string(attributes_string); + } + static _xpath_hash(xpath) { + return this._hash_string(xpath); + } + static _hash_string(value) { + return sha256(value); + } +} diff --git a/dist/dom/dom_tree/index.js b/dist/dom/dom_tree/index.js new file mode 100644 index 00000000..9847ff78 --- /dev/null +++ b/dist/dom/dom_tree/index.js @@ -0,0 +1,1413 @@ +( + args = { + doHighlightElements: true, + focusHighlightIndex: -1, + viewportExpansion: 0, + debugMode: false, + } +) => { + const { doHighlightElements, focusHighlightIndex, viewportExpansion, debugMode } = args; + let highlightIndex = 0; // Reset highlight index + + // Add caching mechanisms at the top level + const DOM_CACHE = { + boundingRects: new WeakMap(), + clientRects: new WeakMap(), + computedStyles: new WeakMap(), + clearCache: () => { + DOM_CACHE.boundingRects = new WeakMap(); + DOM_CACHE.clientRects = new WeakMap(); + DOM_CACHE.computedStyles = new WeakMap(); + } + }; + + /** + * Gets the cached bounding rect for an element. + * + * @param {HTMLElement} element - The element to get the bounding rect for. + * @returns {DOMRect | null} The cached bounding rect, or null if the element is not found. + */ + function getCachedBoundingRect(element) { + if (!element) return null; + + if (DOM_CACHE.boundingRects.has(element)) { + return DOM_CACHE.boundingRects.get(element); + } + + const rect = element.getBoundingClientRect(); + + if (rect) { + DOM_CACHE.boundingRects.set(element, rect); + } + return rect; + } + + /** + * Gets the cached computed style for an element. + * + * @param {HTMLElement} element - The element to get the computed style for. + * @returns {CSSStyleDeclaration | null} The cached computed style, or null if the element is not found. + */ + function getCachedComputedStyle(element) { + if (!element) return null; + + if (DOM_CACHE.computedStyles.has(element)) { + return DOM_CACHE.computedStyles.get(element); + } + + const style = window.getComputedStyle(element); + + if (style) { + DOM_CACHE.computedStyles.set(element, style); + } + return style; + } + + /** + * Gets the cached client rects for an element. + * + * @param {HTMLElement} element - The element to get the client rects for. + * @returns {DOMRectList | null} The cached client rects, or null if the element is not found. + */ + function getCachedClientRects(element) { + if (!element) return null; + + if (DOM_CACHE.clientRects.has(element)) { + return DOM_CACHE.clientRects.get(element); + } + + const rects = element.getClientRects(); + + if (rects) { + DOM_CACHE.clientRects.set(element, rects); + } + return rects; + } + + /** + * Hash map of DOM nodes indexed by their highlight index. + * + * @type {Object} + */ + const DOM_HASH_MAP = {}; + + const ID = { current: 0 }; + + const HIGHLIGHT_CONTAINER_ID = "playwright-highlight-container"; + + // Add a WeakMap cache for XPath strings + const xpathCache = new WeakMap(); + + // // Initialize once and reuse + // const viewportObserver = new IntersectionObserver( + // (entries) => { + // entries.forEach(entry => { + // elementVisibilityMap.set(entry.target, entry.isIntersecting); + // }); + // }, + // { rootMargin: `${viewportExpansion}px` } + // ); + + /** + * Highlights an element in the DOM and returns the index of the next element. + * + * @param {HTMLElement} element - The element to highlight. + * @param {number} index - The index of the element. + * @param {HTMLElement | null} parentIframe - The parent iframe node. + * @returns {number} The index of the next element. + */ + function highlightElement(element, index, parentIframe = null) { + if (!element) return index; + + const overlays = []; + /** + * @type {HTMLElement | null} + */ + let label = null; + let labelWidth = 20; + let labelHeight = 16; + let cleanupFn = null; + + try { + // Create or get highlight container + let container = document.getElementById(HIGHLIGHT_CONTAINER_ID); + if (!container) { + container = document.createElement("div"); + container.id = HIGHLIGHT_CONTAINER_ID; + container.style.position = "fixed"; + container.style.pointerEvents = "none"; + container.style.top = "0"; + container.style.left = "0"; + container.style.width = "100%"; + container.style.height = "100%"; + // Use the maximum valid value in zIndex to ensure the element is not blocked by overlapping elements. + container.style.zIndex = "2147483647"; + container.style.backgroundColor = 'transparent'; + document.body.appendChild(container); + } + + // Get element client rects + const rects = element.getClientRects(); // Use getClientRects() + + if (!rects || rects.length === 0) return index; // Exit if no rects + + // Generate a color based on the index + const colors = [ + "#FF0000", + "#00FF00", + "#0000FF", + "#FFA500", + "#800080", + "#008080", + "#FF69B4", + "#4B0082", + "#FF4500", + "#2E8B57", + "#DC143C", + "#4682B4", + ]; + const colorIndex = index % colors.length; + const baseColor = colors[colorIndex]; + const backgroundColor = baseColor + "1A"; // 10% opacity version of the color + + // Get iframe offset if necessary + let iframeOffset = { x: 0, y: 0 }; + if (parentIframe) { + const iframeRect = parentIframe.getBoundingClientRect(); // Keep getBoundingClientRect for iframe offset + iframeOffset.x = iframeRect.left; + iframeOffset.y = iframeRect.top; + } + + // Create fragment to hold overlay elements + const fragment = document.createDocumentFragment(); + + // Create highlight overlays for each client rect + for (const rect of rects) { + if (rect.width === 0 || rect.height === 0) continue; // Skip empty rects + + const overlay = document.createElement("div"); + overlay.style.position = "fixed"; + overlay.style.border = `2px solid ${baseColor}`; + overlay.style.backgroundColor = backgroundColor; + overlay.style.pointerEvents = "none"; + overlay.style.boxSizing = "border-box"; + + const top = rect.top + iframeOffset.y; + const left = rect.left + iframeOffset.x; + + overlay.style.top = `${top}px`; + overlay.style.left = `${left}px`; + overlay.style.width = `${rect.width}px`; + overlay.style.height = `${rect.height}px`; + + fragment.appendChild(overlay); + overlays.push({ element: overlay, initialRect: rect }); // Store overlay and its rect + } + + // Create and position a single label relative to the first rect + const firstRect = rects[0]; + label = document.createElement("div"); + label.className = "playwright-highlight-label"; + label.style.position = "fixed"; + label.style.background = baseColor; + label.style.color = "white"; + label.style.padding = "1px 4px"; + label.style.borderRadius = "4px"; + label.style.fontSize = `${Math.min(12, Math.max(8, firstRect.height / 2))}px`; + label.textContent = index.toString(); + + labelWidth = label.offsetWidth > 0 ? label.offsetWidth : labelWidth; // Update actual width if possible + labelHeight = label.offsetHeight > 0 ? label.offsetHeight : labelHeight; // Update actual height if possible + + const firstRectTop = firstRect.top + iframeOffset.y; + const firstRectLeft = firstRect.left + iframeOffset.x; + + let labelTop = firstRectTop + 2; + let labelLeft = firstRectLeft + firstRect.width - labelWidth - 2; + + // Adjust label position if first rect is too small + if (firstRect.width < labelWidth + 4 || firstRect.height < labelHeight + 4) { + labelTop = firstRectTop - labelHeight - 2; + labelLeft = firstRectLeft + firstRect.width - labelWidth; // Align with right edge + if (labelLeft < iframeOffset.x) labelLeft = firstRectLeft; // Prevent going off-left + } + + // Ensure label stays within viewport bounds slightly better + labelTop = Math.max(0, Math.min(labelTop, window.innerHeight - labelHeight)); + labelLeft = Math.max(0, Math.min(labelLeft, window.innerWidth - labelWidth)); + + + label.style.top = `${labelTop}px`; + label.style.left = `${labelLeft}px`; + + fragment.appendChild(label); + + // Update positions on scroll/resize + const updatePositions = () => { + const newRects = element.getClientRects(); // Get fresh rects + let newIframeOffset = { x: 0, y: 0 }; + + if (parentIframe) { + const iframeRect = parentIframe.getBoundingClientRect(); // Keep getBoundingClientRect for iframe + newIframeOffset.x = iframeRect.left; + newIframeOffset.y = iframeRect.top; + } + + // Update each overlay + overlays.forEach((overlayData, i) => { + if (i < newRects.length) { // Check if rect still exists + const newRect = newRects[i]; + const newTop = newRect.top + newIframeOffset.y; + const newLeft = newRect.left + newIframeOffset.x; + + overlayData.element.style.top = `${newTop}px`; + overlayData.element.style.left = `${newLeft}px`; + overlayData.element.style.width = `${newRect.width}px`; + overlayData.element.style.height = `${newRect.height}px`; + overlayData.element.style.display = (newRect.width === 0 || newRect.height === 0) ? 'none' : 'block'; + } else { + // If fewer rects now, hide extra overlays + overlayData.element.style.display = 'none'; + } + }); + + // If there are fewer new rects than overlays, hide the extras + if (newRects.length < overlays.length) { + for (let i = newRects.length; i < overlays.length; i++) { + overlays[i].element.style.display = 'none'; + } + } + + // Update label position based on the first new rect + if (label && newRects.length > 0) { + const firstNewRect = newRects[0]; + const firstNewRectTop = firstNewRect.top + newIframeOffset.y; + const firstNewRectLeft = firstNewRect.left + newIframeOffset.x; + + let newLabelTop = firstNewRectTop + 2; + let newLabelLeft = firstNewRectLeft + firstNewRect.width - labelWidth - 2; + + if (firstNewRect.width < labelWidth + 4 || firstNewRect.height < labelHeight + 4) { + newLabelTop = firstNewRectTop - labelHeight - 2; + newLabelLeft = firstNewRectLeft + firstNewRect.width - labelWidth; + if (newLabelLeft < newIframeOffset.x) newLabelLeft = firstNewRectLeft; + } + + // Ensure label stays within viewport bounds + newLabelTop = Math.max(0, Math.min(newLabelTop, window.innerHeight - labelHeight)); + newLabelLeft = Math.max(0, Math.min(newLabelLeft, window.innerWidth - labelWidth)); + + label.style.top = `${newLabelTop}px`; + label.style.left = `${newLabelLeft}px`; + label.style.display = 'block'; + } else if (label) { + // Hide label if element has no rects anymore + label.style.display = 'none'; + } + }; + + + const throttleFunction = (func, delay) => { + let lastCall = 0; + return (...args) => { + const now = performance.now(); + if (now - lastCall < delay) return; + lastCall = now; + return func(...args); + }; + }; + + const throttledUpdatePositions = throttleFunction(updatePositions, 16); // ~60fps + window.addEventListener('scroll', throttledUpdatePositions, true); + window.addEventListener('resize', throttledUpdatePositions); + + // Add cleanup function + cleanupFn = () => { + window.removeEventListener('scroll', throttledUpdatePositions, true); + window.removeEventListener('resize', throttledUpdatePositions); + // Remove overlay elements if needed + overlays.forEach(overlay => overlay.element.remove()); + if (label) label.remove(); + }; + + // Then add fragment to container in one operation + container.appendChild(fragment); + + return index + 1; + } finally { + // Store cleanup function for later use + if (cleanupFn) { + // Keep a reference to cleanup functions in a global array + (window._highlightCleanupFunctions = window._highlightCleanupFunctions || []).push(cleanupFn); + } + } + } + + function cleanupHighlights() { + try { + const cleanupFns = Array.isArray(window._highlightCleanupFunctions) + ? window._highlightCleanupFunctions + : []; + for (const fn of cleanupFns) { + try { + if (typeof fn === 'function') { + fn(); + } + } catch (error) { + // Ignore cleanup callback failures to keep extraction resilient. + } + } + window._highlightCleanupFunctions = []; + const container = document.getElementById(HIGHLIGHT_CONTAINER_ID); + if (container) { + container.remove(); + } + } catch (error) { + // Ignore cleanup failures and continue with DOM extraction. + } + } + + /** + * Gets the position of an element in its parent. + * + * @param {HTMLElement} currentElement - The element to get the position for. + * @returns {number} The position of the element in its parent. + */ + function getElementPosition(currentElement) { + if (!currentElement.parentElement) { + return 0; // No parent means no siblings + } + + const tagName = currentElement.nodeName.toLowerCase(); + + const siblings = Array.from(currentElement.parentElement.children) + .filter((sib) => sib.nodeName.toLowerCase() === tagName); + + if (siblings.length === 1) { + return 0; // Only element of its type + } + + const index = siblings.indexOf(currentElement) + 1; // 1-based index + return index; + } + + + function getXPathTree(element, stopAtBoundary = true) { + if (xpathCache.has(element)) return xpathCache.get(element); + + const segments = []; + let currentElement = element; + + while (currentElement && currentElement.nodeType === Node.ELEMENT_NODE) { + // Stop if we hit a shadow root or iframe + if ( + stopAtBoundary && + (currentElement.parentNode instanceof ShadowRoot || + currentElement.parentNode instanceof HTMLIFrameElement) + ) { + break; + } + + const position = getElementPosition(currentElement); + const tagName = currentElement.nodeName.toLowerCase(); + const xpathIndex = position > 0 ? `[${position}]` : ""; + segments.unshift(`${tagName}${xpathIndex}`); + + currentElement = currentElement.parentNode; + } + + const result = segments.join("/"); + xpathCache.set(element, result); + return result; + } + + /** + * Checks if a text node is visible. + * + * @param {Text} textNode - The text node to check. + * @returns {boolean} Whether the text node is visible. + */ + function isTextNodeVisible(textNode) { + try { + // Special case: when viewportExpansion is -1, consider all text nodes as visible + if (viewportExpansion === -1) { + // Still check parent visibility for basic filtering + const parentElement = textNode.parentElement; + if (!parentElement) return false; + + try { + return parentElement.checkVisibility({ + checkOpacity: true, + checkVisibilityCSS: true, + }); + } catch (e) { + // Fallback if checkVisibility is not supported + const style = window.getComputedStyle(parentElement); + return style.display !== 'none' && + style.visibility !== 'hidden' && + style.opacity !== '0'; + } + } + + const range = document.createRange(); + range.selectNodeContents(textNode); + const rects = range.getClientRects(); // Use getClientRects for Range + + if (!rects || rects.length === 0) { + return false; + } + + let isAnyRectVisible = false; + let isAnyRectInViewport = false; + + for (const rect of rects) { + // Check size + if (rect.width > 0 && rect.height > 0) { + isAnyRectVisible = true; + + // Viewport check for this rect + if (!( + rect.bottom < -viewportExpansion || + rect.top > window.innerHeight + viewportExpansion || + rect.right < -viewportExpansion || + rect.left > window.innerWidth + viewportExpansion + )) { + isAnyRectInViewport = true; + break; // Found a visible rect in viewport, no need to check others + } + } + } + + if (!isAnyRectVisible || !isAnyRectInViewport) { + return false; + } + + // Check parent visibility + const parentElement = textNode.parentElement; + if (!parentElement) return false; + + try { + return parentElement.checkVisibility({ + checkOpacity: true, + checkVisibilityCSS: true, + }); + } catch (e) { + // Fallback if checkVisibility is not supported + const style = window.getComputedStyle(parentElement); + return style.display !== 'none' && + style.visibility !== 'hidden' && + style.opacity !== '0'; + } + } catch (e) { + console.warn('Error checking text node visibility:', e); + return false; + } + } + + /** + * Checks if an element is accepted. + * + * @param {HTMLElement} element - The element to check. + * @returns {boolean} Whether the element is accepted. + */ + function isElementAccepted(element) { + if (!element || !element.tagName) return false; + + // Always accept body and common container elements + const alwaysAccept = new Set([ + "body", "div", "main", "article", "section", "nav", "header", "footer" + ]); + const tagName = element.tagName.toLowerCase(); + + if (alwaysAccept.has(tagName)) return true; + + const leafElementDenyList = new Set([ + "svg", + "script", + "style", + "link", + "meta", + "noscript", + "template", + ]); + + return !leafElementDenyList.has(tagName); + } + + /** + * Checks if an element is visible. + * + * @param {HTMLElement} element - The element to check. + * @returns {boolean} Whether the element is visible. + */ + function isElementVisible(element) { + const style = getCachedComputedStyle(element); + return ( + element.offsetWidth > 0 && + element.offsetHeight > 0 && + style?.visibility !== "hidden" && + style?.display !== "none" + ); + } + + /** + * Checks if an element is interactive. + * + * lots of comments, and uncommented code - to show the logic of what we already tried + * + * One of the things we tried at the beginning was also to use event listeners, and other fancy class, style stuff -> what actually worked best was just combining most things with computed cursor style :) + * + * @param {HTMLElement} element - The element to check. + */ + function isInteractiveElement(element) { + if (!element || element.nodeType !== Node.ELEMENT_NODE) { + return false; + } + + // Cache the tagName and style lookups + const tagName = element.tagName.toLowerCase(); + const style = getCachedComputedStyle(element); + + // Define interactive cursors + const interactiveCursors = new Set([ + 'pointer', // Link/clickable elements + 'move', // Movable elements + 'text', // Text selection + 'grab', // Grabbable elements + 'grabbing', // Currently grabbing + 'cell', // Table cell selection + 'copy', // Copy operation + 'alias', // Alias creation + 'all-scroll', // Scrollable content + 'col-resize', // Column resize + 'context-menu', // Context menu available + 'crosshair', // Precise selection + 'e-resize', // East resize + 'ew-resize', // East-west resize + 'help', // Help available + 'n-resize', // North resize + 'ne-resize', // Northeast resize + 'nesw-resize', // Northeast-southwest resize + 'ns-resize', // North-south resize + 'nw-resize', // Northwest resize + 'nwse-resize', // Northwest-southeast resize + 'row-resize', // Row resize + 's-resize', // South resize + 'se-resize', // Southeast resize + 'sw-resize', // Southwest resize + 'vertical-text', // Vertical text selection + 'w-resize', // West resize + 'zoom-in', // Zoom in + 'zoom-out' // Zoom out + ]); + + // Define non-interactive cursors + const nonInteractiveCursors = new Set([ + 'not-allowed', // Action not allowed + 'no-drop', // Drop not allowed + 'wait', // Processing + 'progress', // In progress + 'initial', // Initial value + 'inherit' // Inherited value + //? Let's just include all potentially clickable elements that are not specifically blocked + // 'none', // No cursor + // 'default', // Default cursor + // 'auto', // Browser default + ]); + + /** + * Checks if an element has an interactive pointer. + * + * @param {HTMLElement} element - The element to check. + * @returns {boolean} Whether the element has an interactive pointer. + */ + function doesElementHaveInteractivePointer(element) { + if (element.tagName.toLowerCase() === "html") return false; + + if (style?.cursor && interactiveCursors.has(style.cursor)) return true; + + return false; + } + + let isInteractiveCursor = doesElementHaveInteractivePointer(element); + + // Genius fix for almost all interactive elements + if (isInteractiveCursor) { + return true; + } + + const interactiveElements = new Set([ + "a", // Links + "button", // Buttons + "input", // All input types (text, checkbox, radio, etc.) + "select", // Dropdown menus + "textarea", // Text areas + "details", // Expandable details + "summary", // Summary element (clickable part of details) + "label", // Form labels (often clickable) + "option", // Select options + "optgroup", // Option groups + "fieldset", // Form fieldsets (can be interactive with legend) + "legend", // Fieldset legends + ]); + + // Define explicit disable attributes and properties + const explicitDisableTags = new Set([ + 'disabled', // Standard disabled attribute + // 'aria-disabled', // ARIA disabled state + 'readonly', // Read-only state + // 'aria-readonly', // ARIA read-only state + // 'aria-hidden', // Hidden from accessibility + // 'hidden', // Hidden attribute + // 'inert', // Inert attribute + // 'aria-inert', // ARIA inert state + // 'tabindex="-1"', // Removed from tab order + // 'aria-hidden="true"' // Hidden from screen readers + ]); + + // handle inputs, select, checkbox, radio, textarea, button and make sure they are not cursor style disabled/not-allowed + if (interactiveElements.has(tagName)) { + // Check for non-interactive cursor + if (style?.cursor && nonInteractiveCursors.has(style.cursor)) { + return false; + } + + // Check for explicit disable attributes + for (const disableTag of explicitDisableTags) { + if (element.hasAttribute(disableTag) || + element.getAttribute(disableTag) === 'true' || + element.getAttribute(disableTag) === '') { + return false; + } + } + + // Check for disabled property on form elements + if (element.disabled) { + return false; + } + + // Check for readonly property on form elements + if (element.readOnly) { + return false; + } + + // Check for inert property + if (element.inert) { + return false; + } + + return true; + } + + const role = element.getAttribute("role"); + const ariaRole = element.getAttribute("aria-role"); + + // Check for contenteditable attribute + if (element.getAttribute("contenteditable") === "true" || element.isContentEditable) { + return true; + } + + // Added enhancement to capture dropdown interactive elements + if (element.classList && ( + element.classList.contains("button") || + element.classList.contains('dropdown-toggle') || + element.getAttribute('data-index') || + element.getAttribute('data-toggle') === 'dropdown' || + element.getAttribute('aria-haspopup') === 'true' + )) { + return true; + } + + + const interactiveRoles = new Set([ + 'button', // Directly clickable element + // 'link', // Clickable link + 'menu', // Menu container (ARIA menus) + 'menubar', // Menu bar container + 'menuitem', // Clickable menu item + 'menuitemradio', // Radio-style menu item (selectable) + 'menuitemcheckbox', // Checkbox-style menu item (toggleable) + 'radio', // Radio button (selectable) + 'checkbox', // Checkbox (toggleable) + 'tab', // Tab (clickable to switch content) + 'switch', // Toggle switch (clickable to change state) + 'slider', // Slider control (draggable) + 'spinbutton', // Number input with up/down controls + 'combobox', // Dropdown with text input + 'searchbox', // Search input field + 'textbox', // Text input field + 'listbox', // Selectable list + 'option', // Selectable option in a list + 'scrollbar' // Scrollable control + ]); + + + // Basic role/attribute checks + const hasInteractiveRole = + interactiveElements.has(tagName) || + (role && interactiveRoles.has(role)) || + (ariaRole && interactiveRoles.has(ariaRole)); + + if (hasInteractiveRole) return true; + + + // check whether element has event listeners by window.getEventListeners + try { + if (typeof getEventListeners === 'function') { + const listeners = getEventListeners(element); + const mouseEvents = ['click', 'mousedown', 'mouseup', 'dblclick']; + for (const eventType of mouseEvents) { + if (listeners[eventType] && listeners[eventType].length > 0) { + return true; // Found a mouse interaction listener + } + } + } + + const getEventListenersForNode = element?.ownerDocument?.defaultView?.getEventListenersForNode || window.getEventListenersForNode; + if (typeof getEventListenersForNode === 'function') { + const listeners = getEventListenersForNode(element); + const interactionEvents = ['click', 'mousedown', 'mouseup', 'keydown', 'keyup', 'submit', 'change', 'input', 'focus', 'blur']; + for (const eventType of interactionEvents) { + for (const listener of listeners) { + if (listener.type === eventType) { + return true; // Found a common interaction listener + } + } + } + } + // Fallback: Check common event attributes if getEventListeners is not available (getEventListeners doesn't work in page.evaluate context) + const commonMouseAttrs = ['onclick', 'onmousedown', 'onmouseup', 'ondblclick']; + for (const attr of commonMouseAttrs) { + if (element.hasAttribute(attr) || typeof element[attr] === 'function') { + return true; + } + } + } catch (e) { + // console.warn(`Could not check event listeners for ${element.tagName}:`, e); + // If checking listeners fails, rely on other checks + } + + return false + } + + + /** + * Checks if an element is the topmost element at its position. + * + * @param {HTMLElement} element - The element to check. + * @returns {boolean} Whether the element is the topmost element at its position. + */ + function isTopElement(element) { + // Special case: when viewportExpansion is -1, consider all elements as "top" elements + if (viewportExpansion === -1) { + return true; + } + + const rects = getCachedClientRects(element); // Replace element.getClientRects() + + if (!rects || rects.length === 0) { + return false; // No geometry, cannot be top + } + + let isAnyRectInViewport = false; + for (const rect of rects) { + // Use the same logic as isInExpandedViewport check + if (rect.width > 0 && rect.height > 0 && !( // Only check non-empty rects + rect.bottom < -viewportExpansion || + rect.top > window.innerHeight + viewportExpansion || + rect.right < -viewportExpansion || + rect.left > window.innerWidth + viewportExpansion + )) { + isAnyRectInViewport = true; + break; + } + } + + if (!isAnyRectInViewport) { + return false; // All rects are outside the viewport area + } + + + // Find the correct document context and root element + let doc = element.ownerDocument; + + // If we're in an iframe, elements are considered top by default + if (doc !== window.document) { + return true; + } + + // For shadow DOM, we need to check within its own root context + const shadowRoot = element.getRootNode(); + if (shadowRoot instanceof ShadowRoot) { + const centerX = rects[Math.floor(rects.length / 2)].left + rects[Math.floor(rects.length / 2)].width / 2; + const centerY = rects[Math.floor(rects.length / 2)].top + rects[Math.floor(rects.length / 2)].height / 2; + + try { + const topEl = shadowRoot.elementFromPoint(centerX, centerY); + if (!topEl) return false; + + let current = topEl; + while (current && current !== shadowRoot) { + if (current === element) return true; + current = current.parentElement; + } + return false; + } catch (e) { + return true; + } + } + + const margin = 5; + const rect = rects[Math.floor(rects.length / 2)]; + + // For elements in viewport, check if they're topmost. Do the check in the + // center of the element and at the corners to ensure we catch more cases. + const checkPoints = [ + // Initially only this was used, but it was not enough + { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }, + { x: rect.left + margin, y: rect.top + margin }, // top left + // { x: rect.right - margin, y: rect.top + margin }, // top right + // { x: rect.left + margin, y: rect.bottom - margin }, // bottom left + { x: rect.right - margin, y: rect.bottom - margin }, // bottom right + ]; + + return checkPoints.some(({ x, y }) => { + try { + const topEl = document.elementFromPoint(x, y); + if (!topEl) return false; + + let current = topEl; + while (current && current !== document.documentElement) { + if (current === element) return true; + current = current.parentElement; + } + return false; + } catch (e) { + return true; + } + }); + } + + /** + * Checks if an element is within the expanded viewport. + * + * @param {HTMLElement} element - The element to check. + * @param {number} viewportExpansion - The viewport expansion. + * @returns {boolean} Whether the element is within the expanded viewport. + */ + function isInExpandedViewport(element, viewportExpansion) { + if (viewportExpansion === -1) { + return true; + } + + const rects = element.getClientRects(); // Use getClientRects + + if (!rects || rects.length === 0) { + // Fallback to getBoundingClientRect if getClientRects is empty, + // useful for elements like that might not have client rects but have a bounding box. + const boundingRect = getCachedBoundingRect(element); + if (!boundingRect || boundingRect.width === 0 || boundingRect.height === 0) { + return false; + } + return !( + boundingRect.bottom < -viewportExpansion || + boundingRect.top > window.innerHeight + viewportExpansion || + boundingRect.right < -viewportExpansion || + boundingRect.left > window.innerWidth + viewportExpansion + ); + } + + // Check if *any* client rect is within the viewport + for (const rect of rects) { + if (rect.width === 0 || rect.height === 0) continue; // Skip empty rects + + if (!( + rect.bottom < -viewportExpansion || + rect.top > window.innerHeight + viewportExpansion || + rect.right < -viewportExpansion || + rect.left > window.innerWidth + viewportExpansion + )) { + return true; // Found at least one rect in the viewport + } + } + + return false; // No rects were found in the viewport + } + + // /** + // * Gets the effective scroll of an element. + // * + // * @param {HTMLElement} element - The element to get the effective scroll for. + // * @returns {Object} The effective scroll of the element. + // */ + // function getEffectiveScroll(element) { + // let currentEl = element; + // let scrollX = 0; + // let scrollY = 0; + + // while (currentEl && currentEl !== document.documentElement) { + // if (currentEl.scrollLeft || currentEl.scrollTop) { + // scrollX += currentEl.scrollLeft; + // scrollY += currentEl.scrollTop; + // } + // currentEl = currentEl.parentElement; + // } + + // scrollX += window.scrollX; + // scrollY += window.scrollY; + + // return { scrollX, scrollY }; + // } + + /** + * Checks if an element is an interactive candidate. + * + * @param {HTMLElement} element - The element to check. + * @returns {boolean} Whether the element is an interactive candidate. + */ + function isInteractiveCandidate(element) { + if (!element || element.nodeType !== Node.ELEMENT_NODE) return false; + + const tagName = element.tagName.toLowerCase(); + + // Fast-path for common interactive elements + const interactiveElements = new Set([ + "a", "button", "input", "select", "textarea", "details", "summary", "label" + ]); + + if (interactiveElements.has(tagName)) return true; + + // Quick attribute checks without getting full lists + const hasQuickInteractiveAttr = element.hasAttribute("onclick") || + element.hasAttribute("role") || + element.hasAttribute("tabindex") || + element.hasAttribute("aria-") || + element.hasAttribute("data-action") || + element.getAttribute("contenteditable") === "true"; + + return hasQuickInteractiveAttr; + } + + // --- Define constants for distinct interaction check --- + const DISTINCT_INTERACTIVE_TAGS = new Set([ + 'a', 'button', 'input', 'select', 'textarea', 'summary', 'details', 'label', 'option' + ]); + const INTERACTIVE_ROLES = new Set([ + 'button', 'link', 'menuitem', 'menuitemradio', 'menuitemcheckbox', + 'radio', 'checkbox', 'tab', 'switch', 'slider', 'spinbutton', + 'combobox', 'searchbox', 'textbox', 'listbox', 'option', 'scrollbar' + ]); + + + /** + * Heuristically determines if an element should be considered as independently interactive, + * even if it's nested inside another interactive container. + * + * This function helps detect deeply nested actionable elements (e.g., menu items within a button) + * that may not be picked up by strict interactivity checks. + * + * @param {HTMLElement} element - The element to check. + * @returns {boolean} Whether the element is heuristically interactive. + */ + function isHeuristicallyInteractive(element) { + if (!element || element.nodeType !== Node.ELEMENT_NODE) return false; + + // Skip non-visible elements early for performance + if (!isElementVisible(element)) return false; + + // Check for common attributes that often indicate interactivity + const hasInteractiveAttributes = + element.hasAttribute('role') || + element.hasAttribute('tabindex') || + element.hasAttribute('onclick') || + typeof element.onclick === 'function'; + + // Check for semantic class names suggesting interactivity + const hasInteractiveClass = /\b(btn|clickable|menu|item|entry|link)\b/i.test(element.className || ''); + + // Determine whether the element is inside a known interactive container + const isInKnownContainer = Boolean( + element.closest('button,a,[role="button"],.menu,.dropdown,.list,.toolbar') + ); + + // Ensure the element has at least one visible child (to avoid marking empty wrappers) + const hasVisibleChildren = [...element.children].some(isElementVisible); + + // Avoid highlighting elements whose parent is (top-level wrappers) + const isParentBody = element.parentElement && element.parentElement.isSameNode(document.body); + + return ( + (isInteractiveElement(element) || hasInteractiveAttributes || hasInteractiveClass) && + hasVisibleChildren && + isInKnownContainer && + !isParentBody + ); + } + + + /** + * Checks if an element likely represents a distinct interaction + * separate from its parent (if the parent is also interactive). + * + * @param {HTMLElement} element - The element to check. + * @returns {boolean} Whether the element is a distinct interaction. + */ + function isElementDistinctInteraction(element) { + if (!element || element.nodeType !== Node.ELEMENT_NODE) { + return false; + } + + const tagName = element.tagName.toLowerCase(); + const role = element.getAttribute('role'); + + // Check if it's an iframe - always distinct boundary + if (tagName === 'iframe') { + return true; + } + + // Check tag name + if (DISTINCT_INTERACTIVE_TAGS.has(tagName)) { + return true; + } + // Check interactive roles + if (role && INTERACTIVE_ROLES.has(role)) { + return true; + } + // Check contenteditable + if (element.isContentEditable || element.getAttribute('contenteditable') === 'true') { + return true; + } + // Check for common testing/automation attributes + if (element.hasAttribute('data-testid') || element.hasAttribute('data-cy') || element.hasAttribute('data-test')) { + return true; + } + // Check for explicit onclick handler (attribute or property) + if (element.hasAttribute('onclick') || typeof element.onclick === 'function') { + return true; + } + + // return false + + // Check for other common interaction event listeners + try { + const getEventListenersForNode = element?.ownerDocument?.defaultView?.getEventListenersForNode || window.getEventListenersForNode; + if (typeof getEventListenersForNode === 'function') { + const listeners = getEventListenersForNode(element); + const interactionEvents = ['click', 'mousedown', 'mouseup', 'keydown', 'keyup', 'submit', 'change', 'input', 'focus', 'blur']; + for (const eventType of interactionEvents) { + for (const listener of listeners) { + if (listener.type === eventType) { + return true; // Found a common interaction listener + } + } + } + } + // Fallback: Check common event attributes if getEventListeners is not available (getEventListenersForNode doesn't work in page.evaluate context) + const commonEventAttrs = ['onmousedown', 'onmouseup', 'onkeydown', 'onkeyup', 'onsubmit', 'onchange', 'oninput', 'onfocus', 'onblur']; + if (commonEventAttrs.some(attr => element.hasAttribute(attr))) { + return true; + } + } catch (e) { + // console.warn(`Could not check event listeners for ${element.tagName}:`, e); + // If checking listeners fails, rely on other checks + } + + + + // if the element is not strictly interactive but appears clickable based on heuristic signals + if (isHeuristicallyInteractive(element)) { + return true; + } + + // Default to false: if it's interactive but doesn't match above, + // assume it triggers the same action as the parent. + return false; + } + // --- End distinct interaction check --- + + /** + * Handles the logic for deciding whether to highlight an element and performing the highlight. + * @param { + { + tagName: string; + attributes: Record; + xpath: any; + children: never[]; + isVisible?: boolean; + isTopElement?: boolean; + isInteractive?: boolean; + isInViewport?: boolean; + highlightIndex?: number; + shadowRoot?: boolean; + }} nodeData - The node data object. + * @param {HTMLElement} node - The node to highlight. + * @param {HTMLElement | null} parentIframe - The parent iframe node. + * @param {boolean} isParentHighlighted - Whether the parent node is highlighted. + * @returns {boolean} Whether the element was highlighted. + */ + function handleHighlighting(nodeData, node, parentIframe, isParentHighlighted) { + if (!nodeData.isInteractive) return false; // Not interactive, definitely don't highlight + + let shouldHighlight = false; + if (!isParentHighlighted) { + // Parent wasn't highlighted, this interactive node can be highlighted. + shouldHighlight = true; + } else { + // Parent *was* highlighted. Only highlight this node if it represents a distinct interaction. + if (isElementDistinctInteraction(node)) { + shouldHighlight = true; + } else { + // console.log(`Skipping highlight for ${nodeData.tagName} (parent highlighted)`); + shouldHighlight = false; + } + } + + if (shouldHighlight) { + // Check viewport status before assigning index and highlighting + nodeData.isInViewport = isInExpandedViewport(node, viewportExpansion); + + // When viewportExpansion is -1, all interactive elements should get a highlight index + // regardless of viewport status + if (nodeData.isInViewport || viewportExpansion === -1) { + nodeData.highlightIndex = highlightIndex++; + + if (doHighlightElements) { + if (focusHighlightIndex >= 0) { + if (focusHighlightIndex === nodeData.highlightIndex) { + highlightElement(node, nodeData.highlightIndex, parentIframe); + } + } else { + highlightElement(node, nodeData.highlightIndex, parentIframe); + } + return true; // Successfully highlighted + } + } else { + // console.log(`Skipping highlight for ${nodeData.tagName} (outside viewport)`); + } + } + + return false; // Did not highlight + } + + /** + * Creates a node data object for a given node and its descendants. + * + * @param {HTMLElement} node - The node to process. + * @param {HTMLElement | null} parentIframe - The parent iframe node. + * @param {boolean} isParentHighlighted - Whether the parent node is highlighted. + * @returns {string | null} The ID of the node data object, or null if the node is not processed. + */ + function buildDomTree(node, parentIframe = null, isParentHighlighted = false) { + // Fast rejection checks first + if (!node || node.id === HIGHLIGHT_CONTAINER_ID || + (node.nodeType !== Node.ELEMENT_NODE && node.nodeType !== Node.TEXT_NODE)) { + return null; + } + + if (!node || node.id === HIGHLIGHT_CONTAINER_ID) { + return null; + } + + // Special handling for root node (body) + if (node === document.body) { + const nodeData = { + tagName: 'body', + attributes: {}, + xpath: '/body', + children: [], + }; + + // Process children of body + for (const child of node.childNodes) { + const domElement = buildDomTree(child, parentIframe, false); // Body's children have no highlighted parent initially + if (domElement) nodeData.children.push(domElement); + } + + const id = `${ID.current++}`; + DOM_HASH_MAP[id] = nodeData; + return id; + } + + // Early bailout for non-element nodes except text + if (node.nodeType !== Node.ELEMENT_NODE && node.nodeType !== Node.TEXT_NODE) { + return null; + } + + // Process text nodes + if (node.nodeType === Node.TEXT_NODE) { + const textContent = node.textContent?.trim(); + if (!textContent) { + return null; + } + + // Only check visibility for text nodes that might be visible + const parentElement = node.parentElement; + if (!parentElement || parentElement.tagName.toLowerCase() === 'script') { + return null; + } + + const id = `${ID.current++}`; + DOM_HASH_MAP[id] = { + type: "TEXT_NODE", + text: textContent, + isVisible: isTextNodeVisible(node), + }; + return id; + } + + // Quick checks for element nodes + if (node.nodeType === Node.ELEMENT_NODE && !isElementAccepted(node)) { + return null; + } + + // Early viewport check - only filter out elements clearly outside viewport + // The getBoundingClientRect() of the Shadow DOM host element may return width/height = 0 + if (viewportExpansion !== -1 && !node.shadowRoot) { + const rect = getCachedBoundingRect(node); // Keep for initial quick check + const style = getCachedComputedStyle(node); + + // Skip viewport check for fixed/sticky elements as they may appear anywhere + const isFixedOrSticky = style && (style.position === 'fixed' || style.position === 'sticky'); + + // Check if element has actual dimensions using offsetWidth/Height (quick check) + const hasSize = node.offsetWidth > 0 || node.offsetHeight > 0; + + // Use getBoundingClientRect for the quick OUTSIDE check. + // isInExpandedViewport will do the more accurate check later if needed. + if (!rect || (!isFixedOrSticky && !hasSize && ( + rect.bottom < -viewportExpansion || + rect.top > window.innerHeight + viewportExpansion || + rect.right < -viewportExpansion || + rect.left > window.innerWidth + viewportExpansion + ))) { + // console.log("Skipping node outside viewport (quick check):", node.tagName, rect); + return null; + } + } + + /** + * @type { + { + tagName: string; + attributes: Record; + xpath: any; + children: never[]; + isVisible?: boolean; + isTopElement?: boolean; + isInteractive?: boolean; + isInViewport?: boolean; + highlightIndex?: number; + shadowRoot?: boolean; + } + } nodeData - The node data object. + */ + const nodeData = { + tagName: node.tagName.toLowerCase(), + attributes: {}, + xpath: getXPathTree(node, true), + children: [], + }; + + // Get attributes for interactive elements or potential text containers + if (isInteractiveCandidate(node) || node.tagName.toLowerCase() === 'iframe' || node.tagName.toLowerCase() === 'body') { + const attributeNames = node.getAttributeNames?.() || []; + for (const name of attributeNames) { + const value = node.getAttribute(name); + nodeData.attributes[name] = value; + } + } + + let nodeWasHighlighted = false; + // Perform visibility, interactivity, and highlighting checks + if (node.nodeType === Node.ELEMENT_NODE) { + nodeData.isVisible = isElementVisible(node); // isElementVisible uses offsetWidth/Height, which is fine + if (nodeData.isVisible) { + nodeData.isTopElement = isTopElement(node); + + // Special handling for ARIA menu containers - check interactivity even if not top element + const role = node.getAttribute('role'); + const isMenuContainer = role === 'menu' || role === 'menubar' || role === 'listbox'; + + if (nodeData.isTopElement || isMenuContainer) { + nodeData.isInteractive = isInteractiveElement(node); + // Call the dedicated highlighting function + nodeWasHighlighted = handleHighlighting(nodeData, node, parentIframe, isParentHighlighted); + } + } + } + + // Process children, with special handling for iframes and rich text editors + if (node.tagName) { + const tagName = node.tagName.toLowerCase(); + + // Handle iframes + if (tagName === "iframe") { + try { + const iframeDoc = node.contentDocument || node.contentWindow?.document; + if (iframeDoc) { + for (const child of iframeDoc.childNodes) { + const domElement = buildDomTree(child, node, false); + if (domElement) nodeData.children.push(domElement); + } + } + } catch (e) { + console.warn("Unable to access iframe:", e); + } + } + // Handle rich text editors and contenteditable elements + else if ( + node.isContentEditable || + node.getAttribute("contenteditable") === "true" || + node.id === "tinymce" || + node.classList.contains("mce-content-body") || + (tagName === "body" && node.getAttribute("data-id")?.startsWith("mce_")) + ) { + // Process all child nodes to capture formatted text + for (const child of node.childNodes) { + const domElement = buildDomTree(child, parentIframe, nodeWasHighlighted); + if (domElement) nodeData.children.push(domElement); + } + } + else { + // Handle shadow DOM + if (node.shadowRoot) { + nodeData.shadowRoot = true; + for (const child of node.shadowRoot.childNodes) { + const domElement = buildDomTree(child, parentIframe, nodeWasHighlighted); + if (domElement) nodeData.children.push(domElement); + } + } + // Handle regular elements + for (const child of node.childNodes) { + // Pass the highlighted status of the *current* node to its children + const passHighlightStatusToChild = nodeWasHighlighted || isParentHighlighted; + const domElement = buildDomTree(child, parentIframe, passHighlightStatusToChild); + if (domElement) nodeData.children.push(domElement); + } + } + } + + // Skip empty anchor tags only if they have no dimensions and no children + if (nodeData.tagName === 'a' && nodeData.children.length === 0 && !nodeData.attributes.href) { + // Check if the anchor has actual dimensions + const rect = getCachedBoundingRect(node); + const hasSize = (rect && rect.width > 0 && rect.height > 0) || (node.offsetWidth > 0 || node.offsetHeight > 0); + + if (!hasSize) { + return null; + } + } + + const id = `${ID.current++}`; + DOM_HASH_MAP[id] = nodeData; + return id; + } + + cleanupHighlights(); + const rootId = buildDomTree(document.body); + + // Clear the cache before starting + DOM_CACHE.clearCache(); + + return { rootId, map: DOM_HASH_MAP }; +}; diff --git a/dist/dom/history-tree-processor/service.d.ts b/dist/dom/history-tree-processor/service.d.ts new file mode 100644 index 00000000..5034b5ab --- /dev/null +++ b/dist/dom/history-tree-processor/service.d.ts @@ -0,0 +1,19 @@ +import { DOMHistoryElement, HashedDomElement } from './view.js'; +import { DOMElementNode } from '../views.js'; +export declare class HistoryTreeProcessor { + static get_accessible_name(dom_element: DOMElementNode): string | null; + static compute_element_hash(dom_element: DOMElementNode): string; + static compute_stable_hash(dom_element: DOMElementNode): string; + static _filter_dynamic_classes(class_value: string): string; + static _compute_element_hash(dom_element: DOMElementNode, stable: boolean): string; + static convert_dom_element_to_history_element(dom_element: DOMElementNode, css_selector?: string | null): DOMHistoryElement; + static find_history_element_in_tree(dom_history_element: DOMHistoryElement, tree: DOMElementNode): DOMElementNode | null; + static compare_history_element_and_dom_element(dom_history_element: DOMHistoryElement, dom_element: DOMElementNode): boolean; + static _hash_dom_history_element(dom_history_element: DOMHistoryElement): HashedDomElement; + static _hash_dom_element(dom_element: DOMElementNode): HashedDomElement; + static _get_parent_branch_path(dom_element: DOMElementNode): string[]; + static _parent_branch_path_hash(parent_branch_path: string[]): string; + static _attributes_hash(attributes: Record): string; + static _xpath_hash(xpath: string): string; + static _text_hash(dom_element: DOMElementNode): string; +} diff --git a/dist/dom/history-tree-processor/service.js b/dist/dom/history-tree-processor/service.js new file mode 100644 index 00000000..f2246a54 --- /dev/null +++ b/dist/dom/history-tree-processor/service.js @@ -0,0 +1,230 @@ +import crypto from 'node:crypto'; +import { DOMHistoryElement, HashedDomElement } from './view.js'; +import { DOMElementNode } from '../views.js'; +const sha256 = (value) => crypto.createHash('sha256').update(value).digest('hex'); +const STATIC_HASH_ATTRIBUTES = new Set([ + 'class', + 'id', + 'name', + 'type', + 'placeholder', + 'aria-label', + 'title', + 'role', + 'data-testid', + 'data-test', + 'data-cy', + 'for', + 'required', + 'disabled', + 'readonly', + 'checked', + 'selected', + 'multiple', + 'href', + 'target', + 'rel', + 'aria-describedby', + 'aria-labelledby', + 'aria-controls', + 'aria-owns', + 'aria-live', + 'aria-atomic', + 'aria-busy', + 'aria-disabled', + 'aria-hidden', + 'aria-pressed', + 'aria-autocomplete', + 'aria-checked', + 'aria-selected', + 'list', + 'tabindex', + 'alt', + 'src', + 'lang', + 'itemscope', + 'itemtype', + 'itemprop', + 'aria-valuemin', + 'aria-valuemax', + 'aria-valuenow', + 'aria-placeholder', +]); +const DYNAMIC_CLASS_PATTERNS = [ + 'focus', + 'hover', + 'active', + 'selected', + 'disabled', + 'animation', + 'transition', + 'loading', + 'open', + 'closed', + 'expanded', + 'collapsed', + 'visible', + 'hidden', + 'pressed', + 'checked', + 'highlighted', + 'current', + 'entering', + 'leaving', +]; +export class HistoryTreeProcessor { + static get_accessible_name(dom_element) { + const ariaLabel = dom_element.attributes?.['aria-label']?.trim(); + if (ariaLabel) { + return ariaLabel; + } + const title = dom_element.attributes?.title?.trim(); + if (title) { + return title; + } + const text = dom_element.get_all_text_till_next_clickable_element().trim(); + return text ? text : null; + } + static compute_element_hash(dom_element) { + return this._compute_element_hash(dom_element, false); + } + static compute_stable_hash(dom_element) { + return this._compute_element_hash(dom_element, true); + } + static _filter_dynamic_classes(class_value) { + const classes = class_value + .split(/\s+/) + .map((value) => value.trim()) + .filter(Boolean); + const stable = classes.filter((cls) => !DYNAMIC_CLASS_PATTERNS.some((pattern) => cls.toLowerCase().includes(pattern))); + return stable.sort().join(' '); + } + static _compute_element_hash(dom_element, stable) { + const parent_branch_path = this._get_parent_branch_path(dom_element); + const parent_branch_path_string = parent_branch_path.join('/'); + const normalized_attributes = Object.entries(dom_element.attributes ?? {}) + .filter(([key, value]) => { + if (!STATIC_HASH_ATTRIBUTES.has(key)) { + return false; + } + return String(value).trim().length > 0; + }) + .map(([key, value]) => { + if (stable && key === 'class') { + const filtered = this._filter_dynamic_classes(String(value)); + return [key, filtered]; + } + return [key, String(value)]; + }) + .filter(([, value]) => value.length > 0) + .sort(([a], [b]) => a.localeCompare(b)); + const attributes_string = normalized_attributes + .map(([key, value]) => `${key}=${value}`) + .join(''); + const ax_name = this.get_accessible_name(dom_element); + const ax_suffix = ax_name ? `|ax_name=${ax_name}` : ''; + const combined_string = `${parent_branch_path_string}|${attributes_string}${ax_suffix}`; + return sha256(combined_string).slice(0, 16); + } + static convert_dom_element_to_history_element(dom_element, css_selector = null) { + const parent_branch_path = this._get_parent_branch_path(dom_element); + const axName = this.get_accessible_name(dom_element); + const elementHash = this.compute_element_hash(dom_element); + const stableHash = this.compute_stable_hash(dom_element); + return new DOMHistoryElement(dom_element.tag_name, dom_element.xpath, dom_element.highlight_index ?? null, parent_branch_path, dom_element.attributes, dom_element.shadow_root, css_selector, dom_element.page_coordinates, dom_element.viewport_coordinates, dom_element.viewport_info, elementHash, stableHash, axName); + } + static find_history_element_in_tree(dom_history_element, tree) { + const process_node = (node, matcher) => { + if (node.highlight_index !== null && + node.highlight_index !== undefined && + matcher(node)) { + return node; + } + for (const child of node.children) { + if (!(child instanceof DOMElementNode)) { + continue; + } + const result = process_node(child, matcher); + if (result) { + return result; + } + } + return null; + }; + if (dom_history_element.element_hash) { + const exact = process_node(tree, (candidate) => this.compute_element_hash(candidate) === + dom_history_element.element_hash); + if (exact) { + return exact; + } + } + if (dom_history_element.stable_hash) { + const stable = process_node(tree, (candidate) => this.compute_stable_hash(candidate) === + dom_history_element.stable_hash); + if (stable) { + return stable; + } + } + const hashed = this._hash_dom_history_element(dom_history_element); + return process_node(tree, (candidate) => { + const hashed_node = this._hash_dom_element(candidate); + return hashed_node.equals(hashed); + }); + } + static compare_history_element_and_dom_element(dom_history_element, dom_element) { + if (dom_history_element.element_hash) { + if (this.compute_element_hash(dom_element) === + dom_history_element.element_hash) { + return true; + } + } + if (dom_history_element.stable_hash) { + if (this.compute_stable_hash(dom_element) === + dom_history_element.stable_hash) { + return true; + } + } + const hashed_history = this._hash_dom_history_element(dom_history_element); + const hashed_dom = this._hash_dom_element(dom_element); + return hashed_history.equals(hashed_dom); + } + static _hash_dom_history_element(dom_history_element) { + const branch_path_hash = this._parent_branch_path_hash(dom_history_element.entire_parent_branch_path); + const attributes_hash = this._attributes_hash(dom_history_element.attributes); + const xpath_hash = this._xpath_hash(dom_history_element.xpath); + return new HashedDomElement(branch_path_hash, attributes_hash, xpath_hash); + } + static _hash_dom_element(dom_element) { + const parent_branch_path = this._get_parent_branch_path(dom_element); + const branch_path_hash = this._parent_branch_path_hash(parent_branch_path); + const attributes_hash = this._attributes_hash(dom_element.attributes); + const xpath_hash = this._xpath_hash(dom_element.xpath); + return new HashedDomElement(branch_path_hash, attributes_hash, xpath_hash); + } + static _get_parent_branch_path(dom_element) { + const parents = []; + let current = dom_element; + while (current && current.parent) { + parents.push(current); + current = current.parent; + } + parents.reverse(); + return parents.map((parent) => parent.tag_name); + } + static _parent_branch_path_hash(parent_branch_path) { + return sha256(parent_branch_path.join('/')); + } + static _attributes_hash(attributes) { + const attributes_string = Object.entries(attributes) + .map(([key, value]) => `${key}=${value}`) + .join(''); + return sha256(attributes_string); + } + static _xpath_hash(xpath) { + return sha256(xpath); + } + static _text_hash(dom_element) { + const text = dom_element.get_all_text_till_next_clickable_element(); + return sha256(text); + } +} diff --git a/dist/dom/history-tree-processor/view.d.ts b/dist/dom/history-tree-processor/view.d.ts new file mode 100644 index 00000000..8bf56c84 --- /dev/null +++ b/dist/dom/history-tree-processor/view.d.ts @@ -0,0 +1,60 @@ +export declare class HashedDomElement { + branch_path_hash: string; + attributes_hash: string; + xpath_hash: string; + constructor(branch_path_hash: string, attributes_hash: string, xpath_hash: string); + /** + * Check equality with another HashedDomElement + */ + equals(other: HashedDomElement): boolean; +} +export interface Coordinates { + x: number; + y: number; +} +export interface CoordinateSet { + top_left: Coordinates; + top_right: Coordinates; + bottom_left: Coordinates; + bottom_right: Coordinates; + center: Coordinates; + width: number; + height: number; +} +export interface ViewportInfo { + scroll_x?: number | null; + scroll_y?: number | null; + width: number; + height: number; +} +export declare class DOMHistoryElement { + tag_name: string; + xpath: string; + highlight_index: number | null; + entire_parent_branch_path: string[]; + attributes: Record; + shadow_root: boolean; + css_selector: string | null; + page_coordinates: CoordinateSet | null; + viewport_coordinates: CoordinateSet | null; + viewport_info: ViewportInfo | null; + element_hash: string | null; + stable_hash: string | null; + ax_name: string | null; + constructor(tag_name: string, xpath: string, highlight_index: number | null, entire_parent_branch_path: string[], attributes: Record, shadow_root?: boolean, css_selector?: string | null, page_coordinates?: CoordinateSet | null, viewport_coordinates?: CoordinateSet | null, viewport_info?: ViewportInfo | null, element_hash?: string | null, stable_hash?: string | null, ax_name?: string | null); + to_dict(): { + tag_name: string; + xpath: string; + highlight_index: number | null; + entire_parent_branch_path: string[]; + attributes: Record; + shadow_root: boolean; + css_selector: string | null; + page_coordinates: CoordinateSet | null; + viewport_coordinates: CoordinateSet | null; + viewport_info: ViewportInfo | null; + element_hash: string | null; + stable_hash: string | null; + ax_name: string | null; + }; +} diff --git a/dist/dom/history-tree-processor/view.js b/dist/dom/history-tree-processor/view.js new file mode 100644 index 00000000..9bfd431c --- /dev/null +++ b/dist/dom/history-tree-processor/view.js @@ -0,0 +1,65 @@ +export class HashedDomElement { + branch_path_hash; + attributes_hash; + xpath_hash; + constructor(branch_path_hash, attributes_hash, xpath_hash) { + this.branch_path_hash = branch_path_hash; + this.attributes_hash = attributes_hash; + this.xpath_hash = xpath_hash; + } + /** + * Check equality with another HashedDomElement + */ + equals(other) { + return (this.branch_path_hash === other.branch_path_hash && + this.attributes_hash === other.attributes_hash && + this.xpath_hash === other.xpath_hash); + } +} +export class DOMHistoryElement { + tag_name; + xpath; + highlight_index; + entire_parent_branch_path; + attributes; + shadow_root; + css_selector; + page_coordinates; + viewport_coordinates; + viewport_info; + element_hash; + stable_hash; + ax_name; + constructor(tag_name, xpath, highlight_index, entire_parent_branch_path, attributes, shadow_root = false, css_selector = null, page_coordinates = null, viewport_coordinates = null, viewport_info = null, element_hash = null, stable_hash = null, ax_name = null) { + this.tag_name = tag_name; + this.xpath = xpath; + this.highlight_index = highlight_index; + this.entire_parent_branch_path = entire_parent_branch_path; + this.attributes = attributes; + this.shadow_root = shadow_root; + this.css_selector = css_selector; + this.page_coordinates = page_coordinates; + this.viewport_coordinates = viewport_coordinates; + this.viewport_info = viewport_info; + this.element_hash = element_hash; + this.stable_hash = stable_hash; + this.ax_name = ax_name; + } + to_dict() { + return { + tag_name: this.tag_name, + xpath: this.xpath, + highlight_index: this.highlight_index, + entire_parent_branch_path: this.entire_parent_branch_path, + attributes: this.attributes, + shadow_root: this.shadow_root, + css_selector: this.css_selector, + page_coordinates: this.page_coordinates, + viewport_coordinates: this.viewport_coordinates, + viewport_info: this.viewport_info, + element_hash: this.element_hash, + stable_hash: this.stable_hash, + ax_name: this.ax_name, + }; + } +} diff --git a/dist/dom/markdown-extractor.d.ts b/dist/dom/markdown-extractor.d.ts new file mode 100644 index 00000000..4c3542d8 --- /dev/null +++ b/dist/dom/markdown-extractor.d.ts @@ -0,0 +1,37 @@ +export interface MarkdownContentStats { + method: string; + original_html_chars: number; + initial_markdown_chars: number; + filtered_chars_removed: number; + final_filtered_chars: number; + url?: string; + started_from_char?: number; + truncated_at_char?: number; + next_start_char?: number; + chunk_index?: number; + total_chunks?: number; +} +export interface MarkdownChunk { + content: string; + chunk_index: number; + total_chunks: number; + char_offset_start: number; + char_offset_end: number; + overlap_prefix: string; + has_more: boolean; +} +interface ExtractCleanMarkdownOptions { + extract_links?: boolean; + method?: string; + url?: string; +} +export declare const preprocessMarkdownContent: (input: string, maxNewlines?: number) => { + content: string; + chars_filtered: number; +}; +export declare const extractCleanMarkdownFromHtml: (html: string, options?: ExtractCleanMarkdownOptions) => { + content: string; + stats: MarkdownContentStats; +}; +export declare const chunkMarkdownByStructure: (content: string, maxChunkChars?: number, overlapLines?: number, startFromChar?: number) => MarkdownChunk[]; +export {}; diff --git a/dist/dom/markdown-extractor.js b/dist/dom/markdown-extractor.js new file mode 100644 index 00000000..fe4b2f5f --- /dev/null +++ b/dist/dom/markdown-extractor.js @@ -0,0 +1,345 @@ +import TurndownService from 'turndown'; +var BlockType; +(function (BlockType) { + BlockType["Header"] = "header"; + BlockType["CodeFence"] = "code_fence"; + BlockType["Table"] = "table"; + BlockType["ListItem"] = "list_item"; + BlockType["Paragraph"] = "paragraph"; + BlockType["Blank"] = "blank"; +})(BlockType || (BlockType = {})); +const TABLE_ROW_RE = /^\s*\|.*\|\s*$/; +const LIST_ITEM_RE = /^(\s*)([-*+]|\d+[.)]) /; +const LIST_CONTINUATION_RE = /^(\s{2,}|\t)/; +const getBlockSize = (block) => block.char_end - block.char_start; +const blockText = (block) => block.lines.join('\n'); +const getTableHeader = (block) => { + if (block.block_type !== BlockType.Table || block.lines.length < 2) { + return null; + } + const separator = block.lines[1] ?? ''; + if (separator.includes('---') || separator.includes('- -')) { + return `${block.lines[0]}\n${separator}`; + } + return null; +}; +export const preprocessMarkdownContent = (input, maxNewlines = 3) => { + const originalLength = input.length; + let content = input; + content = content.replace(/`\{["\w][\s\S]*?\}`/g, ''); + content = content.replace(/\{"\$type":[^}]{100,}\}/g, ''); + content = content.replace(/\{"[^"]{5,}":\{[^}]{100,}\}/g, ''); + content = content.replace(/\n{4,}/g, '\n'.repeat(maxNewlines)); + const filteredLines = []; + for (const line of content.split('\n')) { + const stripped = line.trim(); + if (!stripped) { + continue; + } + if ((stripped.startsWith('{') || stripped.startsWith('[')) && + stripped.length > 100) { + continue; + } + filteredLines.push(line); + } + content = filteredLines.join('\n').trim(); + return { + content, + chars_filtered: originalLength - content.length, + }; +}; +export const extractCleanMarkdownFromHtml = (html, options = {}) => { + const method = options.method ?? 'html'; + const extractLinks = options.extract_links ?? false; + let pageHtml = html; + if (!extractLinks) { + pageHtml = pageHtml + .replace(/]*>/gi, '') + .replace(/<\/a>/gi, '') + .replace(/]*>/gi, ''); + } + const originalHtmlLength = pageHtml.length; + const turndown = new TurndownService({ + headingStyle: 'atx', + codeBlockStyle: 'fenced', + bulletListMarker: '-', + }); + turndown.remove(['script', 'style']); + let content = turndown.turndown(pageHtml); + const initialMarkdownLength = content.length; + content = content.replace(/%[0-9A-Fa-f]{2}/g, ''); + const preprocessed = preprocessMarkdownContent(content); + content = preprocessed.content; + const stats = { + method, + original_html_chars: originalHtmlLength, + initial_markdown_chars: initialMarkdownLength, + filtered_chars_removed: preprocessed.chars_filtered, + final_filtered_chars: content.length, + }; + if (options.url) { + stats.url = options.url; + } + return { content, stats }; +}; +const parseAtomicBlocks = (content) => { + const lines = content.split('\n'); + const blocks = []; + let i = 0; + let offset = 0; + while (i < lines.length) { + const line = lines[i] ?? ''; + const lineLength = line.length + 1; + if (!line.trim()) { + blocks.push({ + block_type: BlockType.Blank, + lines: [line], + char_start: offset, + char_end: offset + lineLength, + }); + offset += lineLength; + i += 1; + continue; + } + if (line.trim().startsWith('```')) { + const fenceLines = [line]; + let fenceEnd = offset + lineLength; + i += 1; + while (i < lines.length) { + const fenceLine = lines[i] ?? ''; + const fenceLineLength = fenceLine.length + 1; + fenceLines.push(fenceLine); + fenceEnd += fenceLineLength; + i += 1; + if (fenceLine.trim().startsWith('```') && fenceLines.length > 1) { + break; + } + } + blocks.push({ + block_type: BlockType.CodeFence, + lines: fenceLines, + char_start: offset, + char_end: fenceEnd, + }); + offset = fenceEnd; + continue; + } + if (line.trimStart().startsWith('#')) { + blocks.push({ + block_type: BlockType.Header, + lines: [line], + char_start: offset, + char_end: offset + lineLength, + }); + offset += lineLength; + i += 1; + continue; + } + if (TABLE_ROW_RE.test(line)) { + const headerLines = [line]; + let headerEnd = offset + lineLength; + i += 1; + if (i < lines.length && + TABLE_ROW_RE.test(lines[i] ?? '') && + (lines[i] ?? '').includes('---')) { + const separator = lines[i] ?? ''; + const separatorLength = separator.length + 1; + headerLines.push(separator); + headerEnd += separatorLength; + i += 1; + } + blocks.push({ + block_type: BlockType.Table, + lines: headerLines, + char_start: offset, + char_end: headerEnd, + }); + offset = headerEnd; + while (i < lines.length && TABLE_ROW_RE.test(lines[i] ?? '')) { + const row = lines[i] ?? ''; + const rowLength = row.length + 1; + blocks.push({ + block_type: BlockType.Table, + lines: [row], + char_start: offset, + char_end: offset + rowLength, + }); + offset += rowLength; + i += 1; + } + continue; + } + if (LIST_ITEM_RE.test(line)) { + const listLines = [line]; + let listEnd = offset + lineLength; + i += 1; + while (i < lines.length) { + const nextLine = lines[i] ?? ''; + const nextLineLength = nextLine.length + 1; + if (LIST_ITEM_RE.test(nextLine)) { + listLines.push(nextLine); + listEnd += nextLineLength; + i += 1; + continue; + } + if (nextLine.trim() && LIST_CONTINUATION_RE.test(nextLine)) { + listLines.push(nextLine); + listEnd += nextLineLength; + i += 1; + continue; + } + break; + } + blocks.push({ + block_type: BlockType.ListItem, + lines: listLines, + char_start: offset, + char_end: listEnd, + }); + offset = listEnd; + continue; + } + const paragraphLines = [line]; + let paragraphEnd = offset + lineLength; + i += 1; + while (i < lines.length && (lines[i] ?? '').trim()) { + const nextLine = lines[i] ?? ''; + if (nextLine.trimStart().startsWith('#') || + nextLine.trim().startsWith('```') || + TABLE_ROW_RE.test(nextLine) || + LIST_ITEM_RE.test(nextLine)) { + break; + } + const nextLineLength = nextLine.length + 1; + paragraphLines.push(nextLine); + paragraphEnd += nextLineLength; + i += 1; + } + blocks.push({ + block_type: BlockType.Paragraph, + lines: paragraphLines, + char_start: offset, + char_end: paragraphEnd, + }); + offset = paragraphEnd; + } + if (blocks.length > 0 && content && !content.endsWith('\n')) { + const last = blocks[blocks.length - 1]; + blocks[blocks.length - 1] = { + ...last, + char_end: content.length, + }; + } + return blocks; +}; +export const chunkMarkdownByStructure = (content, maxChunkChars = 100_000, overlapLines = 5, startFromChar = 0) => { + if (!content) { + return [ + { + content: '', + chunk_index: 0, + total_chunks: 1, + char_offset_start: 0, + char_offset_end: 0, + overlap_prefix: '', + has_more: false, + }, + ]; + } + if (startFromChar >= content.length) { + return []; + } + const blocks = parseAtomicBlocks(content); + if (!blocks.length) { + return []; + } + const rawChunks = []; + let currentChunk = []; + let currentSize = 0; + for (const block of blocks) { + const blockSize = getBlockSize(block); + if (currentSize + blockSize > maxChunkChars && currentChunk.length > 0) { + let bestSplit = currentChunk.length; + for (let j = currentChunk.length - 1; j >= 1; j -= 1) { + if (currentChunk[j]?.block_type === BlockType.Header) { + const prefixSize = currentChunk + .slice(0, j) + .reduce((sum, part) => sum + getBlockSize(part), 0); + if (prefixSize >= maxChunkChars * 0.5) { + bestSplit = j; + break; + } + } + } + rawChunks.push(currentChunk.slice(0, bestSplit)); + currentChunk = currentChunk.slice(bestSplit); + currentSize = currentChunk.reduce((sum, part) => sum + getBlockSize(part), 0); + } + currentChunk.push(block); + currentSize += blockSize; + } + if (currentChunk.length > 0) { + rawChunks.push(currentChunk); + } + const chunks = []; + const totalChunks = rawChunks.length; + let previousTableHeader = null; + for (let index = 0; index < rawChunks.length; index += 1) { + const chunkBlocks = rawChunks[index] ?? []; + if (!chunkBlocks.length) { + continue; + } + const chunkText = chunkBlocks.map(blockText).join('\n'); + const charStart = chunkBlocks[0]?.char_start ?? 0; + const charEnd = chunkBlocks[chunkBlocks.length - 1]?.char_end ?? charStart; + let overlapPrefix = ''; + if (index > 0) { + const previousBlocks = rawChunks[index - 1] ?? []; + const previousText = previousBlocks.map(blockText).join('\n'); + const previousLines = previousText.split('\n'); + const firstBlock = chunkBlocks[0]; + if (firstBlock?.block_type === BlockType.Table && + previousTableHeader !== null) { + const trailingLines = overlapLines > 0 ? previousLines.slice(-overlapLines) : []; + const headerLines = previousTableHeader.split('\n'); + const merged = [...headerLines]; + for (const trailing of trailingLines) { + if (!merged.includes(trailing)) { + merged.push(trailing); + } + } + overlapPrefix = merged.join('\n'); + } + else if (overlapLines > 0) { + overlapPrefix = previousLines.slice(-overlapLines).join('\n'); + } + } + for (const block of chunkBlocks) { + if (block.block_type !== BlockType.Table) { + continue; + } + const header = getTableHeader(block); + if (header !== null) { + previousTableHeader = header; + } + } + chunks.push({ + content: chunkText, + chunk_index: index, + total_chunks: totalChunks, + char_offset_start: charStart, + char_offset_end: charEnd, + overlap_prefix: overlapPrefix, + has_more: index < totalChunks - 1, + }); + } + if (startFromChar > 0) { + for (let index = 0; index < chunks.length; index += 1) { + if ((chunks[index]?.char_offset_end ?? 0) > startFromChar) { + return chunks.slice(index); + } + } + return []; + } + return chunks; +}; diff --git a/dist/dom/playground/extraction.d.ts b/dist/dom/playground/extraction.d.ts new file mode 100644 index 00000000..30f39340 --- /dev/null +++ b/dist/dom/playground/extraction.d.ts @@ -0,0 +1,19 @@ +/** + * Interactive DOM element testing tool. + * + * This playground allows you to: + * - Navigate to websites + * - Extract DOM state and clickable elements + * - Interactively click elements by index + * - Input text into elements + * - Copy element JSON to clipboard + * - Analyze token counts for LLM prompts + * + * Usage: + * - Enter an element index to click it + * - Enter 'index,text' to input text into an element + * - Enter 'c,index' to copy element JSON to clipboard + * - Enter 'q' to quit + */ +declare function testFocusVsAllElements(): Promise; +export { testFocusVsAllElements }; diff --git a/dist/dom/playground/extraction.js b/dist/dom/playground/extraction.js new file mode 100644 index 00000000..8186a0f0 --- /dev/null +++ b/dist/dom/playground/extraction.js @@ -0,0 +1,187 @@ +import * as fs from 'fs/promises'; +import * as readline from 'readline'; +import { promisify } from 'util'; +import { BrowserProfile, BrowserSession } from '../../browser/index.js'; +import { DomService } from '../service.js'; +import { DEFAULT_INCLUDE_ATTRIBUTES } from '../views.js'; +import { AgentMessagePrompt } from '../../agent/prompts.js'; +import { FileSystem } from '../../filesystem/file-system.js'; +const TIMEOUT = 60; +/** + * Interactive DOM element testing tool. + * + * This playground allows you to: + * - Navigate to websites + * - Extract DOM state and clickable elements + * - Interactively click elements by index + * - Input text into elements + * - Copy element JSON to clipboard + * - Analyze token counts for LLM prompts + * + * Usage: + * - Enter an element index to click it + * - Enter 'index,text' to input text into an element + * - Enter 'c,index' to copy element JSON to clipboard + * - Enter 'q' to quit + */ +async function testFocusVsAllElements() { + const browserSession = new BrowserSession({ + browser_profile: new BrowserProfile({ + window_size: { width: 1100, height: 1000 }, + disable_security: true, + wait_for_network_idle_page_load_time: 1, + headless: false, + }), + }); + const websites = [ + 'https://google.com', + 'https://www.ycombinator.com/companies', + 'https://kayak.com/flights', + 'https://docs.google.com/spreadsheets/d/1INaIcfpYXlMRWO__de61SHFCaqt1lfHlcvtXZPItlpI/edit', + 'https://www.zeiss.com/career/en/job-search.html?page=1', + 'https://www.mlb.com/yankees/stats/', + 'https://www.amazon.com/s?k=laptop&s=review-rank', + 'https://reddit.com', + 'https://codepen.io/geheimschriftstift/pen/mPLvQz', + 'https://www.google.com/search?q=google+hi', + 'https://amazon.com', + 'https://github.com', + ]; + await browserSession.start(); + const page = await browserSession.getCurrentPage(); + if (!page) { + throw new Error('No page available'); + } + const domService = new DomService(page); + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + const question = promisify(rl.question).bind(rl); + for (const website of websites) { + await page.goto(website); + await new Promise((resolve) => setTimeout(resolve, 1000)); + let lastClickedIndex = null; + while (true) { + try { + console.log(`\n${'='.repeat(50)}\nTesting ${website}\n${'='.repeat(50)}`); + console.log('\nGetting page state...'); + const startTime = Date.now(); + const allElementsState = await browserSession.get_state_summary(true); + const endTime = Date.now(); + console.log(`get_state_summary took ${((endTime - startTime) / 1000).toFixed(2)} seconds`); + const selectorMap = allElementsState.selector_map; + const totalElements = Object.keys(selectorMap).length; + console.log(`Total number of elements: ${totalElements}`); + const prompt = new AgentMessagePrompt({ + browser_state_summary: allElementsState, + file_system: new FileSystem('./tmp'), + include_attributes: DEFAULT_INCLUDE_ATTRIBUTES, + step_info: null, + }); + const userMessage = prompt.get_user_message(false).text || ''; + const textToSave = userMessage; + await fs.mkdir('./tmp', { recursive: true }); + await fs.writeFile('./tmp/user_message.txt', textToSave, 'utf-8'); + await fs.writeFile('./tmp/element_tree.json', JSON.stringify(allElementsState.element_tree.toJSON(), null, 0), 'utf-8'); + try { + // Optional: tiktoken is not installed by default + // @ts-ignore - tiktoken is an optional dependency + const { encoding_for_model } = await import('tiktoken'); + const encoding = encoding_for_model('gpt-4o'); + const tokenCount = encoding.encode(textToSave).length; + console.log(`Token count: ${tokenCount}`); + } + catch (error) { + console.log('Could not calculate token count (tiktoken not installed):', error.message); + } + console.log('User message written to ./tmp/user_message.txt'); + console.log('Element tree written to ./tmp/element_tree.json'); + const answer = String(await question("Enter element index to click, 'index,text' to input, 'c,index' to copy element JSON, or 'q' to quit: ")); + if (answer.toLowerCase().trim() === 'q') { + break; + } + try { + if (answer.toLowerCase().startsWith('c,')) { + const parts = answer.split(',', 2); + if (parts.length === 2) { + try { + const targetIndex = parseInt(parts[1].trim(), 10); + if (targetIndex in selectorMap) { + const elementNode = selectorMap[targetIndex]; + const elementJson = JSON.stringify(elementNode.toJSON(), null, 2); + console.log(`Element ${targetIndex} JSON:`); + console.log(elementJson); + console.log(`\nElement: ${elementNode.tag_name}`); + } + else { + console.log(`Invalid index: ${targetIndex}`); + } + } + catch { + console.log(`Invalid index format: ${parts[1]}`); + } + } + else { + console.log("Invalid input format. Use 'c,index'."); + } + } + else if (answer.includes(',')) { + const parts = answer.split(',', 2); + if (parts.length === 2) { + try { + const targetIndex = parseInt(parts[0].trim(), 10); + const textToInput = parts[1]; + if (targetIndex in selectorMap) { + const elementNode = selectorMap[targetIndex]; + console.log(`Inputting text '${textToInput}' into element ${targetIndex}: ${elementNode.tag_name}`); + await browserSession._inputTextElementNode(elementNode, textToInput); + console.log('Input successful.'); + } + else { + console.log(`Invalid index: ${targetIndex}`); + } + } + catch { + console.log(`Invalid index format: ${parts[0]}`); + } + } + else { + console.log("Invalid input format. Use 'index,text'."); + } + } + else { + try { + const clickedIndex = parseInt(answer, 10); + if (clickedIndex in selectorMap) { + const elementNode = selectorMap[clickedIndex]; + console.log(`Clicking element ${clickedIndex}: ${elementNode.tag_name}`); + await browserSession._clickElementNode(elementNode); + console.log('Click successful.'); + } + else { + console.log(`Invalid index: ${clickedIndex}`); + } + } + catch { + console.log(`Invalid input: '${answer}'. Enter an index, 'index,text', 'c,index', or 'q'.`); + } + } + } + catch (actionError) { + console.log(`Action failed: ${actionError.message}`); + } + } + catch (error) { + console.log(`Error in loop: ${error.message}`); + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } + } + rl.close(); + await browserSession.close(); +} +if (require.main === module) { + testFocusVsAllElements().catch(console.error); +} +export { testFocusVsAllElements }; diff --git a/dist/dom/service.d.ts b/dist/dom/service.d.ts new file mode 100644 index 00000000..be56c4cf --- /dev/null +++ b/dist/dom/service.d.ts @@ -0,0 +1,21 @@ +import type { Page } from '../browser/types.js'; +import { DOMState, type SelectorMap } from './views.js'; +import type { PaginationButton } from '../browser/views.js'; +export declare class DomService { + private readonly page; + private readonly logger; + private readonly jsCode; + constructor(page: Page, logger?: import("../logging-config.js").Logger); + get_clickable_elements(highlight_elements?: boolean, focus_element?: number, viewport_expansion?: number): Promise; + get_cross_origin_iframes(): Promise; + private _build_dom_tree; + private _construct_dom_tree; + private _parse_node; + private safeHostname; + private getFrames; + private getFrameUrl; + private isAdUrl; + private getPageUrl; + private isDebugEnabled; + static detect_pagination_buttons(selector_map: SelectorMap): PaginationButton[]; +} diff --git a/dist/dom/service.js b/dist/dom/service.js new file mode 100644 index 00000000..b3b51d87 --- /dev/null +++ b/dist/dom/service.js @@ -0,0 +1,303 @@ +var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) { + var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d; + if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc); + else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r; + return c > 3 && r && Object.defineProperty(target, key, r), r; +}; +import fs from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { createLogger } from '../logging-config.js'; +import { observe_debug } from '../observability.js'; +import { time_execution_async } from '../utils.js'; +import { is_new_tab_page } from '../utils.js'; +import { DOMElementNode, DOMState, DOMTextNode, } from './views.js'; +const DOM_TREE_SCRIPT = fs.readFileSync(fileURLToPath(new URL('./dom_tree/index.js', import.meta.url)), 'utf-8'); +export class DomService { + page; + logger; + jsCode; + constructor(page, logger = createLogger('browser_use.dom.service')) { + this.page = page; + this.logger = logger; + this.jsCode = DOM_TREE_SCRIPT; + } + // @ts-ignore - Decorator type mismatch with TypeScript strict mode + async get_clickable_elements(highlight_elements = true, focus_element = -1, viewport_expansion = 0) { + const [element_tree, selector_map] = await this._build_dom_tree(highlight_elements, focus_element, viewport_expansion); + return new DOMState(element_tree, selector_map); + } + // @ts-ignore - Decorator type mismatch with TypeScript strict mode + async get_cross_origin_iframes() { + const hiddenFrameUrls = await this.page + .locator('iframe') + .evaluateAll((elements) => elements + .filter((el) => { + const element = el; + const style = window.getComputedStyle(element); + const rect = element.getBoundingClientRect(); + return (style.visibility === 'hidden' || + style.display === 'none' || + rect.width === 0 || + rect.height === 0); + }) + .map((el) => el.src)); + const currentHost = this.safeHostname(this.getPageUrl()); + return this.getFrames() + .map((frame) => this.getFrameUrl(frame)) + .filter((url) => { + if (!url) + return false; + const host = this.safeHostname(url); + if (!host) + return false; + if (host === currentHost) + return false; + if (hiddenFrameUrls.includes(url)) + return false; + return !this.isAdUrl(url); + }); + } + // @ts-ignore - Decorator type mismatch with TypeScript strict mode + async _build_dom_tree(highlight_elements, focus_element, viewport_expansion) { + const canEvaluate = await this.page.evaluate(() => 1 + 1); + if (canEvaluate !== 2) { + throw new Error('The page cannot evaluate JavaScript code properly'); + } + const pageUrl = this.getPageUrl(); + if (is_new_tab_page(pageUrl) || pageUrl.startsWith('chrome://')) { + return [ + new DOMElementNode(false, null, 'body', '', {}, []), + {}, + ]; + } + const args = { + doHighlightElements: highlight_elements, + focusHighlightIndex: focus_element, + viewportExpansion: viewport_expansion, + debugMode: this.isDebugEnabled(), + }; + let eval_page; + try { + this.logger.debug(`🔧 Starting JavaScript DOM analysis for ${pageUrl.slice(0, 50)}...`); + eval_page = await this.page.evaluate(({ script, evaluateArgs }) => { + const fn = eval(script); + return fn(evaluateArgs); + }, { script: this.jsCode, evaluateArgs: args }); + this.logger.debug('✅ JavaScript DOM analysis completed'); + } + catch (error) { + this.logger.error(`Error evaluating DOMTree: ${error.message}`); + throw error; + } + if (args.debugMode && eval_page.perfMetrics) { + const perf = eval_page.perfMetrics; + const totalNodes = perf?.nodeMetrics?.totalNodes ?? 0; + let interactiveCount = 0; + if (eval_page.map) { + for (const node of Object.values(eval_page.map)) { + if (node?.isInteractive) { + interactiveCount += 1; + } + } + } + this.logger.debug(`🔎 Ran buildDOMTree.js interactive element detection on: ${pageUrl.slice(0, 50)} interactive=${interactiveCount}/${totalNodes}`); + } + this.logger.debug('🔄 Starting DOM tree construction...'); + const result = await this._construct_dom_tree(eval_page); + this.logger.debug('✅ DOM tree construction completed'); + return result; + } + // @ts-ignore - Decorator type mismatch with TypeScript strict mode + async _construct_dom_tree(eval_page) { + const selector_map = {}; + const node_map = new Map(); + const child_index = new Map(); + for (const [id, node_data] of Object.entries(eval_page.map)) { + const [node, children] = this._parse_node(node_data); + if (!node) + continue; + node_map.set(id, node); + child_index.set(id, children); + if (node instanceof DOMElementNode && + node.highlight_index !== null && + node.highlight_index !== undefined) { + selector_map[node.highlight_index] = node; + } + } + for (const [id, childrenIds] of child_index.entries()) { + const parentNode = node_map.get(id); + if (!(parentNode instanceof DOMElementNode)) + continue; + for (const childId of childrenIds || []) { + const key = String(childId); + const childNode = node_map.get(key); + if (!childNode) + continue; + childNode.parent = parentNode; + parentNode.children.push(childNode); + } + } + const rootNode = node_map.get(String(eval_page.rootId)); + if (!(rootNode instanceof DOMElementNode)) { + throw new Error('Failed to construct DOM tree'); + } + return [rootNode, selector_map]; + } + _parse_node(node_data) { + if (!node_data) { + return [null, []]; + } + if (node_data.type === 'TEXT_NODE') { + const textNode = new DOMTextNode(node_data.isVisible ?? false, null, node_data.text ?? ''); + return [textNode, []]; + } + const children = Array.isArray(node_data.children) + ? node_data.children + : []; + const tag = node_data.tagName ?? 'div'; + const xpath = node_data.xpath ?? ''; + const attributes = node_data.attributes ?? {}; + const element = new DOMElementNode(node_data.isVisible ?? false, null, tag, xpath, attributes, []); + element.is_interactive = Boolean(node_data.isInteractive); + element.is_top_element = Boolean(node_data.isTopElement); + element.is_in_viewport = Boolean(node_data.isInViewport); + element.shadow_root = Boolean(node_data.shadowRoot); + element.highlight_index = + node_data.highlightIndex === undefined || + node_data.highlightIndex === null + ? null + : Number(node_data.highlightIndex); + element.page_coordinates = node_data.pageCoordinates ?? null; + element.viewport_coordinates = + node_data.viewportCoordinates ?? null; + element.viewport_info = node_data.viewportInfo ?? null; + element.is_new = node_data.isNew ?? null; + return [element, children]; + } + safeHostname(url) { + if (!url) + return ''; + try { + return new URL(url).hostname; + } + catch { + return ''; + } + } + getFrames() { + const frames = this.page.frames; + return typeof frames === 'function' + ? frames.call(this.page) + : (frames ?? []); + } + getFrameUrl(frame) { + return typeof frame.url === 'function' ? frame.url() : (frame.url ?? ''); + } + isAdUrl(url) { + const host = this.safeHostname(url); + return ['doubleclick.net', 'adroll.com', 'googletagmanager.com'].some((domain) => host.endsWith(domain)); + } + getPageUrl() { + return typeof this.page.url === 'function' + ? this.page.url() + : (this.page.url ?? ''); + } + isDebugEnabled() { + return ((process.env.BROWSER_USE_LOGGING_LEVEL ?? '').toLowerCase() === 'debug'); + } + static detect_pagination_buttons(selector_map) { + const paginationButtons = []; + const nextPatterns = [ + 'next', + '>', + '>>', + 'siguiente', + 'suivant', + 'weiter', + 'volgende', + ]; + const prevPatterns = [ + 'prev', + 'previous', + '<', + '<<', + 'anterior', + 'precedent', + 'zuruck', + 'vorige', + ]; + const firstPatterns = ['first', 'primera', 'premiere', 'erste']; + const lastPatterns = ['last', 'ultima', 'dernier', 'letzte']; + const hasPattern = (text, patterns) => patterns.some((pattern) => text.includes(pattern)); + for (const [index, node] of Object.entries(selector_map)) { + if (!(node instanceof DOMElementNode)) { + continue; + } + const text = node.get_all_text_till_next_clickable_element().trim(); + const textLower = text.toLowerCase(); + const ariaLabel = String(node.attributes?.['aria-label'] ?? '').toLowerCase(); + const title = String(node.attributes?.title ?? '').toLowerCase(); + const className = String(node.attributes?.class ?? '').toLowerCase(); + const role = String(node.attributes?.role ?? '').toLowerCase(); + const allText = `${textLower} ${ariaLabel} ${title} ${className}`.trim(); + const disabledRaw = node.attributes?.disabled; + const ariaDisabledRaw = node.attributes?.['aria-disabled']; + const disabledAttr = typeof disabledRaw === 'string' ? disabledRaw.toLowerCase() : ''; + const ariaDisabled = typeof ariaDisabledRaw === 'string' + ? ariaDisabledRaw.toLowerCase() + : ''; + const isDisabled = (typeof disabledRaw === 'string' && + disabledAttr !== '' && + disabledAttr !== 'false') || + ariaDisabled === 'true' || + className.includes('disabled'); + let buttonType = null; + if (hasPattern(allText, nextPatterns)) { + buttonType = 'next'; + } + else if (hasPattern(allText, prevPatterns)) { + buttonType = 'prev'; + } + else if (hasPattern(allText, firstPatterns)) { + buttonType = 'first'; + } + else if (hasPattern(allText, lastPatterns)) { + buttonType = 'last'; + } + else if (/^\d{1,2}$/.test(textLower) && + (role === 'button' || role === 'link' || role === '')) { + buttonType = 'page_number'; + } + if (!buttonType) { + continue; + } + paginationButtons.push({ + button_type: buttonType, + backend_node_id: Number(index), + text: text || ariaLabel || title || node.tag_name, + selector: node.xpath, + is_disabled: isDisabled, + }); + } + return paginationButtons; + } +} +__decorate([ + observe_debug({ + ignore_input: true, + ignore_output: true, + name: 'get_clickable_elements', + }) + // @ts-ignore - Decorator type mismatch with TypeScript strict mode + , + time_execution_async('--get_clickable_elements') +], DomService.prototype, "get_clickable_elements", null); +__decorate([ + time_execution_async('--get_cross_origin_iframes') +], DomService.prototype, "get_cross_origin_iframes", null); +__decorate([ + time_execution_async('--build_dom_tree') +], DomService.prototype, "_build_dom_tree", null); +__decorate([ + time_execution_async('--construct_dom_tree') +], DomService.prototype, "_construct_dom_tree", null); diff --git a/dist/dom/utils.d.ts b/dist/dom/utils.d.ts new file mode 100644 index 00000000..bcd0fd29 --- /dev/null +++ b/dist/dom/utils.d.ts @@ -0,0 +1 @@ +export declare const cap_text_length: (text: string, max_length: number) => string; diff --git a/dist/dom/utils.js b/dist/dom/utils.js new file mode 100644 index 00000000..5d3531bb --- /dev/null +++ b/dist/dom/utils.js @@ -0,0 +1,6 @@ +export const cap_text_length = (text, max_length) => { + if (text.length > max_length) { + return `${text.slice(0, max_length)}...`; + } + return text; +}; diff --git a/dist/dom/views.d.ts b/dist/dom/views.d.ts new file mode 100644 index 00000000..76fd70be --- /dev/null +++ b/dist/dom/views.d.ts @@ -0,0 +1,62 @@ +import { CoordinateSet, HashedDomElement, ViewportInfo } from './history-tree-processor/view.js'; +export declare abstract class DOMBaseNode { + is_visible: boolean; + parent: DOMElementNode | null; + constructor(is_visible: boolean, parent?: DOMElementNode | null); + abstract toJSON(): Record; +} +export declare class DOMTextNode extends DOMBaseNode { + text: string; + type: string; + constructor(is_visible: boolean, parent: DOMElementNode | null, text: string); + has_parent_with_highlight_index(): boolean; + is_parent_in_viewport(): boolean; + is_parent_top_element(): boolean; + toJSON(): { + text: string; + type: string; + }; +} +export declare const DEFAULT_INCLUDE_ATTRIBUTES: string[]; +export declare class DOMElementNode extends DOMBaseNode { + tag_name: string; + xpath: string; + attributes: Record; + children: DOMBaseNode[]; + is_interactive: boolean; + is_top_element: boolean; + is_in_viewport: boolean; + shadow_root: boolean; + highlight_index: number | null; + viewport_coordinates: CoordinateSet | null; + page_coordinates: CoordinateSet | null; + viewport_info: ViewportInfo | null; + is_new: boolean | null; + private cached_hash; + constructor(is_visible: boolean, parent: DOMElementNode | null, tag_name: string, xpath: string, attributes: Record, children: DOMBaseNode[]); + toJSON(): { + tag_name: string; + xpath: string; + attributes: Record; + is_visible: boolean; + is_interactive: boolean; + is_top_element: boolean; + is_in_viewport: boolean; + shadow_root: boolean; + highlight_index: number | null; + viewport_coordinates: CoordinateSet | null; + page_coordinates: CoordinateSet | null; + children: Record[]; + }; + toString(): string; + get hash(): HashedDomElement; + get_all_text_till_next_clickable_element(max_depth?: number): string; + clickable_elements_to_string(include_attributes?: string[]): string; +} +export type SelectorMap = Record; +export declare class DOMState { + element_tree: DOMElementNode; + selector_map: SelectorMap; + constructor(element_tree: DOMElementNode, selector_map: SelectorMap); + llm_representation(include_attributes?: string[]): string; +} diff --git a/dist/dom/views.js b/dist/dom/views.js new file mode 100644 index 00000000..50bf669c --- /dev/null +++ b/dist/dom/views.js @@ -0,0 +1,292 @@ +import { cap_text_length } from './utils.js'; +import { time_execution_sync } from '../utils.js'; +import { HistoryTreeProcessor } from './history-tree-processor/service.js'; +export class DOMBaseNode { + is_visible; + parent; + constructor(is_visible, parent = null) { + this.is_visible = is_visible; + this.parent = parent; + } +} +export class DOMTextNode extends DOMBaseNode { + text; + type = 'TEXT_NODE'; + constructor(is_visible, parent, text) { + super(is_visible, parent); + this.text = text; + } + has_parent_with_highlight_index() { + let current = this.parent; + while (current) { + if (current.highlight_index !== null && + current.highlight_index !== undefined) { + return true; + } + current = current.parent; + } + return false; + } + is_parent_in_viewport() { + return Boolean(this.parent?.is_in_viewport); + } + is_parent_top_element() { + return Boolean(this.parent?.is_top_element); + } + toJSON() { + return { + text: this.text, + type: this.type, + }; + } +} +export const DEFAULT_INCLUDE_ATTRIBUTES = [ + 'title', + 'type', + 'checked', + 'id', + 'name', + 'role', + 'value', + 'placeholder', + 'data-date-format', + 'alt', + 'aria-label', + 'aria-expanded', + 'data-state', + 'aria-checked', + 'aria-valuemin', + 'aria-valuemax', + 'aria-valuenow', + 'aria-placeholder', + 'pattern', + 'min', + 'max', + 'minlength', + 'maxlength', + 'step', + 'accept', + 'multiple', + 'inputmode', + 'autocomplete', + 'aria-autocomplete', + 'list', + 'data-mask', + 'data-inputmask', + 'data-datepicker', + 'format', + 'expected_format', + 'contenteditable', + 'pseudo', + 'checked', + 'selected', + 'expanded', + 'pressed', + 'disabled', + 'invalid', + 'valuemin', + 'valuemax', + 'valuenow', + 'keyshortcuts', + 'haspopup', + 'multiselectable', + 'required', + 'valuetext', + 'level', + 'busy', + 'live', + 'ax_name', +]; +export class DOMElementNode extends DOMBaseNode { + tag_name; + xpath; + attributes; + children; + is_interactive = false; + is_top_element = false; + is_in_viewport = false; + shadow_root = false; + highlight_index = null; + viewport_coordinates = null; + page_coordinates = null; + viewport_info = null; + is_new = null; + cached_hash = null; + constructor(is_visible, parent, tag_name, xpath, attributes, children) { + super(is_visible, parent); + this.tag_name = tag_name; + this.xpath = xpath; + this.attributes = attributes; + this.children = children; + } + toJSON() { + return { + tag_name: this.tag_name, + xpath: this.xpath, + attributes: this.attributes, + is_visible: this.is_visible, + is_interactive: this.is_interactive, + is_top_element: this.is_top_element, + is_in_viewport: this.is_in_viewport, + shadow_root: this.shadow_root, + highlight_index: this.highlight_index, + viewport_coordinates: this.viewport_coordinates, + page_coordinates: this.page_coordinates, + children: this.children.map((child) => child.toJSON()), + }; + } + toString() { + let tag_str = `<${this.tag_name}`; + for (const [key, value] of Object.entries(this.attributes)) { + tag_str += ` ${key}="${value}"`; + } + tag_str += '>'; + const extras = []; + if (this.is_interactive) + extras.push('interactive'); + if (this.is_top_element) + extras.push('top'); + if (this.shadow_root) + extras.push('shadow-root'); + if (this.highlight_index !== null && this.highlight_index !== undefined) { + extras.push(`highlight:${this.highlight_index}`); + } + if (this.is_in_viewport) + extras.push('in-viewport'); + if (extras.length) { + tag_str += ` [${extras.join(', ')}]`; + } + return tag_str; + } + get hash() { + if (!this.cached_hash) { + this.cached_hash = HistoryTreeProcessor._hash_dom_element(this); + } + return this.cached_hash; + } + get_all_text_till_next_clickable_element(max_depth = -1) { + const text_parts = []; + const collect_text = (node, current_depth) => { + if (max_depth !== -1 && current_depth > max_depth) { + return; + } + if (node instanceof DOMElementNode && + node !== this && + node.highlight_index !== null && + node.highlight_index !== undefined) { + return; + } + if (node instanceof DOMTextNode) { + text_parts.push(node.text); + } + else if (node instanceof DOMElementNode) { + for (const child of node.children) { + collect_text(child, current_depth + 1); + } + } + }; + collect_text(this, 0); + return text_parts.join('\n').trim(); + } + clickable_elements_to_string(include_attributes) { + return CLICKABLE_ELEMENTS_TO_STRING_IMPL.call(this, include_attributes); + } +} +const CLICKABLE_ELEMENTS_TO_STRING_IMPL = time_execution_sync('--clickable_elements_to_string')(function (include_attributes) { + const formatted_text = []; + const attributes_list = include_attributes ?? DEFAULT_INCLUDE_ATTRIBUTES; + const process_node = (node, depth) => { + const next_depth = depth; + const depth_str = '\t'.repeat(depth); + if (node instanceof DOMElementNode) { + let working_depth = next_depth; + if (node.highlight_index !== null && node.highlight_index !== undefined) { + working_depth += 1; + let text = node.get_all_text_till_next_clickable_element(); + let attributes_html_str = null; + if (attributes_list.length) { + const attributes_to_include = Object.fromEntries(Object.entries(node.attributes) + .filter(([key, value]) => attributes_list.includes(key) && String(value).trim() !== '') + .map(([key, value]) => [key, String(value).trim()])); + const ordered_keys = attributes_list.filter((key) => key in attributes_to_include); + if (ordered_keys.length > 1) { + const keys_to_remove = new Set(); + const seen_values = {}; + for (const key of ordered_keys) { + const value = attributes_to_include[key]; + if (value && value.length > 5) { + if (seen_values[value]) { + keys_to_remove.add(key); + } + else { + seen_values[value] = key; + } + } + } + for (const key of keys_to_remove) { + delete attributes_to_include[key]; + } + } + if (node.tag_name === attributes_to_include.role) { + delete attributes_to_include.role; + } + for (const attr of ['aria-label', 'placeholder', 'title']) { + if (attributes_to_include[attr] && + attributes_to_include[attr].trim().toLowerCase() === + text.trim().toLowerCase()) { + delete attributes_to_include[attr]; + } + } + if (Object.entries(attributes_to_include).length) { + attributes_html_str = Object.entries(attributes_to_include) + .map(([key, value]) => `${key}=${cap_text_length(value, 15)}`) + .join(' '); + } + } + const highlight_indicator = node.is_new + ? `*[${node.highlight_index}]` + : `[${node.highlight_index}]`; + let line = `${depth_str}${highlight_indicator}<${node.tag_name}`; + if (attributes_html_str) { + line += ` ${attributes_html_str}`; + } + if (text) { + text = text.trim(); + if (!attributes_html_str) { + line += ' '; + } + line += `>${text}`; + } + else if (!attributes_html_str) { + line += ' '; + } + line += ' />'; + formatted_text.push(line); + } + for (const child of node.children) { + process_node(child, working_depth); + } + } + else if (node instanceof DOMTextNode) { + if (node.has_parent_with_highlight_index()) { + return; + } + if (node.parent?.is_visible && node.parent.is_top_element) { + formatted_text.push(`${depth_str}${node.text}`); + } + } + }; + process_node(this, 0); + return formatted_text.join('\n'); +}); +export class DOMState { + element_tree; + selector_map; + constructor(element_tree, selector_map) { + this.element_tree = element_tree; + this.selector_map = selector_map; + } + llm_representation(include_attributes) { + return this.element_tree.clickable_elements_to_string(include_attributes); + } +} diff --git a/dist/event-bus.d.ts b/dist/event-bus.d.ts new file mode 100644 index 00000000..3a7b2031 --- /dev/null +++ b/dist/event-bus.d.ts @@ -0,0 +1,111 @@ +export interface EventPayload { + event_type?: string; + event_id?: string; + event_parent_id?: string | null; + event_timeout?: number | null; + event_created_at?: Date; + event_result?: unknown; + event_error?: unknown; +} +export interface EventBusEventInit { + event_id?: string; + event_parent_id?: string | null; + event_timeout?: number | null; + event_created_at?: Date; + event_result?: TResult | null; + event_error?: unknown; +} +export declare class EventBusEvent implements EventPayload { + event_type: string; + event_id: string; + event_parent_id: string | null; + event_timeout: number | null; + event_created_at: Date; + event_result: TResult | null; + event_error: unknown; + constructor(event_type: string, init?: EventBusEventInit); +} +export type EventTypeReference = string | (new (...args: any[]) => TEvent); +export type EventHandler = (event: TEvent) => TResult | Promise; +export interface EventSubscriptionOptions { + once?: boolean; + handler_id?: string; + allow_duplicate?: boolean; +} +export interface EventDispatchOptions { + throw_on_error?: boolean; + timeout_ms?: number | null; + parallel_handlers?: boolean; +} +export interface EventBusOptions { + event_history_limit?: number; + throw_on_error_by_default?: boolean; +} +export interface EventHandlerExecutionResult { + handler_id: string; + event_type: string; + status: 'fulfilled' | 'rejected' | 'timed_out'; + started_at: Date; + completed_at: Date; + duration_ms: number; + result?: unknown; + error?: unknown; +} +export interface EventDispatchResult { + event: TEvent; + event_id: string; + event_type: string; + event_parent_id: string | null; + event_timeout: number | null; + started_at: Date; + completed_at: Date; + duration_ms: number; + status: 'pending' | 'fulfilled' | 'rejected' | 'timed_out'; + handler_results: EventHandlerExecutionResult[]; + errors: unknown[]; +} +export declare class EventHandlerTimeoutError extends Error { + event_type: string; + handler_id: string; + timeout_ms: number; + constructor(event_type: string, handler_id: string, timeout_ms: number); +} +export declare class EventDispatchError extends Error { + dispatch_result: EventDispatchResult; + constructor(dispatch_result: EventDispatchResult); +} +interface EventHandlerRegistration { + event_type: string; + handler: EventHandler; + handler_id: string; + once: boolean; +} +export declare class EventBus { + readonly name: string; + readonly handlers: Map; + readonly event_history: Map>; + private readonly history_limit; + private readonly throw_on_error_by_default; + private readonly dispatch_context; + constructor(name: string, options?: EventBusOptions); + on(event_type_ref: EventTypeReference, handler: EventHandler, options?: EventSubscriptionOptions): () => void; + once(event_type_ref: EventTypeReference, handler: EventHandler, options?: Omit): () => void; + off(event_type_ref: EventTypeReference, handler_or_id?: EventHandler | string): void; + dispatch(event: TEvent, options?: EventDispatchOptions): Promise>; + dispatch_or_throw(event: TEvent, options?: Omit): Promise>; + getHandlers(event_type_ref: EventTypeReference): EventHandlerRegistration[]; + stop(): Promise; + private resolveEventType; + private resolveEventTypeFromRef; + private resolveHandlerId; + private resolveEventId; + private resolveParentEventId; + private resolveTimeoutMs; + private assignEventMetadata; + private assignEventResult; + private assignEventError; + private safeAssign; + private withTimeout; + private pruneHistory; +} +export {}; diff --git a/dist/event-bus.js b/dist/event-bus.js new file mode 100644 index 00000000..620594af --- /dev/null +++ b/dist/event-bus.js @@ -0,0 +1,322 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import { randomUUID } from 'node:crypto'; +export class EventBusEvent { + event_type; + event_id; + event_parent_id; + event_timeout; + event_created_at; + event_result; + event_error; + constructor(event_type, init = {}) { + this.event_type = event_type; + this.event_id = init.event_id ?? randomUUID(); + this.event_parent_id = init.event_parent_id ?? null; + this.event_timeout = init.event_timeout ?? null; + this.event_created_at = init.event_created_at ?? new Date(); + this.event_result = init.event_result ?? null; + this.event_error = init.event_error ?? null; + } +} +export class EventHandlerTimeoutError extends Error { + event_type; + handler_id; + timeout_ms; + constructor(event_type, handler_id, timeout_ms) { + super(`Handler ${handler_id} timed out after ${timeout_ms}ms for ${event_type}`); + this.name = 'EventHandlerTimeoutError'; + this.event_type = event_type; + this.handler_id = handler_id; + this.timeout_ms = timeout_ms; + } +} +export class EventDispatchError extends Error { + dispatch_result; + constructor(dispatch_result) { + super(`Event ${dispatch_result.event_type}#${dispatch_result.event_id} failed with ${dispatch_result.errors.length} error(s)`); + this.name = 'EventDispatchError'; + this.dispatch_result = dispatch_result; + } +} +export class EventBus { + name; + handlers = new Map(); + event_history = new Map(); + history_limit; + throw_on_error_by_default; + dispatch_context = new AsyncLocalStorage(); + constructor(name, options = {}) { + this.name = name; + this.history_limit = options.event_history_limit ?? 500; + this.throw_on_error_by_default = options.throw_on_error_by_default ?? false; + } + on(event_type_ref, handler, options = {}) { + const event_type = this.resolveEventTypeFromRef(event_type_ref); + const handler_id = options.handler_id ?? + this.resolveHandlerId(event_type, handler); + const registrations = this.handlers.get(event_type) ?? []; + if (!options.allow_duplicate) { + const hasDuplicate = registrations.some((existing) => existing.handler === handler || existing.handler_id === handler_id); + if (hasDuplicate) { + throw new Error(`Duplicate handler registration for ${event_type} (${handler_id})`); + } + } + const registration = { + event_type, + handler: handler, + handler_id, + once: options.once ?? false, + }; + registrations.push(registration); + this.handlers.set(event_type, registrations); + return () => { + this.off(event_type_ref, handler_id); + }; + } + once(event_type_ref, handler, options = {}) { + return this.on(event_type_ref, handler, { ...options, once: true }); + } + off(event_type_ref, handler_or_id) { + const event_type = this.resolveEventTypeFromRef(event_type_ref); + const registrations = this.handlers.get(event_type); + if (!registrations || registrations.length === 0) { + return; + } + if (!handler_or_id) { + this.handlers.delete(event_type); + return; + } + const next = registrations.filter((entry) => { + if (typeof handler_or_id === 'string') { + return entry.handler_id !== handler_or_id; + } + return entry.handler !== handler_or_id; + }); + if (next.length) { + this.handlers.set(event_type, next); + } + else { + this.handlers.delete(event_type); + } + } + async dispatch(event, options = {}) { + const event_type = this.resolveEventType(event) ?? this.resolveEventTypeFromRef('event'); + const event_id = this.resolveEventId(event); + const event_parent_id = this.resolveParentEventId(event); + const timeout_ms = this.resolveTimeoutMs(event, options.timeout_ms); + const timeout_seconds = timeout_ms == null ? null : timeout_ms / 1000; + this.assignEventMetadata(event, { + event_type, + event_id, + event_parent_id, + event_timeout: timeout_seconds, + }); + const started_at = new Date(); + const registrations = [ + ...(this.handlers.get(event_type) ?? []), + ...(event_type === '*' ? [] : (this.handlers.get('*') ?? [])), + ]; + const dispatch_result = { + event, + event_id, + event_type, + event_parent_id, + event_timeout: timeout_seconds, + started_at, + completed_at: started_at, + duration_ms: 0, + status: 'pending', + handler_results: [], + errors: [], + }; + this.event_history.set(event_id, dispatch_result); + this.pruneHistory(); + const runHandler = async (registration) => { + const handler_started_at = new Date(); + const safeTimeoutMs = timeout_ms ?? undefined; + let handler_status = 'fulfilled'; + let handler_result; + let handler_error; + try { + const execution = this.dispatch_context.run({ event_id }, () => Promise.resolve(registration.handler(event))); + handler_result = + safeTimeoutMs == null + ? await execution + : await this.withTimeout(execution, safeTimeoutMs, event_type, registration.handler_id); + } + catch (error) { + handler_error = error; + handler_status = + error instanceof EventHandlerTimeoutError ? 'timed_out' : 'rejected'; + dispatch_result.errors.push(error); + } + const handler_completed_at = new Date(); + const handler_execution_result = { + handler_id: registration.handler_id, + event_type: registration.event_type, + status: handler_status, + started_at: handler_started_at, + completed_at: handler_completed_at, + duration_ms: handler_completed_at.getTime() - handler_started_at.getTime(), + }; + if (handler_result !== undefined) { + handler_execution_result.result = handler_result; + } + if (handler_error !== undefined) { + handler_execution_result.error = handler_error; + } + dispatch_result.handler_results.push(handler_execution_result); + if (registration.once) { + this.off(registration.event_type, registration.handler_id); + } + }; + if (options.parallel_handlers) { + await Promise.all(registrations.map((registration) => runHandler(registration))); + } + else { + for (const registration of registrations) { + await runHandler(registration); + } + } + const completed_at = new Date(); + dispatch_result.completed_at = completed_at; + dispatch_result.duration_ms = completed_at.getTime() - started_at.getTime(); + if (dispatch_result.errors.length > 0) { + const hasTimeout = dispatch_result.handler_results.some((result) => result.status === 'timed_out'); + dispatch_result.status = hasTimeout ? 'timed_out' : 'rejected'; + } + else { + dispatch_result.status = 'fulfilled'; + } + this.assignEventResult(event, dispatch_result.handler_results); + if (dispatch_result.errors.length > 0) { + this.assignEventError(event, dispatch_result.errors[0] ?? null); + } + const throw_on_error = options.throw_on_error ?? this.throw_on_error_by_default; + if (throw_on_error && dispatch_result.errors.length > 0) { + throw new EventDispatchError(dispatch_result); + } + return dispatch_result; + } + async dispatch_or_throw(event, options = {}) { + return this.dispatch(event, { ...options, throw_on_error: true }); + } + getHandlers(event_type_ref) { + const event_type = this.resolveEventTypeFromRef(event_type_ref); + return [...(this.handlers.get(event_type) ?? [])]; + } + async stop() { + this.handlers.clear(); + this.event_history.clear(); + } + resolveEventType(event) { + const event_type = event.event_type ?? + event.constructor?.name ?? + null; + return event_type && event_type.length > 0 ? event_type : null; + } + resolveEventTypeFromRef(event_type_ref) { + if (typeof event_type_ref === 'string') { + return event_type_ref; + } + return event_type_ref.name; + } + resolveHandlerId(event_type, handler) { + const suffix = typeof handler.name === 'string' && handler.name.length > 0 + ? handler.name + : `handler_${randomUUID().slice(0, 8)}`; + return `${event_type}:${suffix}`; + } + resolveEventId(event) { + if (event.event_id && event.event_id.length > 0) { + return event.event_id; + } + return randomUUID(); + } + resolveParentEventId(event) { + if (typeof event.event_parent_id === 'string' && + event.event_parent_id.length > 0) { + return event.event_parent_id; + } + return this.dispatch_context.getStore()?.event_id ?? null; + } + resolveTimeoutMs(event, dispatch_timeout_ms) { + if (dispatch_timeout_ms !== undefined) { + return dispatch_timeout_ms; + } + if (event.event_timeout == null) { + return null; + } + if (!Number.isFinite(event.event_timeout)) { + return null; + } + if (event.event_timeout < 0) { + return null; + } + return event.event_timeout * 1000; + } + assignEventMetadata(event, metadata) { + this.safeAssign(event, 'event_type', metadata.event_type); + this.safeAssign(event, 'event_id', metadata.event_id); + this.safeAssign(event, 'event_parent_id', metadata.event_parent_id); + if (event.event_timeout === undefined) { + this.safeAssign(event, 'event_timeout', metadata.event_timeout); + } + if (event.event_created_at === undefined) { + this.safeAssign(event, 'event_created_at', new Date()); + } + } + assignEventResult(event, handler_results) { + const first_defined_result = handler_results.find((result) => result.status === 'fulfilled' && + Object.prototype.hasOwnProperty.call(result, 'result')); + if (!first_defined_result) { + return; + } + this.safeAssign(event, 'event_result', first_defined_result.result); + } + assignEventError(event, error) { + this.safeAssign(event, 'event_error', error); + } + safeAssign(event, key, value) { + try { + event[key] = value; + } + catch { + // Read-only event objects should still be dispatchable. + } + } + async withTimeout(promise, timeout_ms, event_type, handler_id) { + if (timeout_ms <= 0) { + throw new EventHandlerTimeoutError(event_type, handler_id, timeout_ms); + } + let timeout_handle = null; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timeout_handle = setTimeout(() => { + reject(new EventHandlerTimeoutError(event_type, handler_id, timeout_ms)); + }, timeout_ms); + }), + ]); + } + finally { + if (timeout_handle) { + clearTimeout(timeout_handle); + } + } + } + pruneHistory() { + if (this.history_limit <= 0) { + this.event_history.clear(); + return; + } + while (this.event_history.size > this.history_limit) { + const firstKey = this.event_history.keys().next().value; + if (!firstKey) { + break; + } + this.event_history.delete(firstKey); + } + } +} diff --git a/dist/exceptions.d.ts b/dist/exceptions.d.ts new file mode 100644 index 00000000..fb12b0fc --- /dev/null +++ b/dist/exceptions.d.ts @@ -0,0 +1,10 @@ +export declare class LLMException extends Error { + readonly statusCode: number; + readonly detail: string; + constructor(statusCode: number, detail: string); +} +export declare class URLNotAllowedError extends Error { + readonly url: string; + readonly allowedDomains: string[]; + constructor(url: string, allowedDomains: string[]); +} diff --git a/dist/exceptions.js b/dist/exceptions.js new file mode 100644 index 00000000..8929b494 --- /dev/null +++ b/dist/exceptions.js @@ -0,0 +1,22 @@ +export class LLMException extends Error { + statusCode; + detail; + constructor(statusCode, detail) { + super(`Error ${statusCode}: ${detail}`); + this.statusCode = statusCode; + this.detail = detail; + this.name = 'LLMException'; + } +} +export class URLNotAllowedError extends Error { + url; + allowedDomains; + constructor(url, allowedDomains) { + super(`URL "${url}" is not allowed. ` + + `Only domains matching ${JSON.stringify(allowedDomains)} are permitted. ` + + `This is enforced because sensitive_data was provided to Agent.`); + this.url = url; + this.allowedDomains = allowedDomains; + this.name = 'URLNotAllowedError'; + } +} diff --git a/dist/filesystem/file-system.d.ts b/dist/filesystem/file-system.d.ts new file mode 100644 index 00000000..889103cd --- /dev/null +++ b/dist/filesystem/file-system.d.ts @@ -0,0 +1,86 @@ +export declare function extractPdfText(buffer: Buffer): Promise<{ + text: string; + totalPages: number; +}>; +export declare function extractPdfTextByPage(buffer: Buffer): Promise<{ + numPages: number; + pageTexts: string[]; + totalChars: number; +}>; +export declare const INVALID_FILENAME_ERROR_MESSAGE = "Error: Invalid filename format. Must be alphanumeric with supported extension."; +export declare const DEFAULT_FILE_SYSTEM_PATH = "browseruse_agent_data"; +export declare class FileSystemError extends Error { +} +declare abstract class BaseFile { + name: string; + protected content: string; + constructor(name: string, content?: string); + abstract get extension(): string; + get fullName(): string; + get size(): number; + get lineCount(): number; + protected writeFileContent(content: string): void; + protected appendFileContent(content: string): void; + read(): string; + syncToDisk(dir: string): Promise; + syncToDiskSync(dir: string): void; + write(content: string, dir: string): Promise; + writeSync(content: string, dir: string): void; + append(content: string, dir: string): Promise; + appendSync(content: string, dir: string): void; + toJSON(): { + name: string; + content: string; + }; +} +export interface FileState { + type: string; + data: { + name: string; + content: string; + }; +} +export interface FileSystemState { + files: Record; + base_dir: string; + extracted_content_count: number; +} +export declare class FileSystem { + private files; + private readonly defaultFiles; + private readonly baseDir; + readonly dataDir: string; + extractedContentCount: number; + constructor(baseDir: string, createDefaultFiles?: boolean); + private createDefaultFiles; + private isValidFilename; + static sanitize_filename(fileName: string): string; + private resolveFilename; + private parseFilename; + private getFileClass; + private instantiateFile; + get_allowed_extensions(): string[]; + get_dir(): string; + get_file(filename: string): BaseFile | null; + list_files(): string[]; + display_file(filename: string): string | null; + read_file_structured(filename: string, externalFile?: boolean): Promise<{ + message: string; + images: Array<{ + name: string; + data: string; + }> | null; + }>; + read_file(filename: string, externalFile?: boolean): Promise; + write_file(filename: string, content: string): Promise; + append_file(filename: string, content: string): Promise; + replace_file_str(filename: string, oldStr: string, newStr: string): Promise; + save_extracted_content(content: string): Promise; + describe(): string; + get_todo_contents(): string; + get_state(): FileSystemState; + nuke(): Promise; + static from_state_sync(state: FileSystemState): FileSystem; + static from_state(state: FileSystemState): Promise; +} +export {}; diff --git a/dist/filesystem/file-system.js b/dist/filesystem/file-system.js new file mode 100644 index 00000000..9173ac43 --- /dev/null +++ b/dist/filesystem/file-system.js @@ -0,0 +1,900 @@ +import fsSync from 'node:fs'; +import { promises as fsp } from 'node:fs'; +import path from 'node:path'; +import AdmZip from 'adm-zip'; +import PDFDocument from 'pdfkit'; +import { createRequire } from 'node:module'; +import { spawnSync } from 'node:child_process'; +const require = createRequire(import.meta.url); +export async function extractPdfText(buffer) { + const pdfParseModule = (await import('pdf-parse')); + if (typeof pdfParseModule.default === 'function') { + const legacyParser = pdfParseModule.default; + const parsed = await legacyParser(buffer); + return { + text: parsed.text ?? '', + totalPages: parsed.numpages ?? 0, + }; + } + if (typeof pdfParseModule.PDFParse === 'function') { + const Parser = pdfParseModule.PDFParse; + const parser = new Parser({ data: buffer }); + try { + const parsed = await parser.getText(); + return { + text: parsed.text ?? '', + totalPages: parsed.total ?? 0, + }; + } + finally { + if (typeof parser.destroy === 'function') { + await parser.destroy(); + } + } + } + throw new FileSystemError("Error: Could not parse PDF file due to unsupported 'pdf-parse' module format."); +} +export async function extractPdfTextByPage(buffer) { + const pdfParseModule = (await import('pdf-parse')); + if (typeof pdfParseModule.PDFParse === 'function') { + const Parser = pdfParseModule.PDFParse; + const parser = new Parser({ data: buffer }); + try { + let numPages = 0; + try { + const info = await parser.getInfo?.({ parsePageInfo: false }); + numPages = Number(info?.total ?? 0); + } + catch { + numPages = 0; + } + if (!Number.isFinite(numPages) || numPages <= 0) { + const full = await parser.getText(); + const text = typeof full?.text === 'string' ? full.text : ''; + return { + numPages: 1, + pageTexts: [text], + totalChars: text.length, + }; + } + const pageTexts = []; + let totalChars = 0; + for (let pageNumber = 1; pageNumber <= numPages; pageNumber += 1) { + const pageResult = await parser.getText({ partial: [pageNumber] }); + const text = typeof pageResult?.text === 'string' ? pageResult.text : ''; + pageTexts.push(text); + totalChars += text.length; + } + return { + numPages, + pageTexts, + totalChars, + }; + } + finally { + if (typeof parser.destroy === 'function') { + await parser.destroy(); + } + } + } + const parsed = await extractPdfText(buffer); + const text = parsed.text ?? ''; + return { + numPages: Math.max(parsed.totalPages, 1), + pageTexts: [text], + totalChars: text.length, + }; +} +export const INVALID_FILENAME_ERROR_MESSAGE = 'Error: Invalid filename format. Must be alphanumeric with supported extension.'; +export const DEFAULT_FILE_SYSTEM_PATH = 'browseruse_agent_data'; +const UNSUPPORTED_BINARY_EXTENSIONS = new Set([ + 'png', + 'jpg', + 'jpeg', + 'gif', + 'bmp', + 'svg', + 'webp', + 'ico', + 'mp3', + 'mp4', + 'wav', + 'avi', + 'mov', + 'zip', + 'tar', + 'gz', + 'rar', + 'exe', + 'bin', + 'dll', + 'so', +]); +const DEFAULT_EXTENSIONS = [ + 'md', + 'txt', + 'json', + 'jsonl', + 'csv', + 'pdf', + 'docx', + 'html', + 'xml', +]; +const escapeRegex = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +const buildFilenameRegex = (extensions) => new RegExp(`^[a-zA-Z0-9_\\-.() \\u4e00-\\u9fff]+\\.(${extensions.map(escapeRegex).join('|')})$`); +const buildFilenameErrorMessage = (fileName, supportedExtensions) => { + const base = path.basename(fileName); + const supported = supportedExtensions.map((ext) => `.${ext}`).join(', '); + if (base.includes('.')) { + const ext = base.slice(base.lastIndexOf('.') + 1).toLowerCase(); + if (UNSUPPORTED_BINARY_EXTENSIONS.has(ext)) { + return (`Error: Cannot write binary/image file '${base}'. ` + + 'The write_file tool only supports text-based files. ' + + `Supported extensions: ${supported}. ` + + 'For screenshots, the browser automatically captures them - do not try to save screenshots as files.'); + } + if (!supportedExtensions.includes(ext)) { + return (`Error: Unsupported file extension '.${ext}' in '${base}'. ` + + `Supported extensions: ${supported}. ` + + 'Please rename the file to use a supported extension.'); + } + } + else { + return (`Error: Filename '${base}' has no extension. ` + + `Please add a supported extension: ${supported}.`); + } + return (`Error: Invalid filename '${base}'. ` + + 'Filenames must contain only letters, numbers, underscores, hyphens, dots, parentheses, and spaces. ' + + `Supported extensions: ${supported}.`); +}; +const escapeXmlText = (value) => value + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +const decodeXmlText = (value) => value + .replace(/'/g, "'") + .replace(/"/g, '"') + .replace(/>/g, '>') + .replace(/</g, '<') + .replace(/&/g, '&'); +const DOCX_CONTENT_TYPES_XML = ` + + + + +`; +const DOCX_ROOT_RELS_XML = ` + + +`; +const DOCX_DOCUMENT_RELS_XML = ` +`; +const buildDocxDocumentXml = (content) => { + const lines = content.split(/\r?\n/); + const paragraphs = lines + .map((line) => { + if (!line) { + return ''; + } + return `${escapeXmlText(line)}`; + }) + .join(''); + return ` + + ${paragraphs} +`; +}; +const buildDocxBuffer = (content) => { + const zip = new AdmZip(); + zip.addFile('[Content_Types].xml', Buffer.from(DOCX_CONTENT_TYPES_XML, 'utf-8')); + zip.addFile('_rels/.rels', Buffer.from(DOCX_ROOT_RELS_XML, 'utf-8')); + zip.addFile('word/_rels/document.xml.rels', Buffer.from(DOCX_DOCUMENT_RELS_XML, 'utf-8')); + zip.addFile('word/document.xml', Buffer.from(buildDocxDocumentXml(content), 'utf-8')); + return zip.toBuffer(); +}; +const readDocxText = (fileBuffer) => { + const zip = new AdmZip(fileBuffer); + const documentEntry = zip.getEntry('word/document.xml'); + if (!documentEntry) { + throw new FileSystemError('Error: Could not parse DOCX file content.'); + } + const xml = documentEntry.getData().toString('utf-8'); + const normalizedXml = xml.replace(/]*)\/>/g, ''); + const paragraphMatches = normalizedXml.match(//g) ?? []; + const lines = paragraphMatches.map((paragraph) => { + const textMatches = Array.from(paragraph.matchAll(/]*)?>([\s\S]*?)<\/w:t>/g)); + if (!textMatches.length) { + return ''; + } + return textMatches.map((match) => decodeXmlText(match[1] ?? '')).join(''); + }); + return lines.join('\n').trim(); +}; +export class FileSystemError extends Error { +} +class BaseFile { + name; + content; + constructor(name, content = '') { + this.name = name; + this.content = content; + } + get fullName() { + return `${this.name}.${this.extension}`; + } + get size() { + return this.content.length; + } + get lineCount() { + return this.content ? this.content.split(/\r?\n/).length : 0; + } + writeFileContent(content) { + this.content = content; + } + appendFileContent(content) { + this.content = `${this.content}${content}`; + } + read() { + return this.content; + } + async syncToDisk(dir) { + await fsp.writeFile(path.join(dir, this.fullName), this.content, 'utf-8'); + } + syncToDiskSync(dir) { + fsSync.writeFileSync(path.join(dir, this.fullName), this.content, 'utf-8'); + } + async write(content, dir) { + this.writeFileContent(content); + await this.syncToDisk(dir); + } + writeSync(content, dir) { + this.writeFileContent(content); + this.syncToDiskSync(dir); + } + async append(content, dir) { + this.appendFileContent(content); + await this.syncToDisk(dir); + } + appendSync(content, dir) { + this.appendFileContent(content); + this.syncToDiskSync(dir); + } + toJSON() { + return { name: this.name, content: this.content }; + } +} +class MarkdownFile extends BaseFile { + get extension() { + return 'md'; + } +} +class TxtFile extends BaseFile { + get extension() { + return 'txt'; + } +} +class JsonFile extends BaseFile { + get extension() { + return 'json'; + } +} +class JsonlFile extends BaseFile { + get extension() { + return 'jsonl'; + } +} +class CsvFile extends BaseFile { + get extension() { + return 'csv'; + } +} +class PdfFile extends BaseFile { + get extension() { + return 'pdf'; + } + async syncToDisk(dir) { + const filePath = path.join(dir, this.fullName); + await new Promise((resolve, reject) => { + const doc = new PDFDocument({ autoFirstPage: true }); + const stream = fsSync.createWriteStream(filePath); + doc.pipe(stream); + doc.fontSize(12).text(this.content || '', { width: 500, align: 'left' }); + doc.end(); + stream.on('finish', resolve); + stream.on('error', reject); + }); + } + syncToDiskSync(dir) { + const filePath = path.join(dir, this.fullName); + const script = ` +const { createWriteStream } = require('fs'); +const PDFDocument = require(${JSON.stringify(require.resolve('pdfkit'))}); +const filePath = ${JSON.stringify(filePath)}; +const content = ${JSON.stringify(this.content ?? '')}; +const doc = new PDFDocument({ autoFirstPage: true }); +const stream = createWriteStream(filePath); +doc.pipe(stream); +doc.fontSize(12).text(content || '', { width: 500, align: 'left' }); +doc.end(); +stream.on('finish', () => process.exit(0)); +stream.on('error', (err) => { + console.error(err); + process.exit(1); +}); +`; + const result = spawnSync(process.execPath, ['-e', script], { + stdio: ['ignore', 'ignore', 'pipe'], + }); + if (result.status !== 0) { + const errorMsg = result.stderr?.toString() || + `Could not write to file '${this.fullName}'.`; + throw new FileSystemError(`Error: ${errorMsg.trim()}`); + } + } +} +class DocxFile extends BaseFile { + get extension() { + return 'docx'; + } + async syncToDisk(dir) { + const filePath = path.join(dir, this.fullName); + const docxBuffer = buildDocxBuffer(this.content || ''); + await fsp.writeFile(filePath, docxBuffer); + } + syncToDiskSync(dir) { + const filePath = path.join(dir, this.fullName); + const docxBuffer = buildDocxBuffer(this.content || ''); + fsSync.writeFileSync(filePath, docxBuffer); + } +} +class HtmlFile extends BaseFile { + get extension() { + return 'html'; + } +} +class XmlFile extends BaseFile { + get extension() { + return 'xml'; + } +} +const FILE_TYPES = { + md: MarkdownFile, + txt: TxtFile, + json: JsonFile, + jsonl: JsonlFile, + csv: CsvFile, + pdf: PdfFile, + docx: DocxFile, + html: HtmlFile, + xml: XmlFile, +}; +const TYPE_NAME_MAP = { + MarkdownFile, + TxtFile, + JsonFile, + JsonlFile, + CsvFile, + PdfFile, + DocxFile, + HtmlFile, + XmlFile, +}; +export class FileSystem { + files = new Map(); + defaultFiles = ['todo.md']; + baseDir; + dataDir; + extractedContentCount = 0; + constructor(baseDir, createDefaultFiles = true) { + this.baseDir = path.resolve(baseDir); + fsSync.mkdirSync(this.baseDir, { recursive: true }); + this.dataDir = path.join(this.baseDir, DEFAULT_FILE_SYSTEM_PATH); + if (fsSync.existsSync(this.dataDir)) { + fsSync.rmSync(this.dataDir, { recursive: true, force: true }); + } + fsSync.mkdirSync(this.dataDir, { recursive: true }); + if (createDefaultFiles) { + this.createDefaultFiles(); + } + } + createDefaultFiles() { + for (const filename of this.defaultFiles) { + const file = this.instantiateFile(filename); + this.files.set(filename, file); + fsSync.writeFileSync(path.join(this.dataDir, filename), file.read(), 'utf-8'); + } + } + isValidFilename(filename) { + const base = path.basename(filename); + const regex = buildFilenameRegex(this.get_allowed_extensions()); + if (!regex.test(base)) { + return false; + } + const idx = base.lastIndexOf('.'); + if (idx <= 0) { + return false; + } + return base.slice(0, idx).trim().length > 0; + } + static sanitize_filename(fileName) { + const base = path.basename(fileName); + const idx = base.lastIndexOf('.'); + if (idx === -1) { + return base; + } + const ext = base.slice(idx + 1).toLowerCase(); + let namePart = base.slice(0, idx); + namePart = namePart.replace(/ /g, '-'); + namePart = namePart.replace(/[^a-zA-Z0-9_\-.()\u4e00-\u9fff]/g, ''); + namePart = namePart.replace(/-{2,}/g, '-'); + namePart = namePart.replace(/^[-.]+|[-.]+$/g, ''); + if (!namePart) { + namePart = 'file'; + } + return `${namePart}.${ext}`; + } + resolveFilename(filename) { + const base = path.basename(filename); + const wasChanged = base !== filename; + if (this.isValidFilename(base)) { + return [base, wasChanged]; + } + const sanitized = FileSystem.sanitize_filename(base); + if (sanitized !== base && this.isValidFilename(sanitized)) { + return [sanitized, true]; + } + return [base, wasChanged]; + } + parseFilename(filename) { + const idx = filename.lastIndexOf('.'); + if (idx === -1) { + throw new FileSystemError(INVALID_FILENAME_ERROR_MESSAGE); + } + const name = filename.slice(0, idx); + const extension = filename.slice(idx + 1).toLowerCase(); + return [name, extension]; + } + getFileClass(extension) { + return FILE_TYPES[extension]; + } + instantiateFile(fullFilename, content = '') { + const [name, extension] = this.parseFilename(fullFilename); + const FileCtor = this.getFileClass(extension); + if (!FileCtor) { + throw new FileSystemError(INVALID_FILENAME_ERROR_MESSAGE); + } + return new FileCtor(name, content); + } + get_allowed_extensions() { + return Object.keys(FILE_TYPES); + } + get_dir() { + return this.dataDir; + } + get_file(filename) { + const [resolved] = this.resolveFilename(filename); + if (!this.isValidFilename(resolved)) { + return null; + } + return this.files.get(resolved) ?? null; + } + list_files() { + return Array.from(this.files.values()).map((file) => file.fullName); + } + display_file(filename) { + const [resolved] = this.resolveFilename(filename); + if (!this.isValidFilename(resolved)) { + return null; + } + const file = this.files.get(resolved) ?? null; + return file ? file.read() : null; + } + async read_file_structured(filename, externalFile = false) { + const result = { + message: '', + images: null, + }; + if (externalFile) { + try { + const base = path.basename(filename); + const idx = base.lastIndexOf('.'); + if (idx === -1) { + result.message = + `Error: Invalid filename format ${filename}. ` + + 'Must be alphanumeric with a supported extension.'; + return result; + } + const extension = base.slice(idx + 1).toLowerCase(); + const specialExtensions = new Set([ + 'docx', + 'pdf', + 'jpg', + 'jpeg', + 'png', + ]); + const textExtensions = this.get_allowed_extensions().filter((ext) => !specialExtensions.has(ext)); + if (textExtensions.includes(extension)) { + const content = await fsp.readFile(filename, 'utf-8'); + result.message = `Read from file ${filename}.\n\n${content}\n`; + return result; + } + if (extension === 'pdf') { + const MAX_CHARS = 60000; + const buffer = await fsp.readFile(filename); + const pdf = await extractPdfTextByPage(buffer); + const numPages = pdf.numPages; + const pageTexts = pdf.pageTexts; + const totalChars = pdf.totalChars; + if (totalChars <= MAX_CHARS) { + const contentParts = []; + for (let pageNumber = 1; pageNumber <= pageTexts.length; pageNumber += 1) { + const text = pageTexts[pageNumber - 1] ?? ''; + if (!text.trim()) { + continue; + } + contentParts.push(`--- Page ${pageNumber} ---\n${text}`); + } + result.message = + `Read from file ${filename} (${numPages} pages, ${totalChars.toLocaleString()} chars).\n` + + `\n${contentParts.join('\n\n')}\n`; + return result; + } + const wordToPages = new Map(); + const pageWords = new Map(); + for (let pageNumber = 1; pageNumber <= pageTexts.length; pageNumber += 1) { + const text = pageTexts[pageNumber - 1] ?? ''; + const words = new Set((text.toLowerCase().match(/\b[a-zA-Z]{4,}\b/g) ?? []).map((word) => word)); + pageWords.set(pageNumber, words); + for (const word of words) { + if (!wordToPages.has(word)) { + wordToPages.set(word, new Set()); + } + wordToPages.get(word).add(pageNumber); + } + } + const pageScores = new Map(); + for (const [pageNumber, words] of pageWords.entries()) { + let score = 0; + for (const word of words) { + const pagesWithWord = wordToPages.get(word)?.size ?? 1; + score += Math.log(Math.max(numPages, 1) / pagesWithWord); + } + pageScores.set(pageNumber, score); + } + const priorityPages = [1]; + const sortedPages = Array.from(pageScores.entries()).sort((a, b) => b[1] - a[1]); + for (const [pageNumber] of sortedPages) { + if (!priorityPages.includes(pageNumber)) { + priorityPages.push(pageNumber); + } + } + for (let pageNumber = 1; pageNumber <= numPages; pageNumber += 1) { + if (!priorityPages.includes(pageNumber)) { + priorityPages.push(pageNumber); + } + } + const contentParts = []; + let charsUsed = 0; + const pagesIncluded = []; + const pagesIncludedSet = new Set(); + for (const pageNumber of priorityPages) { + const text = pageTexts[pageNumber - 1] ?? ''; + if (!text.trim()) { + continue; + } + const pageHeader = `--- Page ${pageNumber} ---\n`; + const truncationSuffix = '\n[...truncated]'; + const remaining = MAX_CHARS - charsUsed; + const minUseful = pageHeader.length + truncationSuffix.length + 50; + if (remaining < minUseful) { + break; + } + let pageContent = `${pageHeader}${text}`; + if (pageContent.length > remaining) { + pageContent = + pageContent.slice(0, Math.max(0, remaining - truncationSuffix.length)) + truncationSuffix; + } + contentParts.push({ pageNumber, content: pageContent }); + charsUsed += pageContent.length; + pagesIncluded.push(pageNumber); + pagesIncludedSet.add(pageNumber); + if (charsUsed >= MAX_CHARS) { + break; + } + } + contentParts.sort((a, b) => a.pageNumber - b.pageNumber); + const extractedText = contentParts + .map((part) => part.content) + .join('\n\n'); + let truncationNote = ''; + const pagesNotShown = numPages - pagesIncluded.length; + if (pagesNotShown > 0) { + const skipped = []; + for (let pageNumber = 1; pageNumber <= numPages; pageNumber += 1) { + if (!pagesIncludedSet.has(pageNumber)) { + skipped.push(pageNumber); + } + } + const skippedPreview = skipped.slice(0, 10).join(', '); + const skippedSuffix = skipped.length > 10 ? ', ...' : ''; + truncationNote = + `\n\n[Showing ${pagesIncluded.length} of ${numPages} pages. ` + + `Skipped pages: [${skippedPreview}${skippedSuffix}]. ` + + 'Use extract with start_from_char to read further into the file.]'; + } + result.message = + `Read from file ${filename} (${numPages} pages, ${totalChars.toLocaleString()} chars total).\n` + + `\n${extractedText}${truncationNote}\n`; + return result; + } + if (extension === 'docx') { + const fileBuffer = await fsp.readFile(filename); + const content = readDocxText(fileBuffer); + result.message = `Read from file ${filename}.\n\n${content}\n`; + return result; + } + if (extension === 'jpg' || + extension === 'jpeg' || + extension === 'png') { + const fileBuffer = await fsp.readFile(filename); + result.message = `Read image file ${filename}.`; + result.images = [ + { + name: base, + data: fileBuffer.toString('base64'), + }, + ]; + return result; + } + result.message = `Error: Cannot read file ${filename} as ${extension} extension is not supported.`; + return result; + } + catch (error) { + if (error?.code === 'ENOENT') { + result.message = `Error: File '${filename}' not found.`; + return result; + } + if (error?.code === 'EACCES') { + result.message = `Error: Permission denied to read file '${filename}'.`; + return result; + } + result.message = + `Error: Could not read file '${filename}'. ${error instanceof Error ? error.message : ''}`.trim(); + return result; + } + } + const originalFilename = filename; + const [resolved, wasSanitized] = this.resolveFilename(filename); + if (!this.isValidFilename(resolved)) { + result.message = buildFilenameErrorMessage(filename, this.get_allowed_extensions()); + return result; + } + const file = this.files.get(resolved) ?? null; + if (!file) { + if (wasSanitized) { + result.message = + `File '${resolved}' not found. ` + + `(Filename was auto-corrected from '${originalFilename}')`; + } + else { + result.message = `File '${originalFilename}' not found.`; + } + return result; + } + try { + const content = file.read(); + const sanitizeNote = wasSanitized + ? `Note: filename was auto-corrected from '${originalFilename}' to '${resolved}'. ` + : ''; + result.message = `${sanitizeNote}Read from file ${resolved}.\n\n${content}\n`; + return result; + } + catch (error) { + result.message = + error instanceof FileSystemError + ? error.message + : `Error: Could not read file '${originalFilename}'.`; + return result; + } + } + async read_file(filename, externalFile = false) { + const result = await this.read_file_structured(filename, externalFile); + return result.message; + } + async write_file(filename, content) { + const originalFilename = filename; + const [resolved, wasSanitized] = this.resolveFilename(filename); + if (!this.isValidFilename(resolved)) { + return buildFilenameErrorMessage(filename, this.get_allowed_extensions()); + } + filename = resolved; + const file = this.files.get(filename) ?? this.instantiateFile(filename); + this.files.set(filename, file); + try { + await file.write(content, this.dataDir); + const sanitizeNote = wasSanitized + ? ` (auto-corrected from '${originalFilename}')` + : ''; + return `Data written to file ${filename} successfully.${sanitizeNote}`; + } + catch (error) { + return `Error: Could not write to file '${filename}'. ${error.message}`; + } + } + async append_file(filename, content) { + const originalFilename = filename; + const [resolved, wasSanitized] = this.resolveFilename(filename); + if (!this.isValidFilename(resolved)) { + return buildFilenameErrorMessage(filename, this.get_allowed_extensions()); + } + filename = resolved; + const file = this.get_file(filename); + if (!file) { + if (wasSanitized) { + return (`File '${filename}' not found. ` + + `(Filename was auto-corrected from '${originalFilename}')`); + } + return `File '${filename}' not found.`; + } + try { + await file.append(content, this.dataDir); + const sanitizeNote = wasSanitized + ? ` (auto-corrected from '${originalFilename}')` + : ''; + return `Data appended to file ${filename} successfully.${sanitizeNote}`; + } + catch (error) { + return `Error: Could not append to file '${filename}'. ${error.message}`; + } + } + async replace_file_str(filename, oldStr, newStr) { + const originalFilename = filename; + const [resolved, wasSanitized] = this.resolveFilename(filename); + if (!this.isValidFilename(resolved)) { + return buildFilenameErrorMessage(filename, this.get_allowed_extensions()); + } + filename = resolved; + if (!oldStr) { + return 'Error: Cannot replace empty string. Please provide a non-empty string to replace.'; + } + const file = this.get_file(filename); + if (!file) { + if (wasSanitized) { + return (`File '${filename}' not found. ` + + `(Filename was auto-corrected from '${originalFilename}')`); + } + return `File '${filename}' not found.`; + } + try { + const content = file.read().replaceAll(oldStr, newStr); + await file.write(content, this.dataDir); + const sanitizeNote = wasSanitized + ? ` (auto-corrected from '${originalFilename}')` + : ''; + return (`Successfully replaced all occurrences of "${oldStr}" with "${newStr}" in file ${filename}` + + sanitizeNote); + } + catch (error) { + return `Error: Could not replace string in file '${filename}'. ${error.message}`; + } + } + async save_extracted_content(content) { + const filename = `extracted_content_${this.extractedContentCount}.md`; + const file = new MarkdownFile(`extracted_content_${this.extractedContentCount}`); + await file.write(content, this.dataDir); + this.files.set(filename, file); + this.extractedContentCount += 1; + return filename; + } + describe() { + const DISPLAY_CHARS = 400; + let description = ''; + for (const file of this.files.values()) { + if (file.fullName === 'todo.md') { + continue; + } + const content = file.read(); + if (!content) { + description += `\n${file.fullName} - [empty file]\n\n`; + continue; + } + const lines = content.split(/\r?\n/); + const lineCount = lines.length; + if (content.length < DISPLAY_CHARS * 1.5) { + description += `\n${file.fullName} - ${lineCount} lines\n\n${content}\n\n\n`; + continue; + } + const halfChars = Math.floor(DISPLAY_CHARS / 2); + let startPreview = ''; + let startLines = 0; + let accumulated = 0; + for (const line of lines) { + if (accumulated + line.length + 1 > halfChars) { + break; + } + startPreview += `${line}\n`; + accumulated += line.length + 1; + startLines += 1; + } + let endPreview = ''; + let endLines = 0; + accumulated = 0; + for (let i = lines.length - 1; i >= 0; i -= 1) { + const line = lines[i]; + if (accumulated + line.length + 1 > halfChars) { + break; + } + endPreview = `${line}\n${endPreview}`; + accumulated += line.length + 1; + endLines += 1; + } + const middleLines = lineCount - startLines - endLines; + if (middleLines <= 0) { + description += `\n${file.fullName} - ${lineCount} lines\n\n${content}\n\n\n`; + continue; + } + description += `\n${file.fullName} - ${lineCount} lines\n\n${startPreview.trim()}\n`; + description += `... ${middleLines} more lines ...\n`; + description += `${endPreview.trim()}\n\n\n`; + } + return description.trim(); + } + get_todo_contents() { + const todo = this.get_file('todo.md'); + return todo?.read() ?? ''; + } + get_state() { + const files = {}; + for (const [filename, file] of this.files.entries()) { + files[filename] = { type: file.constructor.name, data: file.toJSON() }; + } + return { + files, + base_dir: this.baseDir, + extracted_content_count: this.extractedContentCount, + }; + } + async nuke() { + await fsp.rm(this.dataDir, { recursive: true, force: true }); + } + static from_state_sync(state) { + const fsInstance = new FileSystem(state.base_dir, false); + fsInstance.extractedContentCount = state.extracted_content_count; + for (const [filename, fileState] of Object.entries(state.files)) { + const FileCtor = TYPE_NAME_MAP[fileState.type]; + if (!FileCtor) { + continue; + } + const file = new FileCtor(fileState.data.name, fileState.data.content); + fsInstance.files.set(filename, file); + try { + file.writeSync(fileState.data.content, fsInstance.dataDir); + } + catch (error) { + throw new FileSystemError(`Error restoring file '${filename}': ${error.message}`); + } + } + return fsInstance; + } + static async from_state(state) { + return FileSystem.from_state_sync(state); + } +} diff --git a/dist/filesystem/index.d.ts b/dist/filesystem/index.d.ts new file mode 100644 index 00000000..80e92416 --- /dev/null +++ b/dist/filesystem/index.d.ts @@ -0,0 +1 @@ +export * from './file-system.js'; diff --git a/dist/filesystem/index.js b/dist/filesystem/index.js new file mode 100644 index 00000000..80e92416 --- /dev/null +++ b/dist/filesystem/index.js @@ -0,0 +1 @@ +export * from './file-system.js'; diff --git a/dist/index.d.ts b/dist/index.d.ts new file mode 100644 index 00000000..ae6c56f3 --- /dev/null +++ b/dist/index.d.ts @@ -0,0 +1,38 @@ +export * from './config.js'; +export * from './logging-config.js'; +export { observe, observe_debug, isLmnrAvailable, isDebugMode, getObservabilityStatus, } from './observability.js'; +export { observeDebug, observeDebugMethod, OperationTrace, trackPerformance, withObservability, PerformanceCounter, } from './observability-decorators.js'; +export { URLNotAllowedError as BaseURLNotAllowedError } from './exceptions.js'; +export * from './utils.js'; +export * from './browser/index.js'; +export * from './dom/views.js'; +export * from './dom/history-tree-processor/view.js'; +export * from './dom/history-tree-processor/service.js'; +export * from './dom/service.js'; +export * from './dom/clickable-element-processor/service.js'; +export * from './screenshots/service.js'; +export * from './controller/views.js'; +export * from './controller/registry/service.js'; +export * from './controller/service.js'; +export { Tools } from './tools/service.js'; +export type { ToolsOptions, ToolsActParams } from './tools/service.js'; +export * from './filesystem/file-system.js'; +export * from './agent/views.js'; +export * from './telemetry/views.js'; +export * from './telemetry/service.js'; +export * from './llm/messages.js'; +export * from './llm/models.js'; +export * from './llm/views.js'; +export * from './llm/base.js'; +export * from './llm/exceptions.js'; +export * from './llm/schema.js'; +export * from './tokens/views.js'; +export * from './agent/message-manager/views.js'; +export * from './agent/prompts.js'; +export * from './agent/message-manager/service.js'; +export * from './agent/service.js'; +export * from './agent/message-manager/utils.js'; +export * from './skills/index.js'; +export * from './sandbox/index.js'; +export * from './code-use/index.js'; +export * from './skill-cli/index.js'; diff --git a/dist/index.js b/dist/index.js new file mode 100644 index 00000000..8eec57e6 --- /dev/null +++ b/dist/index.js @@ -0,0 +1,39 @@ +export * from './config.js'; +export * from './logging-config.js'; +// Export observability - note observeDebug exists in both files +export { observe, observe_debug, isLmnrAvailable, isDebugMode, getObservabilityStatus, } from './observability.js'; +export { observeDebug, observeDebugMethod, OperationTrace, trackPerformance, withObservability, PerformanceCounter, } from './observability-decorators.js'; +// Export exceptions - note URLNotAllowedError defined in both exceptions.js and browser/views.js +export { URLNotAllowedError as BaseURLNotAllowedError } from './exceptions.js'; +export * from './utils.js'; +export * from './browser/index.js'; +export * from './dom/views.js'; +export * from './dom/history-tree-processor/view.js'; +export * from './dom/history-tree-processor/service.js'; +export * from './dom/service.js'; +export * from './dom/clickable-element-processor/service.js'; +export * from './screenshots/service.js'; +export * from './controller/views.js'; +export * from './controller/registry/service.js'; +export * from './controller/service.js'; +export { Tools } from './tools/service.js'; +export * from './filesystem/file-system.js'; +export * from './agent/views.js'; +export * from './telemetry/views.js'; +export * from './telemetry/service.js'; +export * from './llm/messages.js'; +export * from './llm/models.js'; +export * from './llm/views.js'; +export * from './llm/base.js'; +export * from './llm/exceptions.js'; +export * from './llm/schema.js'; +export * from './tokens/views.js'; +export * from './agent/message-manager/views.js'; +export * from './agent/prompts.js'; +export * from './agent/message-manager/service.js'; +export * from './agent/service.js'; +export * from './agent/message-manager/utils.js'; +export * from './skills/index.js'; +export * from './sandbox/index.js'; +export * from './code-use/index.js'; +export * from './skill-cli/index.js'; diff --git a/dist/integrations/gmail/actions.d.ts b/dist/integrations/gmail/actions.d.ts new file mode 100644 index 00000000..eb3c7b30 --- /dev/null +++ b/dist/integrations/gmail/actions.d.ts @@ -0,0 +1,12 @@ +/** + * Gmail Actions for Browser Use + * Defines agent actions for Gmail integration including 2FA code retrieval, + * email reading, and authentication management. + */ +import { GmailService } from './service.js'; +import type { Tools } from '../../tools/service.js'; +/** + * Register Gmail actions with the provided tools registry + */ +export declare function registerGmailActions(tools: Tools, gmailService?: GmailService | null, accessToken?: string | null): Tools; +export declare const register_gmail_actions: typeof registerGmailActions; diff --git a/dist/integrations/gmail/actions.js b/dist/integrations/gmail/actions.js new file mode 100644 index 00000000..c74dfe9e --- /dev/null +++ b/dist/integrations/gmail/actions.js @@ -0,0 +1,113 @@ +/** + * Gmail Actions for Browser Use + * Defines agent actions for Gmail integration including 2FA code retrieval, + * email reading, and authentication management. + */ +import { z } from 'zod'; +import { createLogger } from '../../logging-config.js'; +import { ActionResult } from '../../agent/views.js'; +import { GmailService } from './service.js'; +const logger = createLogger('browser_use.gmail.actions'); +// Global Gmail service instance - initialized when actions are registered +let _gmailService = null; +// Schema for get_recent_emails action +const GetRecentEmailsParamsSchema = z.object({ + keyword: z + .string() + .default('') + .describe('A single keyword for search, e.g. github, airbnb, etc.'), + max_results: z + .number() + .int() + .min(1) + .max(50) + .default(3) + .describe('Maximum number of emails to retrieve (1-50, default: 3)'), +}); +/** + * Register Gmail actions with the provided tools registry + */ +export function registerGmailActions(tools, gmailService, accessToken) { + // Use provided service or create a new one with access token if provided + if (gmailService) { + _gmailService = gmailService; + } + else if (accessToken) { + _gmailService = new GmailService({ access_token: accessToken }); + } + else { + _gmailService = new GmailService(); + } + // Register get_recent_emails action + tools.registry.action('Get recent emails from the mailbox with a keyword to retrieve verification codes, OTP, 2FA tokens, magic links, or any recent email content. Keep your query a single keyword.', GetRecentEmailsParamsSchema)(async (params) => { + try { + if (!_gmailService) { + throw new Error('Gmail service not initialized'); + } + // Ensure authentication + if (!_gmailService.isAuthenticated()) { + logger.info('📧 Gmail not authenticated, attempting authentication...'); + const authenticated = await _gmailService.authenticate(); + if (!authenticated) { + return new ActionResult({ + extracted_content: 'Failed to authenticate with Gmail. Please ensure Gmail credentials are set up properly.', + long_term_memory: 'Gmail authentication failed', + }); + } + } + // Use specified max_results (1-50, default 3), last 5 minutes + const maxResults = params.max_results; + const timeFilter = '5m'; + // Build query with time filter and optional user query + const queryParts = [`newer_than:${timeFilter}`]; + if (params.keyword.trim()) { + queryParts.push(params.keyword.trim()); + } + const query = queryParts.join(' '); + logger.info(`🔍 Gmail search query: ${query}`); + // Get emails + const emails = await _gmailService.getRecentEmails({ + max_results: maxResults, + query: query, + time_filter: timeFilter, + }); + if (!emails.length) { + const queryInfo = params.keyword.trim() + ? ` matching '${params.keyword}'` + : ''; + const memory = `No recent emails found from last ${timeFilter}${queryInfo}`; + return new ActionResult({ + extracted_content: memory, + long_term_memory: memory, + }); + } + // Format with full email content for large display + let content = `Found ${emails.length} recent email${emails.length > 1 ? 's' : ''} from the last ${timeFilter}:\n\n`; + for (let i = 0; i < emails.length; i++) { + const email = emails[i]; + content += `Email ${i + 1}:\n`; + content += `From: ${email.from}\n`; + content += `Subject: ${email.subject}\n`; + content += `Date: ${email.date}\n`; + content += `Content:\n${email.body}\n`; + content += '-'.repeat(50) + '\n\n'; + } + logger.info(`📧 Retrieved ${emails.length} recent emails`); + return new ActionResult({ + extracted_content: content, + include_extracted_content_only_once: true, + long_term_memory: `Retrieved ${emails.length} recent emails from last ${timeFilter} for query ${query}.`, + }); + } + catch (error) { + logger.error(`Error getting recent emails: ${error.message || error}`); + return new ActionResult({ + error: `Error getting recent emails: ${error.message || error}`, + long_term_memory: 'Failed to get recent emails due to error', + }); + } + }); + return tools; +} +// Backward compatibility export +export const register_gmail_actions = registerGmailActions; diff --git a/dist/integrations/gmail/index.d.ts b/dist/integrations/gmail/index.d.ts new file mode 100644 index 00000000..33ad2568 --- /dev/null +++ b/dist/integrations/gmail/index.d.ts @@ -0,0 +1,2 @@ +export * from './service.js'; +export * from './actions.js'; diff --git a/dist/integrations/gmail/index.js b/dist/integrations/gmail/index.js new file mode 100644 index 00000000..33ad2568 --- /dev/null +++ b/dist/integrations/gmail/index.js @@ -0,0 +1,2 @@ +export * from './service.js'; +export * from './actions.js'; diff --git a/dist/integrations/gmail/service.d.ts b/dist/integrations/gmail/service.d.ts new file mode 100644 index 00000000..8223e54a --- /dev/null +++ b/dist/integrations/gmail/service.d.ts @@ -0,0 +1,61 @@ +/** + * Gmail API Service for Browser Use + * Handles Gmail API authentication, email reading, and 2FA code extraction. + * This service provides a clean interface for agents to interact with Gmail. + */ +import type { gmail_v1 } from 'googleapis'; +export interface EmailData { + id: string; + thread_id: string; + subject: string; + from: string; + to: string; + date: string; + timestamp: number; + body: string; + raw_message: gmail_v1.Schema$Message; +} +export declare class GmailService { + private static readonly SCOPES; + private configDir; + private credentialsFile; + private tokenFile; + private accessToken; + private service; + private creds; + private _authenticated; + constructor(options?: { + credentials_file?: string; + token_file?: string; + config_dir?: string; + access_token?: string; + }); + /** + * Check if Gmail service is authenticated + */ + isAuthenticated(): boolean; + /** + * Handle OAuth authentication and token management + */ + authenticate(): Promise; + /** + * Get recent emails with optional query filter + */ + getRecentEmails(options?: { + max_results?: number; + query?: string; + time_filter?: string; + }): Promise; + /** + * Parse Gmail message into readable format + */ + private _parseEmail; + /** + * Extract email body from payload + */ + private _extractBody; + /** + * Send an email + */ + sendMessage(to: string, subject: string, body: string): Promise; +} diff --git a/dist/integrations/gmail/service.js b/dist/integrations/gmail/service.js new file mode 100644 index 00000000..272e6ee8 --- /dev/null +++ b/dist/integrations/gmail/service.js @@ -0,0 +1,260 @@ +/** + * Gmail API Service for Browser Use + * Handles Gmail API authentication, email reading, and 2FA code extraction. + * This service provides a clean interface for agents to interact with Gmail. + */ +import path from 'node:path'; +import fs from 'node:fs'; +import { google } from 'googleapis'; +import { createLogger } from '../../logging-config.js'; +import { CONFIG } from '../../config.js'; +const logger = createLogger('browser_use.gmail'); +export class GmailService { + static SCOPES = [ + 'https://www.googleapis.com/auth/gmail.readonly', + ]; + configDir; + credentialsFile; + tokenFile; + accessToken; + service = null; + creds = null; + _authenticated = false; + constructor(options = {}) { + // Set up configuration directory + this.configDir = options.config_dir || CONFIG.BROWSER_USE_CONFIG_DIR; + // Direct access token support + this.accessToken = options.access_token || null; + // Ensure config directory exists (only if not using direct token) + if (!this.accessToken) { + if (!fs.existsSync(this.configDir)) { + fs.mkdirSync(this.configDir, { recursive: true }); + } + } + // Set up credential paths + this.credentialsFile = + options.credentials_file || + path.join(this.configDir, 'gmail_credentials.json'); + this.tokenFile = + options.token_file || path.join(this.configDir, 'gmail_token.json'); + } + /** + * Check if Gmail service is authenticated + */ + isAuthenticated() { + return this._authenticated && this.service !== null; + } + /** + * Handle OAuth authentication and token management + */ + async authenticate() { + try { + logger.info('🔐 Authenticating with Gmail API...'); + // Check if using direct access token + if (this.accessToken) { + logger.info('🔑 Using provided access token'); + const auth = new google.auth.OAuth2(); + auth.setCredentials({ access_token: this.accessToken }); + this.service = google.gmail({ version: 'v1', auth }); + this._authenticated = true; + logger.info('✅ Gmail API ready with access token!'); + return true; + } + // Original file-based authentication flow + // Try to load existing tokens + if (fs.existsSync(this.tokenFile)) { + const tokenData = JSON.parse(fs.readFileSync(this.tokenFile, 'utf-8')); + const auth = new google.auth.OAuth2(); + auth.setCredentials(tokenData); + this.creds = auth; + logger.debug('📁 Loaded existing tokens'); + } + // If no valid credentials, run OAuth flow + if (!this.creds || + !this.creds.credentials || + !this.creds.credentials.access_token) { + if (this.creds && + this.creds.credentials && + this.creds.credentials.refresh_token) { + logger.info('🔄 Refreshing expired tokens...'); + await this.creds.refreshAccessToken(); + } + else { + logger.info('🌐 Starting OAuth flow...'); + if (!fs.existsSync(this.credentialsFile)) { + logger.error(`❌ Gmail credentials file not found: ${this.credentialsFile}\n` + + 'Please download it from Google Cloud Console:\n' + + '1. Go to https://console.cloud.google.com/\n' + + '2. APIs & Services > Credentials\n' + + '3. Download OAuth 2.0 Client JSON\n' + + `4. Save as 'gmail_credentials.json' in ${this.configDir}/`); + return false; + } + const credentials = JSON.parse(fs.readFileSync(this.credentialsFile, 'utf-8')); + const { client_secret, client_id, redirect_uris } = credentials.installed || credentials.web; + const oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]); + const authUrl = oAuth2Client.generateAuthUrl({ + access_type: 'offline', + scope: GmailService.SCOPES, + }); + logger.info(`🔗 Please visit this URL to authorize:\n${authUrl}`); + logger.info('⏳ Waiting for authorization code...'); + // Note: In a real implementation, you would use a web server to handle the OAuth callback + // For now, we'll throw an error with instructions + throw new Error('OAuth flow requires manual intervention. Please:\n' + + `1. Visit: ${authUrl}\n` + + '2. Authorize the application\n' + + '3. Copy the authorization code\n' + + '4. Implement token exchange logic'); + } + // Save tokens for next time + fs.writeFileSync(this.tokenFile, JSON.stringify(this.creds.credentials)); + logger.info(`💾 Tokens saved to ${this.tokenFile}`); + } + // Build Gmail service + this.service = google.gmail({ version: 'v1', auth: this.creds }); + this._authenticated = true; + logger.info('✅ Gmail API ready!'); + return true; + } + catch (error) { + logger.error(`❌ Gmail authentication failed: ${error}`); + return false; + } + } + /** + * Get recent emails with optional query filter + */ + async getRecentEmails(options = {}) { + const { max_results = 10, query = '', time_filter = '1h' } = options; + if (!this.isAuthenticated()) { + logger.error('❌ Gmail service not authenticated. Call authenticate() first.'); + return []; + } + try { + // Add time filter to query if provided + let fullQuery = query; + if (time_filter && !query.includes('newer_than:')) { + fullQuery = `newer_than:${time_filter} ${query}`.trim(); + } + logger.info(`📧 Fetching ${max_results} recent emails...`); + if (fullQuery) { + logger.debug(`🔍 Query: ${fullQuery}`); + } + // Get message list + const results = (await this.service.users.messages.list({ + userId: 'me', + maxResults: max_results, + q: fullQuery, + })); + const messages = results.data.messages || []; + if (!messages.length) { + logger.info('📭 No messages found'); + return []; + } + logger.info(`📨 Found ${messages.length} messages, fetching details...`); + // Get full message details + const emails = []; + for (let i = 0; i < messages.length; i++) { + const message = messages[i]; + logger.debug(`📖 Reading email ${i + 1}/${messages.length}...`); + const fullMessage = (await this.service.users.messages.get({ + userId: 'me', + id: message.id, + format: 'full', + })); + const emailData = this._parseEmail(fullMessage.data); + emails.push(emailData); + } + return emails; + } + catch (error) { + logger.error(`❌ Gmail API error: ${error.message || error}`); + return []; + } + } + /** + * Parse Gmail message into readable format + */ + _parseEmail(message) { + const headers = message.payload?.headers || []; + const headerMap = {}; + for (const header of headers) { + if (header.name && header.value) { + headerMap[header.name] = header.value; + } + } + return { + id: message.id || '', + thread_id: message.threadId || '', + subject: headerMap['Subject'] || '', + from: headerMap['From'] || '', + to: headerMap['To'] || '', + date: headerMap['Date'] || '', + timestamp: parseInt(message.internalDate || '0'), + body: this._extractBody(message.payload || {}), + raw_message: message, + }; + } + /** + * Extract email body from payload + */ + _extractBody(payload) { + let body = ''; + if (payload.body?.data) { + // Simple email body + body = Buffer.from(payload.body.data, 'base64').toString('utf-8'); + } + else if (payload.parts) { + // Multi-part email + for (const part of payload.parts) { + if (part.mimeType === 'text/plain' && part.body?.data) { + const partBody = Buffer.from(part.body.data, 'base64').toString('utf-8'); + body += partBody; + } + else if (part.mimeType === 'text/html' && !body && part.body?.data) { + // Fallback to HTML if no plain text + body = Buffer.from(part.body.data, 'base64').toString('utf-8'); + } + } + } + return body; + } + /** + * Send an email + */ + async sendMessage(to, subject, body) { + if (!this.isAuthenticated()) { + logger.error('❌ Gmail service not authenticated.'); + return null; + } + try { + const utf8Subject = `=?utf-8?B?${Buffer.from(subject).toString('base64')}?=`; + const messageParts = [ + `To: ${to}`, + 'Content-Type: text/html; charset=utf-8', + 'MIME-Version: 1.0', + `Subject: ${utf8Subject}`, + '', + body, + ]; + const message = messageParts.join('\n'); + const encodedMessage = Buffer.from(message) + .toString('base64') + .replace(/\+/g, '-') + .replace(/\//g, '_') + .replace(/=+$/, ''); + const res = await this.service.users.messages.send({ + userId: 'me', + requestBody: { + raw: encodedMessage, + }, + }); + return res.data; + } + catch (error) { + logger.error(`❌ Failed to send email: ${error.message || error}`); + return null; + } + } +} diff --git a/dist/llm/anthropic/chat.d.ts b/dist/llm/anthropic/chat.d.ts new file mode 100644 index 00000000..d7010af6 --- /dev/null +++ b/dist/llm/anthropic/chat.d.ts @@ -0,0 +1,45 @@ +import { type ClientOptions } from '@anthropic-ai/sdk'; +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { type Message } from '../messages.js'; +export interface ChatAnthropicOptions { + model?: string; + apiKey?: string; + authToken?: string; + baseURL?: string; + timeout?: number; + maxTokens?: number; + temperature?: number | null; + topP?: number | null; + seed?: number | null; + maxRetries?: number; + defaultHeaders?: Record; + defaultQuery?: Record; + fetchImplementation?: ClientOptions['fetch']; + fetchOptions?: ClientOptions['fetchOptions']; + removeMinItemsFromSchema?: boolean; + removeDefaultsFromSchema?: boolean; +} +export declare class ChatAnthropic implements BaseChatModel { + model: string; + provider: string; + private client; + private maxTokens; + private temperature; + private topP; + private seed; + private removeMinItemsFromSchema; + private removeDefaultsFromSchema; + constructor(options?: string | ChatAnthropicOptions); + get name(): string; + get model_name(): string; + private getModelParams; + private getZodSchemaCandidate; + private parseOutput; + private getTextCompletion; + private getUsage; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/anthropic/chat.js b/dist/llm/anthropic/chat.js new file mode 100644 index 00000000..2fe95db9 --- /dev/null +++ b/dist/llm/anthropic/chat.js @@ -0,0 +1,194 @@ +import Anthropic, { APIConnectionError, APIError, RateLimitError, } from '@anthropic-ai/sdk'; +import { ChatInvokeCompletion } from '../views.js'; +import { AnthropicMessageSerializer } from './serializer.js'; +import { ModelProviderError, ModelRateLimitError } from '../exceptions.js'; +import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js'; +export class ChatAnthropic { + model; + provider = 'anthropic'; + client; + maxTokens; + temperature; + topP; + seed; + removeMinItemsFromSchema; + removeDefaultsFromSchema; + constructor(options = {}) { + const normalizedOptions = typeof options === 'string' ? { model: options } : options; + const { model = 'claude-sonnet-4-20250514', apiKey = process.env.ANTHROPIC_API_KEY, authToken = process.env.ANTHROPIC_AUTH_TOKEN, baseURL, timeout, maxTokens = 8192, temperature = null, topP = null, seed = null, maxRetries = 10, defaultHeaders, defaultQuery, fetchImplementation, fetchOptions, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions; + this.model = model; + this.maxTokens = maxTokens; + this.temperature = temperature; + this.topP = topP; + this.seed = seed; + this.removeMinItemsFromSchema = removeMinItemsFromSchema; + this.removeDefaultsFromSchema = removeDefaultsFromSchema; + this.client = new Anthropic({ + apiKey, + authToken, + baseURL, + timeout, + maxRetries, + defaultHeaders, + defaultQuery, + ...(fetchImplementation ? { fetch: fetchImplementation } : {}), + ...(fetchOptions ? { fetchOptions } : {}), + }); + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + getModelParams() { + const modelParams = {}; + if (this.temperature !== null) { + modelParams.temperature = this.temperature; + } + if (this.topP !== null) { + modelParams.top_p = this.topP; + } + if (this.seed !== null) { + modelParams.seed = this.seed; + } + return modelParams; + } + getZodSchemaCandidate(output_format) { + const output = output_format; + if (output && + typeof output === 'object' && + typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return output; + } + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return output.schema; + } + return null; + } + parseOutput(output_format, payload) { + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + return output.schema.parse(payload); + } + return output.parse(payload); + } + getTextCompletion(response) { + const textBlock = response.content.find((block) => block.type === 'text'); + if (textBlock && textBlock.type === 'text') { + return textBlock.text; + } + const firstBlock = response.content[0]; + return firstBlock ? String(firstBlock) : ''; + } + getUsage(response) { + const cacheReadTokens = response.usage.cache_read_input_tokens ?? 0; + const cacheCreationTokens = response.usage.cache_creation_input_tokens ?? 0; + return { + prompt_tokens: response.usage.input_tokens + cacheReadTokens, + completion_tokens: response.usage.output_tokens, + total_tokens: response.usage.input_tokens + response.usage.output_tokens, + prompt_cached_tokens: cacheReadTokens || null, + prompt_cache_creation_tokens: cacheCreationTokens || null, + prompt_image_tokens: null, + }; + } + async ainvoke(messages, output_format, options = {}) { + const serializer = new AnthropicMessageSerializer(); + const [anthropicMessages, systemPrompt] = serializer.serializeMessages(messages); + const zodSchemaCandidate = this.getZodSchemaCandidate(output_format); + let tools = undefined; + let toolChoice = undefined; + if (output_format && zodSchemaCandidate) { + try { + const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, { + name: 'Response', + target: 'jsonSchema7', + }); + const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, { + removeMinItems: this.removeMinItemsFromSchema, + removeDefaults: this.removeDefaultsFromSchema, + }); + delete optimizedJsonSchema.title; + const toolName = output_format?.name || 'response'; + tools = [ + { + name: toolName, + description: `Extract information in the format of ${toolName}`, + input_schema: optimizedJsonSchema, + cache_control: { type: 'ephemeral' }, + }, + ]; + toolChoice = { type: 'tool', name: toolName }; + } + catch (e) { + console.warn('Failed to convert output_format to JSON schema for Anthropic', e); + } + } + const requestPayload = { + model: this.model, + max_tokens: this.maxTokens, + messages: anthropicMessages, + ...this.getModelParams(), + }; + if (systemPrompt !== undefined) { + requestPayload.system = systemPrompt; + } + if (tools?.length) { + requestPayload.tools = tools; + requestPayload.tool_choice = toolChoice; + } + try { + const response = await this.client.messages.create(requestPayload, options.signal ? { signal: options.signal } : undefined); + let completion = this.getTextCompletion(response); + if (output_format) { + const toolUseBlock = response.content.find((block) => block.type === 'tool_use'); + if (toolUseBlock && toolUseBlock.type === 'tool_use') { + try { + completion = this.parseOutput(output_format, toolUseBlock.input); + } + catch (error) { + if (typeof toolUseBlock.input === 'string') { + completion = this.parseOutput(output_format, JSON.parse(toolUseBlock.input)); + } + else { + throw error; + } + } + } + else if (tools?.length) { + throw new ModelProviderError('Expected tool use in response but none found', 502, this.model); + } + else { + completion = this.parseOutput(output_format, completion); + } + } + else { + completion = this.getTextCompletion(response); + } + const usage = this.getUsage(response); + const stopReason = response.stop_reason ?? null; + return new ChatInvokeCompletion(completion, usage, null, null, stopReason); + } + catch (error) { + if (error instanceof RateLimitError || error?.status === 429) { + throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model); + } + if (error instanceof APIConnectionError) { + throw new ModelProviderError(error?.message ?? 'Connection error', 502, this.model); + } + if (error instanceof APIError) { + throw new ModelProviderError(error?.message ?? 'Anthropic API error', error?.status ?? 502, this.model); + } + throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 502, this.model); + } + } +} diff --git a/dist/llm/anthropic/index.d.ts b/dist/llm/anthropic/index.d.ts new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/anthropic/index.d.ts @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/anthropic/index.js b/dist/llm/anthropic/index.js new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/anthropic/index.js @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/anthropic/serializer.d.ts b/dist/llm/anthropic/serializer.d.ts new file mode 100644 index 00000000..d047728f --- /dev/null +++ b/dist/llm/anthropic/serializer.d.ts @@ -0,0 +1,70 @@ +/** + * Anthropic Message Serializer with Prompt Caching Support + * + * This serializer converts custom message types to Anthropic's MessageParam format + * and implements Anthropic's Prompt Caching feature to reduce costs by up to 90%. + * + * Caching Strategy: + * - Only the last message with cache=true will have cache_control enabled + * - Caching is most effective for system prompts and large conversation histories + * - Cache writes cost 25% more, but cache reads cost 90% less + * + * Example cost savings: + * - Without caching: 10,000 tokens @ $3/M = $0.030 per request + * - With caching (90% hit rate): + * - First request: 10,000 tokens @ $3.75/M (write) = $0.0375 + * - Subsequent: 1,000 tokens @ $3/M + 9,000 tokens @ $0.30/M = $0.0057 + * - Savings: 81% cost reduction + */ +import type { MessageParam, TextBlockParam } from '@anthropic-ai/sdk/resources/messages.mjs'; +import { type Message } from '../messages.js'; +export declare class AnthropicMessageSerializer { + /** + * Serialize a list of messages, extracting any system message + * + * @param messages - List of messages to serialize + * @returns Tuple of [messages, system_message] + */ + serializeMessages(messages: Message[]): [MessageParam[], (string | TextBlockParam[])?]; + /** + * Serialize a single message + */ + serializeMessage(message: Message): MessageParam; + /** + * Serialize cache control parameter + */ + private _serializeCacheControl; + /** + * Serialize text content part with optional caching + */ + private _serializeContentPartText; + /** + * Serialize image content part + */ + private _serializeContentPartImage; + /** + * Serialize content (string or array) with optional caching + */ + private _serializeContent; + /** + * Serialize content to string format (for system messages) + */ + private _serializeContentToStr; + /** + * Check if URL is a base64 encoded image + */ + private _isBase64Image; + /** + * Parse base64 data URL to extract media type and data + */ + private _parseBase64Url; + private _cloneMessage; + private _cloneContent; + /** + * Clean cache settings so only the last cache=true message remains cached + * + * Because of how Claude caching works, only the last cache message matters. + * This method automatically removes cache=True from all messages except the last one. + */ + private _cleanCacheMessages; +} diff --git a/dist/llm/anthropic/serializer.js b/dist/llm/anthropic/serializer.js new file mode 100644 index 00000000..0785c25f --- /dev/null +++ b/dist/llm/anthropic/serializer.js @@ -0,0 +1,357 @@ +/** + * Anthropic Message Serializer with Prompt Caching Support + * + * This serializer converts custom message types to Anthropic's MessageParam format + * and implements Anthropic's Prompt Caching feature to reduce costs by up to 90%. + * + * Caching Strategy: + * - Only the last message with cache=true will have cache_control enabled + * - Caching is most effective for system prompts and large conversation histories + * - Cache writes cost 25% more, but cache reads cost 90% less + * + * Example cost savings: + * - Without caching: 10,000 tokens @ $3/M = $0.030 per request + * - With caching (90% hit rate): + * - First request: 10,000 tokens @ $3.75/M (write) = $0.0375 + * - Subsequent: 1,000 tokens @ $3/M + 9,000 tokens @ $0.30/M = $0.0057 + * - Savings: 81% cost reduction + */ +import { AssistantMessage, ContentPartImageParam, ContentPartRefusalParam, ContentPartTextParam, FunctionCall, ImageURL, ToolCall, UserMessage, SystemMessage, } from '../messages.js'; +export class AnthropicMessageSerializer { + /** + * Serialize a list of messages, extracting any system message + * + * @param messages - List of messages to serialize + * @returns Tuple of [messages, system_message] + */ + serializeMessages(messages) { + // Make deep copies to avoid modifying originals + const messagesCopy = messages.map((message) => this._cloneMessage(message)); + // Separate system messages from normal messages + const normalMessages = []; + let systemMessage = null; + for (const message of messagesCopy) { + if (message instanceof SystemMessage) { + systemMessage = message; + } + else { + normalMessages.push(message); + } + } + // Clean cache messages so only the last cache=true message remains cached + const cleanedMessages = this._cleanCacheMessages(normalMessages); + // Serialize normal messages + const serializedMessages = cleanedMessages.map((msg) => this.serializeMessage(msg)); + // Serialize system message + let serializedSystemMessage = undefined; + if (systemMessage) { + serializedSystemMessage = this._serializeContentToStr(systemMessage.content, systemMessage.cache); + } + return [serializedMessages, serializedSystemMessage]; + } + /** + * Serialize a single message + */ + serializeMessage(message) { + if (message instanceof UserMessage) { + return { + role: 'user', + content: Array.isArray(message.content) + ? message.content.map((part, idx, arr) => { + const isLastPart = idx === arr.length - 1; + const useCache = message.cache && isLastPart; + if (part instanceof ContentPartTextParam) { + return this._serializeContentPartText(part, useCache); + } + if (part instanceof ContentPartImageParam) { + return this._serializeContentPartImage(part); + } + return { type: 'text', text: '' }; + }) + : this._serializeContent(message.content, message.cache), + }; + } + if (message instanceof AssistantMessage) { + const content = []; + // Add content blocks if present + if (message.content) { + if (typeof message.content === 'string') { + content.push({ + type: 'text', + text: message.content, + ...(message.cache && !message.tool_calls?.length + ? { cache_control: this._serializeCacheControl(true) } + : {}), + }); + } + else if (Array.isArray(message.content)) { + message.content.forEach((part, idx, arr) => { + const isLastPart = idx === arr.length - 1; + const useCache = message.cache && isLastPart && !message.tool_calls?.length; + if (part instanceof ContentPartTextParam) { + content.push(this._serializeContentPartText(part, useCache)); + } + }); + } + } + // Add tool use blocks if present + if (message.tool_calls) { + message.tool_calls.forEach((toolCall, idx, arr) => { + const isLastToolCall = idx === arr.length - 1; + const useCache = message.cache && isLastToolCall; + let toolInput; + try { + toolInput = JSON.parse(toolCall.functionCall.arguments); + } + catch { + toolInput = { arguments: toolCall.functionCall.arguments }; + } + content.push({ + type: 'tool_use', + id: toolCall.id, + name: toolCall.functionCall.name, + input: toolInput, + ...(useCache + ? { cache_control: this._serializeCacheControl(true) } + : {}), + }); + }); + } + // If no content or tool calls, add empty text block + if (!content.length) { + content.push({ + type: 'text', + text: '', + ...(message.cache + ? { cache_control: this._serializeCacheControl(true) } + : {}), + }); + } + const normalizedContent = (() => { + if (message.cache || content.length > 1) { + return content; + } + const first = content[0]; + if (first && first.type === 'text' && !first.cache_control) { + return first.text; + } + return content; + })(); + return { + role: 'assistant', + content: normalizedContent, + }; + } + throw new Error(`Unknown message type or unhandled role: ${message.role}`); + } + /** + * Serialize cache control parameter + */ + _serializeCacheControl(useCache) { + return useCache ? { type: 'ephemeral' } : undefined; + } + /** + * Serialize text content part with optional caching + */ + _serializeContentPartText(part, useCache) { + return { + type: 'text', + text: part.text, + ...(useCache ? { cache_control: this._serializeCacheControl(true) } : {}), + }; + } + /** + * Serialize image content part + */ + _serializeContentPartImage(part) { + const url = part.image_url.url; + if (this._isBase64Image(url)) { + // Handle base64 encoded images + const [mediaType, data] = this._parseBase64Url(url); + return { + type: 'image', + source: { + type: 'base64', + media_type: mediaType, + data: data, + }, + }; + } + else { + // Handle URL images + return { + type: 'image', + source: { + type: 'url', + url: url, + }, + }; + } + } + /** + * Serialize content (string or array) with optional caching + */ + _serializeContent(content, useCache) { + if (typeof content === 'string') { + if (useCache) { + return [ + { + type: 'text', + text: content, + cache_control: this._serializeCacheControl(true), + }, + ]; + } + return content; + } + return content.map((part, idx, arr) => { + const isLastPart = idx === arr.length - 1; + const partUseCache = useCache && isLastPart; + if (part instanceof ContentPartTextParam) { + return this._serializeContentPartText(part, partUseCache); + } + else if (part instanceof ContentPartImageParam) { + return this._serializeContentPartImage(part); + } + return { type: 'text', text: '' }; + }); + } + /** + * Serialize content to string format (for system messages) + */ + _serializeContentToStr(content, useCache) { + if (typeof content === 'string') { + if (useCache) { + return [ + { + type: 'text', + text: content, + cache_control: this._serializeCacheControl(true), + }, + ]; + } + return content; + } + return content.map((part, idx, arr) => { + const isLastPart = idx === arr.length - 1; + const partUseCache = useCache && isLastPart; + return this._serializeContentPartText(part, partUseCache); + }); + } + /** + * Check if URL is a base64 encoded image + */ + _isBase64Image(url) { + return url.startsWith('data:image/'); + } + /** + * Parse base64 data URL to extract media type and data + */ + _parseBase64Url(url) { + if (!url.startsWith('data:')) { + throw new Error(`Invalid base64 URL: ${url}`); + } + const [header, data] = url.split(',', 2); + if (!header || !data) { + throw new Error(`Invalid base64 URL format: ${url}`); + } + let mediaType = header.split(';')[0]?.replace('data:', '') || 'image/jpeg'; + // Ensure it's a supported media type + const supportedTypes = [ + 'image/jpeg', + 'image/png', + 'image/gif', + 'image/webp', + ]; + if (!supportedTypes.includes(mediaType)) { + // Default to jpeg if not recognized + mediaType = 'image/jpeg'; + } + return [mediaType, data]; + } + _cloneMessage(message) { + if (message instanceof UserMessage) { + const clone = new UserMessage(this._cloneContent(message.content), message.name); + clone.cache = message.cache; + return clone; + } + if (message instanceof SystemMessage) { + const clone = new SystemMessage(typeof message.content === 'string' + ? message.content + : message.content.map((part) => new ContentPartTextParam(part.text)), message.name); + clone.cache = message.cache; + return clone; + } + if (message instanceof AssistantMessage) { + const clone = new AssistantMessage({ + content: message.content === null + ? null + : typeof message.content === 'string' + ? message.content + : message.content.map((part) => { + if (part instanceof ContentPartTextParam) { + return new ContentPartTextParam(part.text); + } + if (part instanceof ContentPartRefusalParam) { + return new ContentPartRefusalParam(part.refusal); + } + if (part instanceof ContentPartImageParam) { + return new ContentPartImageParam(new ImageURL(part.image_url.url, part.image_url.detail, part.image_url.media_type)); + } + return part; + }), + tool_calls: message.tool_calls + ? message.tool_calls.map((toolCall) => new ToolCall(toolCall.id, new FunctionCall(toolCall.functionCall.name, toolCall.functionCall.arguments))) + : null, + refusal: message.refusal, + }); + clone.cache = message.cache; + return clone; + } + return message; + } + _cloneContent(content) { + if (typeof content === 'string') { + return content; + } + return content.map((part) => { + if (part instanceof ContentPartTextParam) { + return new ContentPartTextParam(part.text); + } + if (part instanceof ContentPartRefusalParam) { + return new ContentPartRefusalParam(part.refusal); + } + return new ContentPartImageParam(new ImageURL(part.image_url.url, part.image_url.detail, part.image_url.media_type)); + }); + } + /** + * Clean cache settings so only the last cache=true message remains cached + * + * Because of how Claude caching works, only the last cache message matters. + * This method automatically removes cache=True from all messages except the last one. + */ + _cleanCacheMessages(messages) { + if (!messages.length) { + return messages; + } + // Create deep copies to avoid modifying originals + const cleanedMessages = messages.map((msg) => this._cloneMessage(msg)); + // Find the last message with cache=true + let lastCacheIndex = -1; + for (let i = cleanedMessages.length - 1; i >= 0; i--) { + if (cleanedMessages[i].cache) { + lastCacheIndex = i; + break; + } + } + // If we found a cached message, disable cache for all others + if (lastCacheIndex !== -1) { + for (let i = 0; i < cleanedMessages.length; i++) { + if (i !== lastCacheIndex && cleanedMessages[i].cache) { + cleanedMessages[i].cache = false; + } + } + } + return cleanedMessages; + } +} diff --git a/dist/llm/aws/chat-anthropic.d.ts b/dist/llm/aws/chat-anthropic.d.ts new file mode 100644 index 00000000..60788335 --- /dev/null +++ b/dist/llm/aws/chat-anthropic.d.ts @@ -0,0 +1,78 @@ +/** + * AWS Bedrock Anthropic Claude chat model. + * + * This is a convenience class that provides Claude-specific defaults + * for the AWS Bedrock service. It inherits all functionality from + * ChatBedrockConverse but sets Anthropic Claude as the default model + * and uses the Anthropic message serializer for better compatibility. + * + * Usage: + * ```typescript + * import { ChatAnthropicBedrock } from './llm/aws/chat-anthropic.js'; + * + * const llm = new ChatAnthropicBedrock({ + * model: 'anthropic.claude-3-5-sonnet-20241022-v2:0', + * region: 'us-east-1' + * }); + * + * const response = await llm.ainvoke(messages); + * ``` + */ +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { type Message } from '../messages.js'; +export interface ChatAnthropicBedrockConfig { + /** Model ID, defaults to Claude 3.5 Sonnet */ + model?: string; + /** AWS region, defaults to us-east-1 */ + region?: string; + /** AWS access key ID */ + awsAccessKeyId?: string; + /** AWS secret access key */ + awsSecretAccessKey?: string; + /** AWS session token */ + awsSessionToken?: string; + /** Retry attempts */ + maxRetries?: number; + /** Maximum tokens to generate */ + max_tokens?: number; + /** Temperature for sampling (0-1) */ + temperature?: number | null; + /** Top-p sampling parameter */ + top_p?: number | null; + /** Top-k sampling parameter */ + top_k?: number | null; + /** Stop sequences */ + stop_sequences?: string[] | null; + /** Remove minItems from schema for provider compatibility */ + removeMinItemsFromSchema?: boolean; + /** Remove default from schema for provider compatibility */ + removeDefaultsFromSchema?: boolean; +} +export declare class ChatAnthropicBedrock implements BaseChatModel { + model: string; + provider: string; + private client; + private max_tokens; + private temperature; + private top_p; + private top_k; + private stop_sequences; + private removeMinItemsFromSchema; + private removeDefaultsFromSchema; + constructor(config?: ChatAnthropicBedrockConfig); + get name(): string; + get model_name(): string; + private _getInferenceParams; + private getZodSchemaCandidate; + private parseOutput; + private getTextCompletion; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + }, options?: ChatInvokeOptions): Promise>; + /** + * Simple Zod to JSON Schema conversion for structured output + */ + private _zodToJsonSchema; +} diff --git a/dist/llm/aws/chat-anthropic.js b/dist/llm/aws/chat-anthropic.js new file mode 100644 index 00000000..3ec60b1f --- /dev/null +++ b/dist/llm/aws/chat-anthropic.js @@ -0,0 +1,265 @@ +/** + * AWS Bedrock Anthropic Claude chat model. + * + * This is a convenience class that provides Claude-specific defaults + * for the AWS Bedrock service. It inherits all functionality from + * ChatBedrockConverse but sets Anthropic Claude as the default model + * and uses the Anthropic message serializer for better compatibility. + * + * Usage: + * ```typescript + * import { ChatAnthropicBedrock } from './llm/aws/chat-anthropic.js'; + * + * const llm = new ChatAnthropicBedrock({ + * model: 'anthropic.claude-3-5-sonnet-20241022-v2:0', + * region: 'us-east-1' + * }); + * + * const response = await llm.ainvoke(messages); + * ``` + */ +import { BedrockRuntimeClient, ConverseCommand, } from '@aws-sdk/client-bedrock-runtime'; +import { ModelProviderError, ModelRateLimitError } from '../exceptions.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { AnthropicMessageSerializer } from '../anthropic/serializer.js'; +import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js'; +export class ChatAnthropicBedrock { + model; + provider = 'anthropic_bedrock'; + client; + max_tokens; + temperature; + top_p; + top_k; + stop_sequences; + removeMinItemsFromSchema; + removeDefaultsFromSchema; + constructor(config = {}) { + // Anthropic Claude specific defaults + this.model = config.model || 'anthropic.claude-3-5-sonnet-20241022-v2:0'; + this.max_tokens = config.max_tokens || 8192; + this.temperature = + config.temperature === undefined ? null : config.temperature; + this.top_p = config.top_p === undefined ? null : config.top_p; + this.top_k = config.top_k === undefined ? null : config.top_k; + this.stop_sequences = + config.stop_sequences === undefined ? null : config.stop_sequences; + this.removeMinItemsFromSchema = config.removeMinItemsFromSchema ?? false; + this.removeDefaultsFromSchema = config.removeDefaultsFromSchema ?? false; + const region = config.region || process.env.AWS_REGION || 'us-east-1'; + const awsSessionToken = config.awsSessionToken || process.env.AWS_SESSION_TOKEN; + const credentials = config.awsAccessKeyId && config.awsSecretAccessKey + ? { + accessKeyId: config.awsAccessKeyId, + secretAccessKey: config.awsSecretAccessKey, + ...(awsSessionToken ? { sessionToken: awsSessionToken } : {}), + } + : undefined; + this.client = new BedrockRuntimeClient({ + region, + ...(credentials ? { credentials } : {}), + ...(config.maxRetries !== undefined + ? { maxAttempts: config.maxRetries } + : {}), + }); + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + _getInferenceParams() { + const params = { + maxTokens: this.max_tokens, + }; + if (this.temperature !== null) { + params.temperature = this.temperature; + } + if (this.top_p !== null) { + params.topP = this.top_p; + } + if (this.stop_sequences !== null && this.stop_sequences.length > 0) { + params.stopSequences = this.stop_sequences; + } + return params; + } + getZodSchemaCandidate(output_format) { + const output = output_format; + if (output && + typeof output === 'object' && + typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return output; + } + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return output.schema; + } + return null; + } + parseOutput(output_format, payload) { + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + return output.schema.parse(payload); + } + return output.parse(payload); + } + getTextCompletion(response) { + const contentBlocks = response?.output?.message?.content; + if (!Array.isArray(contentBlocks)) { + return ''; + } + return contentBlocks + .filter((block) => typeof block?.text === 'string') + .map((block) => block.text) + .join('\n'); + } + async ainvoke(messages, output_format, options = {}) { + // Use Anthropic-specific message serializer + const serializer = new AnthropicMessageSerializer(); + const [anthropicMessages, systemPrompt] = serializer.serializeMessages(messages); + // Convert Anthropic messages to Bedrock format + const bedrockMessages = anthropicMessages.map((msg) => { + const content = Array.isArray(msg.content) + ? msg.content.map((block) => { + if (block.type === 'text') { + return { text: block.text }; + } + else if (block.type === 'tool_use') { + return { + toolUse: { + toolUseId: block.id, + name: block.name, + input: block.input, + }, + }; + } + else if (block.type === 'image') { + if (block.source?.type === 'base64') { + return { + image: { + format: String(block.source.media_type || 'image/jpeg').split('/')[1] ?? 'jpeg', + source: { + bytes: Buffer.from(block.source.data ?? '', 'base64'), + }, + }, + }; + } + return { text: '[Image]' }; + } + return { text: String(block) }; + }) + : [{ text: msg.content }]; + return { + role: msg.role, + content, + }; + }); + // Handle system message + const system = systemPrompt + ? [ + { + text: typeof systemPrompt === 'string' + ? systemPrompt + : JSON.stringify(systemPrompt), + }, + ] + : undefined; + let toolConfig = undefined; + const zodSchemaCandidate = this.getZodSchemaCandidate(output_format); + if (output_format && zodSchemaCandidate) { + // Structured output using tools + try { + const rawSchema = this._zodToJsonSchema(zodSchemaCandidate); + const schema = SchemaOptimizer.createOptimizedJsonSchema(rawSchema, { + removeMinItems: this.removeMinItemsFromSchema, + removeDefaults: this.removeDefaultsFromSchema, + }); + delete schema.title; + const tools = [ + { + toolSpec: { + name: 'extract_structured_data', + description: 'Extract structured data from the response', + inputSchema: { + json: schema, + }, + }, + }, + ]; + toolConfig = { + tools, + toolChoice: { tool: { name: 'extract_structured_data' } }, + }; + } + catch (e) { + console.warn('Failed to convert output_format to JSON schema', e); + } + } + const command = new ConverseCommand({ + modelId: this.model, + messages: bedrockMessages, + system: system, + toolConfig: toolConfig, + inferenceConfig: this._getInferenceParams(), + }); + try { + const response = await this.client.send(command, options.signal ? { abortSignal: options.signal } : undefined); + let completion = this.getTextCompletion(response); + const contentBlocks = response?.output?.message?.content; + const toolUseBlock = Array.isArray(contentBlocks) + ? contentBlocks.find((block) => block?.toolUse) + : undefined; + if (toolUseBlock?.toolUse && output_format) { + const input = toolUseBlock.toolUse.input; + if (typeof input === 'string') { + completion = this.parseOutput(output_format, JSON.parse(input)); + } + else { + completion = this.parseOutput(output_format, input); + } + } + else if (output_format && toolConfig) { + throw new ModelProviderError('Expected tool use in response but none found', 502, this.model); + } + else if (output_format) { + completion = this.parseOutput(output_format, completion); + } + else { + completion = this.getTextCompletion(response); + } + const stopReason = response?.stopReason ?? null; + return new ChatInvokeCompletion(completion, { + prompt_tokens: response.usage?.inputTokens ?? 0, + completion_tokens: response.usage?.outputTokens ?? 0, + total_tokens: response.usage?.totalTokens ?? 0, + }, null, null, stopReason); + } + catch (error) { + const errorName = String(error?.name ?? ''); + const statusCode = error?.$metadata?.httpStatusCode ?? 502; + if (statusCode === 429 || + errorName.includes('Throttling') || + errorName.includes('TooManyRequests')) { + throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model); + } + throw new ModelProviderError(error?.message ?? String(error), statusCode, this.model); + } + } + /** + * Simple Zod to JSON Schema conversion for structured output + */ + _zodToJsonSchema(schema) { + return zodSchemaToJsonSchema(schema, { + name: 'Response', + target: 'jsonSchema7', + }); + } +} diff --git a/dist/llm/aws/chat-bedrock.d.ts b/dist/llm/aws/chat-bedrock.d.ts new file mode 100644 index 00000000..1301c6d5 --- /dev/null +++ b/dist/llm/aws/chat-bedrock.d.ts @@ -0,0 +1,42 @@ +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { type Message } from '../messages.js'; +export interface ChatBedrockConverseOptions { + model?: string; + region?: string; + awsAccessKeyId?: string; + awsSecretAccessKey?: string; + awsSessionToken?: string; + maxTokens?: number | null; + temperature?: number | null; + topP?: number | null; + seed?: number | null; + stopSequences?: string[] | null; + maxRetries?: number; + removeMinItemsFromSchema?: boolean; + removeDefaultsFromSchema?: boolean; +} +export declare class ChatBedrockConverse implements BaseChatModel { + model: string; + provider: string; + private client; + private maxTokens; + private temperature; + private topP; + private seed; + private stopSequences; + private removeMinItemsFromSchema; + private removeDefaultsFromSchema; + constructor(modelOrOptions?: string | ChatBedrockConverseOptions, region?: string); + get name(): string; + get model_name(): string; + private getInferenceConfig; + private getUsage; + private getZodSchemaCandidate; + private parseOutput; + private getTextCompletion; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/aws/chat-bedrock.js b/dist/llm/aws/chat-bedrock.js new file mode 100644 index 00000000..19169aa9 --- /dev/null +++ b/dist/llm/aws/chat-bedrock.js @@ -0,0 +1,207 @@ +import { BedrockRuntimeClient, ConverseCommand, } from '@aws-sdk/client-bedrock-runtime'; +import { ModelProviderError, ModelRateLimitError } from '../exceptions.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js'; +import { AWSBedrockMessageSerializer } from './serializer.js'; +export class ChatBedrockConverse { + model; + provider = 'aws'; + client; + maxTokens; + temperature; + topP; + seed; + stopSequences; + removeMinItemsFromSchema; + removeDefaultsFromSchema; + constructor(modelOrOptions = {}, region) { + const normalizedOptions = typeof modelOrOptions === 'string' + ? { model: modelOrOptions, region } + : modelOrOptions; + const { model = 'anthropic.claude-3-5-sonnet-20240620-v1:0', region: bedrockRegion = process.env.AWS_REGION || 'us-east-1', awsAccessKeyId, awsSecretAccessKey, awsSessionToken = process.env.AWS_SESSION_TOKEN, maxTokens = 4096, temperature = null, topP = null, seed = null, stopSequences = null, maxRetries, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions; + this.model = model; + this.maxTokens = maxTokens; + this.temperature = temperature; + this.topP = topP; + this.seed = seed; + this.stopSequences = stopSequences; + this.removeMinItemsFromSchema = removeMinItemsFromSchema; + this.removeDefaultsFromSchema = removeDefaultsFromSchema; + const credentials = awsAccessKeyId && awsSecretAccessKey + ? { + accessKeyId: awsAccessKeyId, + secretAccessKey: awsSecretAccessKey, + ...(awsSessionToken ? { sessionToken: awsSessionToken } : {}), + } + : undefined; + this.client = new BedrockRuntimeClient({ + region: bedrockRegion, + ...(credentials ? { credentials } : {}), + ...(maxRetries !== undefined ? { maxAttempts: maxRetries } : {}), + }); + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + getInferenceConfig() { + const config = {}; + if (this.maxTokens !== null) { + config.maxTokens = this.maxTokens; + } + if (this.temperature !== null) { + config.temperature = this.temperature; + } + if (this.topP !== null) { + config.topP = this.topP; + } + if (this.seed !== null) { + config.seed = this.seed; + } + if (this.stopSequences !== null) { + config.stopSequences = this.stopSequences; + } + return config; + } + getUsage(response) { + const usage = response?.usage ?? {}; + return { + prompt_tokens: usage.inputTokens ?? 0, + completion_tokens: usage.outputTokens ?? 0, + total_tokens: usage.totalTokens ?? 0, + prompt_cached_tokens: null, + prompt_cache_creation_tokens: null, + prompt_image_tokens: null, + }; + } + getZodSchemaCandidate(output_format) { + const output = output_format; + if (output && + typeof output === 'object' && + typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return output; + } + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return output.schema; + } + return null; + } + parseOutput(output_format, payload) { + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + return output.schema.parse(payload); + } + return output.parse(payload); + } + getTextCompletion(response) { + const contentBlocks = response?.output?.message?.content; + if (!Array.isArray(contentBlocks)) { + return ''; + } + return contentBlocks + .filter((block) => typeof block?.text === 'string') + .map((block) => block.text) + .join('\n'); + } + async ainvoke(messages, output_format, options = {}) { + const serializer = new AWSBedrockMessageSerializer(); + const [bedrockMessages, systemMessage] = serializer.serializeMessages(messages); + const zodSchemaCandidate = this.getZodSchemaCandidate(output_format); + let toolConfig = undefined; + if (output_format && zodSchemaCandidate) { + try { + const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, { + name: 'Response', + target: 'jsonSchema7', + }); + const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, { + removeMinItems: this.removeMinItemsFromSchema, + removeDefaults: this.removeDefaultsFromSchema, + }); + delete optimizedJsonSchema.title; + const tools = [ + { + toolSpec: { + name: 'response', + description: 'Extract information in the format of response', + inputSchema: { + json: optimizedJsonSchema, + }, + }, + }, + ]; + toolConfig = { + tools, + toolChoice: { tool: { name: 'response' } }, + }; + } + catch (e) { + console.warn('Failed to convert output_format to JSON schema for AWS Bedrock', e); + } + } + const requestPayload = { + modelId: this.model, + messages: bedrockMessages, + }; + if (systemMessage) { + requestPayload.system = systemMessage; + } + const inferenceConfig = this.getInferenceConfig(); + if (Object.keys(inferenceConfig).length) { + requestPayload.inferenceConfig = inferenceConfig; + } + if (toolConfig) { + requestPayload.toolConfig = toolConfig; + } + try { + const response = await this.client.send(new ConverseCommand(requestPayload), options.signal ? { abortSignal: options.signal } : undefined); + let completion = this.getTextCompletion(response); + if (output_format) { + const contentBlocks = response?.output?.message?.content; + const toolUseBlock = Array.isArray(contentBlocks) + ? contentBlocks.find((block) => block?.toolUse) + : undefined; + if (toolUseBlock?.toolUse) { + const input = toolUseBlock.toolUse.input; + if (typeof input === 'string') { + completion = this.parseOutput(output_format, JSON.parse(input)); + } + else { + completion = this.parseOutput(output_format, input); + } + } + else if (toolConfig) { + throw new ModelProviderError('Expected tool use in response but none found', 502, this.model); + } + else { + completion = this.parseOutput(output_format, completion); + } + } + else { + completion = this.getTextCompletion(response); + } + const stopReason = response?.stopReason ?? null; + return new ChatInvokeCompletion(completion, this.getUsage(response), null, null, stopReason); + } + catch (error) { + const errorName = String(error?.name ?? ''); + const statusCode = error?.$metadata?.httpStatusCode ?? 502; + if (statusCode === 429 || + errorName.includes('Throttling') || + errorName.includes('TooManyRequests')) { + throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model); + } + throw new ModelProviderError(error?.message ?? String(error), statusCode, this.model); + } + } +} diff --git a/dist/llm/aws/index.d.ts b/dist/llm/aws/index.d.ts new file mode 100644 index 00000000..da1ec973 --- /dev/null +++ b/dist/llm/aws/index.d.ts @@ -0,0 +1,3 @@ +export * from './chat-bedrock.js'; +export * from './chat-anthropic.js'; +export * from './serializer.js'; diff --git a/dist/llm/aws/index.js b/dist/llm/aws/index.js new file mode 100644 index 00000000..da1ec973 --- /dev/null +++ b/dist/llm/aws/index.js @@ -0,0 +1,3 @@ +export * from './chat-bedrock.js'; +export * from './chat-anthropic.js'; +export * from './serializer.js'; diff --git a/dist/llm/aws/serializer.d.ts b/dist/llm/aws/serializer.d.ts new file mode 100644 index 00000000..5cde167d --- /dev/null +++ b/dist/llm/aws/serializer.d.ts @@ -0,0 +1,17 @@ +import { type Message } from '../messages.js'; +type BedrockContentBlock = Record; +type BedrockMessage = { + role: 'user' | 'assistant'; + content: BedrockContentBlock[]; +}; +type BedrockSystemMessage = { + text: string; +}[]; +export declare class AWSBedrockMessageSerializer { + serialize(messages: Message[]): BedrockMessage[]; + serializeMessages(messages: Message[]): [BedrockMessage[], BedrockSystemMessage?]; + private serializeSystemContent; + private serializeImageContent; + private serializeMessage; +} +export {}; diff --git a/dist/llm/aws/serializer.js b/dist/llm/aws/serializer.js new file mode 100644 index 00000000..3dff24ec --- /dev/null +++ b/dist/llm/aws/serializer.js @@ -0,0 +1,107 @@ +import { AssistantMessage, ContentPartImageParam, ContentPartRefusalParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js'; +export class AWSBedrockMessageSerializer { + serialize(messages) { + return this.serializeMessages(messages)[0]; + } + serializeMessages(messages) { + const bedrockMessages = []; + let systemMessage = undefined; + for (const message of messages) { + if (message instanceof SystemMessage) { + systemMessage = this.serializeSystemContent(message.content); + } + else { + bedrockMessages.push(this.serializeMessage(message)); + } + } + return [bedrockMessages, systemMessage]; + } + serializeSystemContent(content) { + if (typeof content === 'string') { + return [{ text: content }]; + } + return content + .filter((part) => part instanceof ContentPartTextParam) + .map((part) => ({ text: part.text })); + } + serializeImageContent(part) { + const url = part.image_url.url; + if (!url.startsWith('data:')) { + throw new Error(`Unsupported image URL format: ${url}`); + } + const [, payload = ''] = url.split(',', 2); + const format = part.image_url.media_type.split('/')[1] ?? 'jpeg'; + return { + image: { + format, + source: { + bytes: Buffer.from(payload, 'base64'), + }, + }, + }; + } + serializeMessage(message) { + if (message instanceof UserMessage) { + return { + role: 'user', + content: Array.isArray(message.content) + ? message.content.map((part) => { + if (part instanceof ContentPartTextParam) { + return { text: part.text }; + } + if (part instanceof ContentPartImageParam) { + return this.serializeImageContent(part); + } + return { text: '' }; + }) + : [{ text: message.content }], + }; + } + if (message instanceof AssistantMessage) { + const content = []; + if (message.content) { + if (typeof message.content === 'string') { + content.push({ text: message.content }); + } + else if (Array.isArray(message.content)) { + message.content.forEach((part) => { + if (part instanceof ContentPartTextParam) { + content.push({ text: part.text }); + } + else if (part instanceof ContentPartRefusalParam) { + content.push({ text: `[Refusal] ${part.refusal}` }); + } + }); + } + } + if (message.tool_calls) { + message.tool_calls.forEach((toolCall) => { + let parsedArguments; + try { + parsedArguments = JSON.parse(toolCall.functionCall.arguments); + } + catch { + parsedArguments = { + arguments: toolCall.functionCall.arguments, + }; + } + content.push({ + toolUse: { + toolUseId: toolCall.id, + name: toolCall.functionCall.name, + input: parsedArguments, + }, + }); + }); + } + if (!content.length) { + content.push({ text: '' }); + } + return { + role: 'assistant', + content: content, + }; + } + throw new Error(`Unknown message type: ${message.constructor.name}`); + } +} diff --git a/dist/llm/azure/chat.d.ts b/dist/llm/azure/chat.d.ts new file mode 100644 index 00000000..efdd2901 --- /dev/null +++ b/dist/llm/azure/chat.d.ts @@ -0,0 +1,66 @@ +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import type { Message } from '../messages.js'; +import { ChatInvokeCompletion } from '../views.js'; +export interface ChatAzureOptions { + model?: string; + apiKey?: string; + endpoint?: string; + baseURL?: string; + apiVersion?: string; + deployment?: string; + azureAdToken?: string | null; + azureAdTokenProvider?: (() => Promise) | null; + timeout?: number | null; + temperature?: number | null; + frequencyPenalty?: number | null; + reasoningEffort?: 'low' | 'medium' | 'high'; + serviceTier?: 'auto' | 'default' | 'flex' | 'priority' | 'scale' | null; + maxCompletionTokens?: number | null; + topP?: number | null; + seed?: number | null; + maxRetries?: number; + defaultHeaders?: Record | null; + defaultQuery?: Record | null; + fetchImplementation?: typeof fetch; + fetchOptions?: RequestInit | null; + useResponsesApi?: boolean | 'auto'; + addSchemaToSystemPrompt?: boolean; + dontForceStructuredOutput?: boolean; + removeMinItemsFromSchema?: boolean; + removeDefaultsFromSchema?: boolean; +} +export declare class ChatAzure implements BaseChatModel { + model: string; + provider: string; + private client; + private temperature; + private frequencyPenalty; + private reasoningEffort; + private serviceTier; + private maxCompletionTokens; + private topP; + private seed; + private useResponsesApi; + private addSchemaToSystemPrompt; + private dontForceStructuredOutput; + private removeMinItemsFromSchema; + private removeDefaultsFromSchema; + constructor(options?: string | ChatAzureOptions); + get name(): string; + get model_name(): string; + private isReasoningModel; + private shouldUseResponsesApi; + private getChatUsage; + private getResponsesUsage; + private getResponseOutputText; + private getModelParamsForCompletions; + private getModelParamsForResponses; + private getZodSchemaCandidate; + private applySchemaToSystemMessage; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; + private invokeChatCompletions; + private invokeResponses; +} diff --git a/dist/llm/azure/chat.js b/dist/llm/azure/chat.js new file mode 100644 index 00000000..dcce8eb6 --- /dev/null +++ b/dist/llm/azure/chat.js @@ -0,0 +1,396 @@ +import { AzureOpenAI } from 'openai'; +import { ModelProviderError, ModelRateLimitError } from '../exceptions.js'; +import { OpenAIMessageSerializer } from '../openai/serializer.js'; +import { ResponsesAPIMessageSerializer } from '../openai/responses-serializer.js'; +import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js'; +import { ChatInvokeCompletion } from '../views.js'; +const RESPONSES_API_ONLY_MODELS = [ + 'gpt-5.1-codex', + 'gpt-5.1-codex-mini', + 'gpt-5.1-codex-max', + 'gpt-5-codex', + 'codex-mini-latest', + 'computer-use-preview', +]; +const REASONING_MODELS = [ + 'o4-mini', + 'o3', + 'o3-mini', + 'o1', + 'o1-pro', + 'o3-pro', + 'gpt-5', + 'gpt-5-mini', + 'gpt-5-nano', +]; +export class ChatAzure { + model; + provider = 'azure'; + client; + temperature; + frequencyPenalty; + reasoningEffort; + serviceTier; + maxCompletionTokens; + topP; + seed; + useResponsesApi; + addSchemaToSystemPrompt; + dontForceStructuredOutput; + removeMinItemsFromSchema; + removeDefaultsFromSchema; + constructor(options = {}) { + const normalizedOptions = typeof options === 'string' ? { model: options } : options; + const { model = 'gpt-4o', apiKey = process.env.AZURE_OPENAI_API_KEY ?? process.env.AZURE_OPENAI_KEY, endpoint = process.env.AZURE_OPENAI_ENDPOINT, baseURL = undefined, apiVersion = process.env.AZURE_OPENAI_API_VERSION ?? '2024-12-01-preview', deployment = process.env.AZURE_OPENAI_DEPLOYMENT ?? model, azureAdToken = null, azureAdTokenProvider = null, timeout = null, temperature = 0.2, frequencyPenalty = 0.3, reasoningEffort = 'low', serviceTier = null, maxCompletionTokens = 4096, topP = null, seed = null, maxRetries = 5, defaultHeaders = null, defaultQuery = null, fetchImplementation, fetchOptions = null, useResponsesApi = 'auto', addSchemaToSystemPrompt = false, dontForceStructuredOutput = false, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions; + this.model = model; + this.temperature = temperature; + this.frequencyPenalty = frequencyPenalty; + this.reasoningEffort = reasoningEffort; + this.serviceTier = serviceTier; + this.maxCompletionTokens = maxCompletionTokens; + this.topP = topP; + this.seed = seed; + this.useResponsesApi = useResponsesApi; + this.addSchemaToSystemPrompt = addSchemaToSystemPrompt; + this.dontForceStructuredOutput = dontForceStructuredOutput; + this.removeMinItemsFromSchema = removeMinItemsFromSchema; + this.removeDefaultsFromSchema = removeDefaultsFromSchema; + this.client = new AzureOpenAI({ + apiKey, + endpoint, + baseURL, + apiVersion, + deployment, + azureADTokenProvider: azureAdTokenProvider ?? + (azureAdToken ? async () => String(azureAdToken) : undefined), + timeout: timeout ?? undefined, + maxRetries, + defaultHeaders: defaultHeaders ?? undefined, + defaultQuery: defaultQuery ?? undefined, + fetch: fetchImplementation, + fetchOptions: (fetchOptions ?? undefined), + }); + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + isReasoningModel() { + return REASONING_MODELS.some((m) => this.model.toLowerCase().includes(m.toLowerCase())); + } + shouldUseResponsesApi() { + if (typeof this.useResponsesApi === 'boolean') { + return this.useResponsesApi; + } + return RESPONSES_API_ONLY_MODELS.some((name) => this.model.toLowerCase().includes(name.toLowerCase())); + } + getChatUsage(response) { + if (!response?.usage) { + return null; + } + let completionTokens = response.usage.completion_tokens; + const completionDetails = response.usage.completion_tokens_details; + if (completionDetails?.reasoning_tokens) { + completionTokens += completionDetails.reasoning_tokens; + } + return { + prompt_tokens: response.usage.prompt_tokens, + prompt_cached_tokens: response.usage.prompt_tokens_details?.cached_tokens ?? null, + prompt_cache_creation_tokens: null, + prompt_image_tokens: null, + completion_tokens: completionTokens, + total_tokens: response.usage.total_tokens, + }; + } + getResponsesUsage(response) { + if (!response?.usage) { + return null; + } + return { + prompt_tokens: response.usage.input_tokens ?? 0, + prompt_cached_tokens: response.usage.input_tokens_details?.cached_tokens ?? null, + prompt_cache_creation_tokens: null, + prompt_image_tokens: null, + completion_tokens: response.usage.output_tokens ?? 0, + total_tokens: response.usage.total_tokens ?? 0, + }; + } + getResponseOutputText(response) { + if (typeof response?.output_text === 'string') { + return response.output_text; + } + const outputs = Array.isArray(response?.output) ? response.output : []; + for (const item of outputs) { + if (Array.isArray(item?.content)) { + for (const part of item.content) { + if (typeof part?.text === 'string') { + return part.text; + } + if (typeof part?.output_text === 'string') { + return part.output_text; + } + } + } + } + return ''; + } + getModelParamsForCompletions() { + const modelParams = {}; + if (!this.isReasoningModel()) { + if (this.temperature !== null) { + modelParams.temperature = this.temperature; + } + if (this.frequencyPenalty !== null) { + modelParams.frequency_penalty = this.frequencyPenalty; + } + } + else { + modelParams.reasoning_effort = this.reasoningEffort; + } + if (this.maxCompletionTokens !== null) { + modelParams.max_completion_tokens = this.maxCompletionTokens; + } + if (this.topP !== null) { + modelParams.top_p = this.topP; + } + if (this.seed !== null) { + modelParams.seed = this.seed; + } + if (this.serviceTier !== null) { + modelParams.service_tier = this.serviceTier; + } + return modelParams; + } + getModelParamsForResponses() { + const modelParams = {}; + if (!this.isReasoningModel()) { + if (this.temperature !== null) { + modelParams.temperature = this.temperature; + } + if (this.frequencyPenalty !== null) { + modelParams.frequency_penalty = this.frequencyPenalty; + } + } + else { + modelParams.reasoning = { effort: this.reasoningEffort }; + } + if (this.maxCompletionTokens !== null) { + modelParams.max_output_tokens = this.maxCompletionTokens; + } + if (this.topP !== null) { + modelParams.top_p = this.topP; + } + if (this.seed !== null) { + modelParams.seed = this.seed; + } + if (this.serviceTier !== null) { + modelParams.service_tier = this.serviceTier; + } + return modelParams; + } + getZodSchemaCandidate(output_format) { + const output = output_format; + if (output && + typeof output === 'object' && + typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return output; + } + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return output.schema; + } + return null; + } + applySchemaToSystemMessage(openaiMessages, responseJsonSchema) { + if (!this.addSchemaToSystemPrompt || openaiMessages.length === 0) { + return; + } + const firstMessage = openaiMessages[0]; + if (firstMessage?.role !== 'system') { + return; + } + const schemaText = `\n\n` + + `${JSON.stringify(responseJsonSchema, null, 2)}\n` + + ``; + if (typeof firstMessage.content === 'string') { + firstMessage.content = (firstMessage.content ?? '') + schemaText; + return; + } + if (Array.isArray(firstMessage.content)) { + firstMessage.content = [ + ...firstMessage.content, + { type: 'text', text: schemaText }, + ]; + } + } + async ainvoke(messages, output_format, options = {}) { + const zodSchemaCandidate = this.getZodSchemaCandidate(output_format); + if (this.shouldUseResponsesApi()) { + return this.invokeResponses(messages, output_format, zodSchemaCandidate, options); + } + return this.invokeChatCompletions(messages, output_format, zodSchemaCandidate, options); + } + async invokeChatCompletions(messages, output_format, zodSchemaCandidate, options) { + const serializer = new OpenAIMessageSerializer(); + const openaiMessages = serializer.serialize(messages); + let responseFormat = undefined; + if (zodSchemaCandidate) { + try { + const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, { + name: 'agent_output', + target: 'jsonSchema7', + }); + const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, { + removeMinItems: this.removeMinItemsFromSchema, + removeDefaults: this.removeDefaultsFromSchema, + }); + const responseJsonSchema = { + name: 'agent_output', + schema: optimizedJsonSchema, + strict: true, + }; + this.applySchemaToSystemMessage(openaiMessages, responseJsonSchema); + if (!this.dontForceStructuredOutput) { + responseFormat = { + type: 'json_schema', + json_schema: responseJsonSchema, + }; + } + } + catch { + responseFormat = undefined; + } + } + try { + const response = await this.client.chat.completions.create({ + model: this.model, + messages: openaiMessages, + response_format: responseFormat, + ...this.getModelParamsForCompletions(), + }, options.signal ? { signal: options.signal } : undefined); + const content = response.choices[0].message.content || ''; + const usage = this.getChatUsage(response); + const stopReason = response.choices[0].finish_reason ?? null; + let completion = content; + if (output_format) { + if (zodSchemaCandidate) { + const parsedJson = JSON.parse(content); + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + completion = output.schema.parse(parsedJson); + } + else { + completion = output.parse(parsedJson); + } + } + else { + completion = output_format.parse(content); + } + } + return new ChatInvokeCompletion(completion, usage, null, null, stopReason); + } + catch (error) { + if (error?.status === 429) { + throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model); + } + throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model); + } + } + async invokeResponses(messages, output_format, zodSchemaCandidate, options) { + const serializer = new ResponsesAPIMessageSerializer(); + const inputMessages = serializer.serialize(messages); + const request = { + model: this.model, + input: inputMessages, + ...this.getModelParamsForResponses(), + }; + if (zodSchemaCandidate) { + try { + const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, { + name: 'agent_output', + target: 'jsonSchema7', + }); + const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, { + removeMinItems: this.removeMinItemsFromSchema, + removeDefaults: this.removeDefaultsFromSchema, + }); + if (this.addSchemaToSystemPrompt && + inputMessages.length > 0 && + inputMessages[0]?.role === 'system') { + const schemaText = `\n\n${JSON.stringify(optimizedJsonSchema)}\n`; + const firstInput = inputMessages[0]; + const firstContent = firstInput?.content; + let patchedContent = firstContent ?? ''; + if (typeof firstContent === 'string') { + patchedContent = firstContent + schemaText; + } + else if (Array.isArray(firstContent)) { + patchedContent = [ + ...firstContent, + { type: 'input_text', text: schemaText }, + ]; + } + inputMessages[0] = { + ...inputMessages[0], + content: patchedContent, + }; + request.input = inputMessages; + } + if (!this.dontForceStructuredOutput) { + request.text = { + format: { + type: 'json_schema', + name: 'agent_output', + strict: true, + schema: optimizedJsonSchema, + }, + }; + } + } + catch { + // Skip structured output forcing when schema conversion fails. + } + } + try { + const response = await this.client.responses.create(request, options.signal ? { signal: options.signal } : undefined); + const content = this.getResponseOutputText(response); + const usage = this.getResponsesUsage(response); + const stopReason = response?.status ?? null; + let completion = content; + if (output_format) { + if (zodSchemaCandidate) { + const parsedJson = JSON.parse(content); + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + completion = output.schema.parse(parsedJson); + } + else { + completion = output.parse(parsedJson); + } + } + else { + completion = output_format.parse(content); + } + } + return new ChatInvokeCompletion(completion, usage, null, null, stopReason); + } + catch (error) { + if (error?.status === 429) { + throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model); + } + throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model); + } + } +} diff --git a/dist/llm/azure/index.d.ts b/dist/llm/azure/index.d.ts new file mode 100644 index 00000000..69a16816 --- /dev/null +++ b/dist/llm/azure/index.d.ts @@ -0,0 +1 @@ +export * from './chat.js'; diff --git a/dist/llm/azure/index.js b/dist/llm/azure/index.js new file mode 100644 index 00000000..69a16816 --- /dev/null +++ b/dist/llm/azure/index.js @@ -0,0 +1 @@ +export * from './chat.js'; diff --git a/dist/llm/base.d.ts b/dist/llm/base.d.ts new file mode 100644 index 00000000..c58bcc28 --- /dev/null +++ b/dist/llm/base.d.ts @@ -0,0 +1,18 @@ +import type { ChatInvokeCompletion } from './views.js'; +import type { Message } from './messages.js'; +export interface ChatInvokeOptions { + signal?: AbortSignal; + request_type?: string; + [key: string]: unknown; +} +export interface BaseChatModel { + model: string; + _verified_api_keys?: boolean; + get provider(): string; + get name(): string; + get model_name(): string; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/base.js b/dist/llm/base.js new file mode 100644 index 00000000..cb0ff5c3 --- /dev/null +++ b/dist/llm/base.js @@ -0,0 +1 @@ +export {}; diff --git a/dist/llm/browser-use/chat.d.ts b/dist/llm/browser-use/chat.d.ts new file mode 100644 index 00000000..c60e1e74 --- /dev/null +++ b/dist/llm/browser-use/chat.d.ts @@ -0,0 +1,40 @@ +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import type { Message } from '../messages.js'; +import { ChatInvokeCompletion } from '../views.js'; +export interface ChatBrowserUseOptions { + model?: string; + apiKey?: string; + baseUrl?: string; + timeout?: number; + maxRetries?: number; + retryBaseDelay?: number; + retryMaxDelay?: number; + fast?: boolean; + fetchImplementation?: typeof fetch; +} +export declare class ChatBrowserUse implements BaseChatModel { + model: string; + provider: string; + private readonly apiKey; + private readonly baseUrl; + private readonly timeoutMs; + private readonly maxRetries; + private readonly retryBaseDelay; + private readonly retryMaxDelay; + private readonly fast; + private readonly fetchImplementation; + constructor(options?: ChatBrowserUseOptions); + get name(): string; + get model_name(): string; + private getOutputSchema; + private parseOutput; + private serializeMessage; + private getUsage; + private raiseHttpError; + private isRetryableNetworkError; + private makeRequest; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/browser-use/chat.js b/dist/llm/browser-use/chat.js new file mode 100644 index 00000000..f007d4d3 --- /dev/null +++ b/dist/llm/browser-use/chat.js @@ -0,0 +1,305 @@ +import { setTimeout as sleep } from 'node:timers/promises'; +import { CONFIG } from '../../config.js'; +import { ModelProviderError, ModelRateLimitError } from '../exceptions.js'; +import { zodSchemaToJsonSchema } from '../schema.js'; +import { ChatInvokeCompletion } from '../views.js'; +const RETRYABLE_STATUS_CODES = new Set([429, 500, 502, 503, 504]); +const VALID_MODELS = new Set(['bu-latest', 'bu-1-0', 'bu-2-0']); +class HttpStatusError extends Error { + statusCode; + detail; + constructor(statusCode, detail) { + super(`HTTP ${statusCode}: ${detail}`); + this.statusCode = statusCode; + this.detail = detail; + this.name = 'HttpStatusError'; + } +} +const isAbortError = (error) => error instanceof Error && error.name === 'AbortError'; +const getJsonErrorDetail = (value) => { + if (!value || typeof value !== 'object') { + return ''; + } + const detail = value.detail ?? value.error ?? value.message; + if (typeof detail === 'string') { + return detail; + } + try { + return JSON.stringify(value); + } + catch { + return ''; + } +}; +export class ChatBrowserUse { + model; + provider = 'browser-use'; + apiKey; + baseUrl; + timeoutMs; + maxRetries; + retryBaseDelay; + retryMaxDelay; + fast; + fetchImplementation; + constructor(options = {}) { + const { model = 'bu-latest', apiKey = process.env.BROWSER_USE_API_KEY, baseUrl = process.env.BROWSER_USE_LLM_URL ?? + 'https://llm.api.browser-use.com', timeout = 120, maxRetries = 5, retryBaseDelay = 1.0, retryMaxDelay = 60.0, fast = false, fetchImplementation = fetch, } = options; + const isValidModel = VALID_MODELS.has(model) || model.startsWith('browser-use/'); + if (!isValidModel) { + throw new Error(`Invalid model: '${model}'. Must be one of bu-latest, bu-1-0, bu-2-0 or start with 'browser-use/'`); + } + this.model = model === 'bu-latest' ? 'bu-1-0' : model; + if (!apiKey) { + throw new Error('You need to set the BROWSER_USE_API_KEY environment variable. Get your key at https://cloud.browser-use.com/new-api-key'); + } + this.apiKey = apiKey; + this.baseUrl = baseUrl.replace(/\/+$/, ''); + this.timeoutMs = Math.max(1, Math.round(timeout * 1000)); + this.maxRetries = Math.max(1, Math.trunc(maxRetries)); + this.retryBaseDelay = Math.max(0.001, retryBaseDelay); + this.retryMaxDelay = Math.max(this.retryBaseDelay, retryMaxDelay); + this.fast = fast; + this.fetchImplementation = fetchImplementation; + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + getOutputSchema(output_format) { + const output = output_format; + if (!output || typeof output !== 'object') { + return null; + } + if (typeof output.model_json_schema === 'function') { + const schema = output.model_json_schema(); + if (schema && typeof schema === 'object') { + return schema; + } + } + if (typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return zodSchemaToJsonSchema(output, { + name: 'Response', + target: 'jsonSchema7', + }); + } + if (output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return zodSchemaToJsonSchema(output.schema, { + name: 'Response', + target: 'jsonSchema7', + }); + } + if (output.schema && typeof output.schema === 'object') { + const schemaCandidate = output.schema; + const schema = typeof schemaCandidate.toJSON === 'function' + ? schemaCandidate.toJSON() + : schemaCandidate; + if (schema && typeof schema === 'object') { + return schema; + } + } + return null; + } + parseOutput(output_format, payload) { + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + return output.schema.parse(payload); + } + return output.parse(payload); + } + serializeMessage(message) { + return { + role: message.role, + content: message.content, + }; + } + getUsage(payload) { + if (!payload || typeof payload !== 'object') { + return null; + } + const usage = payload.usage; + if (!usage || typeof usage !== 'object') { + return null; + } + return { + prompt_tokens: Number(usage.prompt_tokens ?? 0) || 0, + prompt_cached_tokens: usage.prompt_cached_tokens == null + ? null + : Number(usage.prompt_cached_tokens), + prompt_cache_creation_tokens: usage.prompt_cache_creation_tokens == null + ? null + : Number(usage.prompt_cache_creation_tokens), + prompt_image_tokens: usage.prompt_image_tokens == null + ? null + : Number(usage.prompt_image_tokens), + completion_tokens: Number(usage.completion_tokens ?? 0) || 0, + total_tokens: Number(usage.total_tokens ?? 0) || 0, + }; + } + raiseHttpError(statusCode, detail) { + const errorDetail = detail || `HTTP ${statusCode}`; + if (statusCode === 401) { + throw new ModelProviderError(`Invalid API key. ${errorDetail}`, 401, this.model); + } + if (statusCode === 402) { + throw new ModelProviderError(`Insufficient credits. ${errorDetail}`, 402, this.model); + } + if (statusCode === 429) { + throw new ModelRateLimitError(`Rate limit exceeded. ${errorDetail}`, 429, this.model); + } + if (RETRYABLE_STATUS_CODES.has(statusCode)) { + throw new ModelProviderError(`Server error. ${errorDetail}`, statusCode, this.model); + } + throw new ModelProviderError(`API request failed: ${errorDetail}`, statusCode, this.model); + } + isRetryableNetworkError(error) { + if (!(error instanceof Error)) { + return false; + } + if (isAbortError(error)) { + return false; + } + const message = error.message.toLowerCase(); + return (message.includes('fetch failed') || + message.includes('networkerror') || + message.includes('econnreset') || + message.includes('econnrefused') || + message.includes('enotfound') || + message.includes('etimedout') || + message.includes('timeout')); + } + async makeRequest(payload, signal) { + const controller = new AbortController(); + const timeoutHandle = setTimeout(() => controller.abort(), this.timeoutMs); + const onAbort = () => controller.abort(); + if (signal) { + if (signal.aborted) { + controller.abort(); + } + else { + signal.addEventListener('abort', onAbort, { once: true }); + } + } + try { + const response = await this.fetchImplementation(`${this.baseUrl}/v1/chat/completions`, { + method: 'POST', + headers: { + Authorization: `Bearer ${this.apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), + signal: controller.signal, + }); + if (!response.ok) { + let detail = ''; + try { + const errorJson = await response.json(); + detail = getJsonErrorDetail(errorJson); + } + catch { + try { + detail = await response.text(); + } + catch { + detail = ''; + } + } + throw new HttpStatusError(response.status, detail); + } + const result = await response.json(); + return result && typeof result === 'object' + ? result + : {}; + } + catch (error) { + if (isAbortError(error) && !signal?.aborted) { + throw new ModelProviderError(`Request timed out after ${Math.round(this.timeoutMs / 1000)}s`, 408, this.model); + } + throw error; + } + finally { + clearTimeout(timeoutHandle); + signal?.removeEventListener('abort', onAbort); + } + } + async ainvoke(messages, output_format, options = {}) { + const payload = { + model: this.model, + messages: messages.map((message) => this.serializeMessage(message)), + fast: this.fast, + request_type: options.request_type ?? 'browser_agent', + anonymized_telemetry: CONFIG.ANONYMIZED_TELEMETRY, + }; + if (typeof options.session_id === 'string') { + payload.session_id = options.session_id; + } + const schema = this.getOutputSchema(output_format); + if (schema) { + payload.output_format = schema; + } + let result = null; + for (let attempt = 0; attempt < this.maxRetries; attempt += 1) { + try { + result = await this.makeRequest(payload, options.signal); + break; + } + catch (error) { + if (isAbortError(error)) { + throw error; + } + const statusCode = error instanceof HttpStatusError + ? error.statusCode + : (error?.statusCode ?? null); + const retryableHttp = typeof statusCode === 'number' && + RETRYABLE_STATUS_CODES.has(statusCode); + const retryableNetwork = this.isRetryableNetworkError(error); + if (attempt < this.maxRetries - 1 && + (retryableHttp || retryableNetwork)) { + const delaySeconds = Math.min(this.retryBaseDelay * 2 ** attempt, this.retryMaxDelay); + const jitter = Math.random() * delaySeconds * 0.1; + const sleepMs = Math.max(1, Math.round((delaySeconds + jitter) * 1000)); + await sleep(sleepMs); + continue; + } + if (error instanceof HttpStatusError) { + this.raiseHttpError(error.statusCode, error.detail); + } + if (error instanceof ModelProviderError) { + throw error; + } + throw new ModelProviderError(`Failed to connect to browser-use API: ${error instanceof Error ? error.message : String(error)}`, 502, this.model); + } + } + if (result == null) { + throw new ModelProviderError('Request failed without a response payload.', 502, this.model); + } + const usage = this.getUsage(result); + const completionPayload = result.completion; + if (!output_format) { + const textCompletion = typeof completionPayload === 'string' + ? completionPayload + : JSON.stringify(completionPayload ?? ''); + return new ChatInvokeCompletion(textCompletion, usage); + } + const parsedPayload = typeof completionPayload === 'string' + ? (() => { + try { + return JSON.parse(completionPayload); + } + catch { + return completionPayload; + } + })() + : completionPayload; + const completion = this.parseOutput(output_format, parsedPayload); + return new ChatInvokeCompletion(completion, usage); + } +} diff --git a/dist/llm/browser-use/index.d.ts b/dist/llm/browser-use/index.d.ts new file mode 100644 index 00000000..69a16816 --- /dev/null +++ b/dist/llm/browser-use/index.d.ts @@ -0,0 +1 @@ +export * from './chat.js'; diff --git a/dist/llm/browser-use/index.js b/dist/llm/browser-use/index.js new file mode 100644 index 00000000..69a16816 --- /dev/null +++ b/dist/llm/browser-use/index.js @@ -0,0 +1 @@ +export * from './chat.js'; diff --git a/dist/llm/cerebras/chat.d.ts b/dist/llm/cerebras/chat.d.ts new file mode 100644 index 00000000..1aa57b1f --- /dev/null +++ b/dist/llm/cerebras/chat.d.ts @@ -0,0 +1,39 @@ +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import type { Message } from '../messages.js'; +import { ChatInvokeCompletion } from '../views.js'; +export interface ChatCerebrasOptions { + model?: string; + apiKey?: string; + baseURL?: string; + timeout?: number | null; + clientParams?: Record | null; + temperature?: number | null; + maxTokens?: number | null; + topP?: number | null; + seed?: number | null; + maxRetries?: number; + removeMinItemsFromSchema?: boolean; + removeDefaultsFromSchema?: boolean; +} +export declare class ChatCerebras implements BaseChatModel { + model: string; + provider: string; + private client; + private temperature; + private maxTokens; + private topP; + private seed; + private removeMinItemsFromSchema; + private removeDefaultsFromSchema; + constructor(options?: string | ChatCerebrasOptions); + get name(): string; + get model_name(): string; + private getUsage; + private getSchemaCandidate; + private extractJsonFromContent; + private appendJsonInstruction; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/cerebras/chat.js b/dist/llm/cerebras/chat.js new file mode 100644 index 00000000..4c9c0e08 --- /dev/null +++ b/dist/llm/cerebras/chat.js @@ -0,0 +1,178 @@ +import OpenAI from 'openai'; +import { ModelProviderError, ModelRateLimitError } from '../exceptions.js'; +import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { CerebrasMessageSerializer, } from './serializer.js'; +export class ChatCerebras { + model; + provider = 'cerebras'; + client; + temperature; + maxTokens; + topP; + seed; + removeMinItemsFromSchema; + removeDefaultsFromSchema; + constructor(options = {}) { + const normalizedOptions = typeof options === 'string' ? { model: options } : options; + const { model = 'llama3.1-8b', apiKey = process.env.CEREBRAS_API_KEY, baseURL = process.env.CEREBRAS_BASE_URL || 'https://api.cerebras.ai/v1', timeout = null, clientParams = null, temperature = 0.2, maxTokens = 4096, topP = null, seed = null, maxRetries = 5, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions; + this.model = model; + this.temperature = temperature; + this.maxTokens = maxTokens; + this.topP = topP; + this.seed = seed; + this.removeMinItemsFromSchema = removeMinItemsFromSchema; + this.removeDefaultsFromSchema = removeDefaultsFromSchema; + this.client = new OpenAI({ + apiKey, + baseURL, + ...(timeout !== null ? { timeout } : {}), + maxRetries, + ...(clientParams ?? {}), + }); + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + getUsage(response) { + if (!response.usage) { + return null; + } + return { + prompt_tokens: response.usage.prompt_tokens, + prompt_cached_tokens: response.usage.prompt_tokens_details?.cached_tokens ?? null, + prompt_cache_creation_tokens: null, + prompt_image_tokens: null, + completion_tokens: response.usage.completion_tokens, + total_tokens: response.usage.total_tokens, + }; + } + getSchemaCandidate(output_format) { + const output = output_format; + if (output && + typeof output === 'object' && + typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return output; + } + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return output.schema; + } + return null; + } + extractJsonFromContent(content) { + const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/i); + const normalized = (fenced ? fenced[1] : content).trim(); + const firstBrace = normalized.indexOf('{'); + if (firstBrace < 0) { + return normalized; + } + let depth = 0; + for (let idx = firstBrace; idx < normalized.length; idx += 1) { + const char = normalized[idx]; + if (char === '{') { + depth += 1; + } + else if (char === '}') { + depth -= 1; + if (depth === 0) { + return normalized.slice(firstBrace, idx + 1); + } + } + } + return normalized.slice(firstBrace); + } + appendJsonInstruction(serializedMessages, schemaText) { + const instruction = `\n\nPlease respond with a JSON object that follows this exact schema:\n` + + `${schemaText}\n\n` + + `Your response must be valid JSON only, no other text.`; + if (serializedMessages.length === 0) { + return [{ role: 'user', content: instruction }]; + } + const cloned = [...serializedMessages]; + const last = cloned[cloned.length - 1]; + if (last?.role === 'user') { + if (typeof last.content === 'string') { + last.content = `${last.content}${instruction}`; + return cloned; + } + if (Array.isArray(last.content)) { + last.content = [...last.content, { type: 'text', text: instruction }]; + return cloned; + } + } + cloned.push({ role: 'user', content: instruction }); + return cloned; + } + async ainvoke(messages, output_format, options = {}) { + const serializer = new CerebrasMessageSerializer(); + const cerebrasMessages = serializer.serialize(messages); + const modelParams = {}; + if (this.temperature !== null) { + modelParams.temperature = this.temperature; + } + if (this.maxTokens !== null) { + modelParams.max_tokens = this.maxTokens; + } + if (this.topP !== null) { + modelParams.top_p = this.topP; + } + if (this.seed !== null) { + modelParams.seed = this.seed; + } + const zodSchemaCandidate = this.getSchemaCandidate(output_format); + let requestMessages = cerebrasMessages; + if (zodSchemaCandidate) { + const rawSchema = zodSchemaToJsonSchema(zodSchemaCandidate, { + name: 'agent_output', + target: 'jsonSchema7', + }); + const optimizedSchema = SchemaOptimizer.createOptimizedJsonSchema(rawSchema, { + removeMinItems: this.removeMinItemsFromSchema, + removeDefaults: this.removeDefaultsFromSchema, + }); + requestMessages = this.appendJsonInstruction(cerebrasMessages, JSON.stringify(optimizedSchema, null, 2)); + } + try { + const response = await this.client.chat.completions.create({ + model: this.model, + messages: requestMessages, + ...modelParams, + }, options.signal ? { signal: options.signal } : undefined); + const content = response.choices[0].message.content || ''; + const usage = this.getUsage(response); + const stopReason = response.choices[0].finish_reason ?? null; + let completion = content; + if (output_format) { + const jsonSource = zodSchemaCandidate + ? this.extractJsonFromContent(content) + : content; + const parsedJson = JSON.parse(jsonSource); + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + completion = output.schema.parse(parsedJson); + } + else { + completion = output.parse(parsedJson); + } + } + return new ChatInvokeCompletion(completion, usage, null, null, stopReason); + } + catch (error) { + if (error?.status === 429) { + throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model); + } + throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model); + } + } +} diff --git a/dist/llm/cerebras/index.d.ts b/dist/llm/cerebras/index.d.ts new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/cerebras/index.d.ts @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/cerebras/index.js b/dist/llm/cerebras/index.js new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/cerebras/index.js @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/cerebras/serializer.d.ts b/dist/llm/cerebras/serializer.d.ts new file mode 100644 index 00000000..dd51d6de --- /dev/null +++ b/dist/llm/cerebras/serializer.d.ts @@ -0,0 +1,7 @@ +import { type Message } from '../messages.js'; +export type CerebrasMessage = Record; +export declare class CerebrasMessageSerializer { + private serializeContent; + private serializeToolCalls; + serialize(messages: Message[]): CerebrasMessage[]; +} diff --git a/dist/llm/cerebras/serializer.js b/dist/llm/cerebras/serializer.js new file mode 100644 index 00000000..31b23376 --- /dev/null +++ b/dist/llm/cerebras/serializer.js @@ -0,0 +1,82 @@ +import { AssistantMessage, ContentPartImageParam, ContentPartRefusalParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js'; +export class CerebrasMessageSerializer { + serializeContent(content) { + if (content == null) { + return ''; + } + if (typeof content === 'string') { + return content; + } + if (!Array.isArray(content)) { + return String(content); + } + const parts = []; + for (const part of content) { + if (part instanceof ContentPartTextParam) { + parts.push({ type: 'text', text: part.text }); + } + else if (part instanceof ContentPartImageParam) { + parts.push({ + type: 'image_url', + image_url: { + url: part.image_url.url, + }, + }); + } + else if (part instanceof ContentPartRefusalParam) { + parts.push({ type: 'text', text: `[Refusal] ${part.refusal}` }); + } + } + return parts; + } + serializeToolCalls(toolCalls) { + if (!toolCalls || !toolCalls.length) { + return undefined; + } + return toolCalls.map((toolCall) => { + let argumentsPayload; + try { + argumentsPayload = JSON.parse(toolCall.functionCall.arguments); + } + catch { + argumentsPayload = { arguments: toolCall.functionCall.arguments }; + } + return { + id: toolCall.id, + type: 'function', + function: { + name: toolCall.functionCall.name, + arguments: argumentsPayload, + }, + }; + }); + } + serialize(messages) { + return messages.map((message) => { + if (message instanceof UserMessage) { + return { + role: 'user', + content: this.serializeContent(message.content), + }; + } + if (message instanceof SystemMessage) { + return { + role: 'system', + content: this.serializeContent(message.content), + }; + } + if (message instanceof AssistantMessage) { + const payload = { + role: 'assistant', + content: this.serializeContent(message.content), + }; + const toolCalls = this.serializeToolCalls(message.tool_calls); + if (toolCalls) { + payload.tool_calls = toolCalls; + } + return payload; + } + throw new Error(`Unknown message type: ${message?.constructor?.name ?? typeof message}`); + }); + } +} diff --git a/dist/llm/deepseek/chat.d.ts b/dist/llm/deepseek/chat.d.ts new file mode 100644 index 00000000..1eadc5bd --- /dev/null +++ b/dist/llm/deepseek/chat.d.ts @@ -0,0 +1,32 @@ +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import type { Message } from '../messages.js'; +import { ChatInvokeCompletion } from '../views.js'; +export interface ChatDeepSeekOptions { + model?: string; + apiKey?: string; + baseURL?: string; + timeout?: number | null; + clientParams?: Record | null; + temperature?: number | null; + maxTokens?: number | null; + topP?: number | null; + seed?: number | null; + maxRetries?: number; +} +export declare class ChatDeepSeek implements BaseChatModel { + model: string; + provider: string; + private client; + private temperature; + private maxTokens; + private topP; + private seed; + constructor(options?: string | ChatDeepSeekOptions); + get name(): string; + get model_name(): string; + private getUsage; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/deepseek/chat.js b/dist/llm/deepseek/chat.js new file mode 100644 index 00000000..f78d1f46 --- /dev/null +++ b/dist/llm/deepseek/chat.js @@ -0,0 +1,164 @@ +import OpenAI from 'openai'; +import { ModelProviderError, ModelRateLimitError } from '../exceptions.js'; +import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { DeepSeekMessageSerializer } from './serializer.js'; +export class ChatDeepSeek { + model; + provider = 'deepseek'; + client; + temperature; + maxTokens; + topP; + seed; + constructor(options = {}) { + const normalizedOptions = typeof options === 'string' ? { model: options } : options; + const { model = 'deepseek-chat', apiKey = process.env.DEEPSEEK_API_KEY, baseURL = 'https://api.deepseek.com/v1', timeout = null, clientParams = null, temperature = null, maxTokens = null, topP = null, seed = null, maxRetries = 10, } = normalizedOptions; + this.model = model; + this.temperature = temperature; + this.maxTokens = maxTokens; + this.topP = topP; + this.seed = seed; + this.client = new OpenAI({ + apiKey, + baseURL, + ...(timeout !== null ? { timeout } : {}), + maxRetries, + ...(clientParams ?? {}), + }); + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + getUsage(response) { + if (!response.usage) { + return null; + } + return { + prompt_tokens: response.usage.prompt_tokens, + prompt_cached_tokens: response.usage.prompt_tokens_details?.cached_tokens ?? null, + prompt_cache_creation_tokens: null, + prompt_image_tokens: null, + completion_tokens: response.usage.completion_tokens, + total_tokens: response.usage.total_tokens, + }; + } + async ainvoke(messages, output_format, options = {}) { + const serializer = new DeepSeekMessageSerializer(); + const deepseekMessages = serializer.serialize(messages); + const modelParams = {}; + if (this.temperature !== null) { + modelParams.temperature = this.temperature; + } + if (this.maxTokens !== null) { + modelParams.max_tokens = this.maxTokens; + } + if (this.topP !== null) { + modelParams.top_p = this.topP; + } + if (this.seed !== null) { + modelParams.seed = this.seed; + } + const zodSchemaCandidate = (() => { + const output = output_format; + if (output && + typeof output === 'object' && + typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return output; + } + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return output.schema; + } + return null; + })(); + try { + if (output_format && zodSchemaCandidate) { + const rawSchema = zodSchemaToJsonSchema(zodSchemaCandidate, { + name: 'response', + target: 'jsonSchema7', + }); + const optimizedSchema = SchemaOptimizer.createOptimizedJsonSchema(rawSchema); + delete optimizedSchema.title; + const response = await this.client.chat.completions.create({ + model: this.model, + messages: deepseekMessages, + tools: [ + { + type: 'function', + function: { + name: 'response', + description: 'Return a JSON object of type response', + parameters: optimizedSchema, + }, + }, + ], + tool_choice: { + type: 'function', + function: { name: 'response' }, + }, + ...modelParams, + }, options.signal ? { signal: options.signal } : undefined); + const usage = this.getUsage(response); + const stopReason = response.choices[0].finish_reason ?? null; + const toolCalls = response.choices[0].message.tool_calls; + if (!toolCalls?.length) { + throw new ModelProviderError('Expected tool_calls in response but got none', 502, this.model); + } + const rawArguments = toolCalls[0]?.function?.arguments; + const parsedArguments = typeof rawArguments === 'string' + ? JSON.parse(rawArguments) + : rawArguments; + const output = output_format; + const completion = output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function' + ? output.schema.parse(parsedArguments) + : output.parse(parsedArguments); + return new ChatInvokeCompletion(completion, usage, null, null, stopReason); + } + const responseFormat = output_format ? { type: 'json_object' } : undefined; + const response = await this.client.chat.completions.create({ + model: this.model, + messages: deepseekMessages, + response_format: responseFormat, + ...modelParams, + }, options.signal ? { signal: options.signal } : undefined); + const content = response.choices[0].message.content || ''; + const usage = this.getUsage(response); + const stopReason = response.choices[0].finish_reason ?? null; + let completion = content; + if (output_format) { + const parsedJson = JSON.parse(content); + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + completion = output.schema.parse(parsedJson); + } + else { + completion = output_format.parse(parsedJson); + } + } + return new ChatInvokeCompletion(completion, usage, null, null, stopReason); + } + catch (error) { + if (error?.status === 429) { + throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model); + } + if (error?.status >= 500) { + throw new ModelProviderError(error?.message ?? 'Server error', error.status, this.model); + } + throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model); + } + } +} diff --git a/dist/llm/deepseek/index.d.ts b/dist/llm/deepseek/index.d.ts new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/deepseek/index.d.ts @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/deepseek/index.js b/dist/llm/deepseek/index.js new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/deepseek/index.js @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/deepseek/serializer.d.ts b/dist/llm/deepseek/serializer.d.ts new file mode 100644 index 00000000..782710ac --- /dev/null +++ b/dist/llm/deepseek/serializer.d.ts @@ -0,0 +1,6 @@ +import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs'; +import { type Message } from '../messages.js'; +export declare class DeepSeekMessageSerializer { + serialize(messages: Message[]): ChatCompletionMessageParam[]; + private serializeMessage; +} diff --git a/dist/llm/deepseek/serializer.js b/dist/llm/deepseek/serializer.js new file mode 100644 index 00000000..fb3d8e90 --- /dev/null +++ b/dist/llm/deepseek/serializer.js @@ -0,0 +1,57 @@ +import { AssistantMessage, ContentPartImageParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js'; +export class DeepSeekMessageSerializer { + serialize(messages) { + return messages.map((message) => this.serializeMessage(message)); + } + serializeMessage(message) { + if (message instanceof UserMessage) { + return { + role: 'user', + content: Array.isArray(message.content) + ? message.content.map((part) => { + if (part instanceof ContentPartTextParam) { + return { type: 'text', text: part.text }; + } + // DeepSeek might not support images in all models, but we'll include it for now + if (part instanceof ContentPartImageParam) { + return { + type: 'image_url', + image_url: { + url: part.image_url.url, + detail: part.image_url.detail, + }, + }; + } + return { type: 'text', text: '' }; + }) + : message.content, + name: message.name || undefined, + }; + } + if (message instanceof SystemMessage) { + return { + role: 'system', + content: Array.isArray(message.content) + ? message.content.map((part) => part.text).join('\n') + : message.content, + name: message.name || undefined, + }; + } + if (message instanceof AssistantMessage) { + return { + role: 'assistant', + content: typeof message.content === 'string' ? message.content : null, + // DeepSeek supports tool calls in newer models + tool_calls: message.tool_calls?.map((toolCall) => ({ + id: toolCall.id, + type: 'function', + function: { + name: toolCall.functionCall.name, + arguments: toolCall.functionCall.arguments, + }, + })), + }; + } + throw new Error(`Unknown message type: ${message.constructor.name}`); + } +} diff --git a/dist/llm/exceptions.d.ts b/dist/llm/exceptions.d.ts new file mode 100644 index 00000000..5bbc6cdf --- /dev/null +++ b/dist/llm/exceptions.d.ts @@ -0,0 +1,10 @@ +export declare class ModelError extends Error { +} +export declare class ModelProviderError extends ModelError { + statusCode: number; + model: string | null; + constructor(message: string, statusCode?: number, model?: string | null); +} +export declare class ModelRateLimitError extends ModelProviderError { + constructor(message: string, statusCode?: number, model?: string | null); +} diff --git a/dist/llm/exceptions.js b/dist/llm/exceptions.js new file mode 100644 index 00000000..6a2b9c29 --- /dev/null +++ b/dist/llm/exceptions.js @@ -0,0 +1,18 @@ +export class ModelError extends Error { +} +export class ModelProviderError extends ModelError { + statusCode; + model; + constructor(message, statusCode = 502, model = null) { + super(message); + this.statusCode = statusCode; + this.model = model; + this.name = 'ModelProviderError'; + } +} +export class ModelRateLimitError extends ModelProviderError { + constructor(message, statusCode = 429, model = null) { + super(message, statusCode, model); + this.name = 'ModelRateLimitError'; + } +} diff --git a/dist/llm/google/chat.d.ts b/dist/llm/google/chat.d.ts new file mode 100644 index 00000000..fb846007 --- /dev/null +++ b/dist/llm/google/chat.d.ts @@ -0,0 +1,64 @@ +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import { ChatInvokeCompletion } from '../views.js'; +import type { Message } from '../messages.js'; +export interface ChatGoogleOptions { + model?: string; + apiKey?: string; + apiVersion?: string; + baseUrl?: string; + vertexai?: boolean; + vertexAi?: boolean; + project?: string; + location?: string; + httpOptions?: Record; + googleAuthOptions?: Record; + credentials?: Record; + temperature?: number | null; + topP?: number | null; + seed?: number | null; + thinkingBudget?: number | null; + thinkingLevel?: 'minimal' | 'low' | 'medium' | 'high' | null; + maxOutputTokens?: number | null; + config?: Record | null; + includeSystemInUser?: boolean; + supportsStructuredOutput?: boolean; + maxRetries?: number; + retryableStatusCodes?: number[]; + retryBaseDelay?: number; + retryMaxDelay?: number; +} +export declare class ChatGoogle implements BaseChatModel { + model: string; + provider: string; + private client; + private temperature; + private topP; + private seed; + private thinkingBudget; + private thinkingLevel; + private maxOutputTokens; + private config; + private includeSystemInUser; + private supportsStructuredOutput; + private maxRetries; + private retryableStatusCodes; + private retryBaseDelay; + private retryMaxDelay; + constructor(options?: string | ChatGoogleOptions); + get name(): string; + get model_name(): string; + private getUsage; + /** + * Clean up JSON schema for Google's format + * Google API has specific requirements for responseSchema + */ + private _cleanSchemaForGoogle; + private _parseStructuredJson; + private _extractStatusCode; + private _toModelProviderError; + private _sleep; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/google/chat.js b/dist/llm/google/chat.js new file mode 100644 index 00000000..ccbb8c12 --- /dev/null +++ b/dist/llm/google/chat.js @@ -0,0 +1,349 @@ +import { GoogleGenAI } from '@google/genai'; +import { ModelProviderError } from '../exceptions.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js'; +import { GoogleMessageSerializer } from './serializer.js'; +export class ChatGoogle { + model; + provider = 'google'; + client; + temperature; + topP; + seed; + thinkingBudget; + thinkingLevel; + maxOutputTokens; + config; + includeSystemInUser; + supportsStructuredOutput; + maxRetries; + retryableStatusCodes; + retryBaseDelay; + retryMaxDelay; + constructor(options = {}) { + const normalizedOptions = typeof options === 'string' ? { model: options } : options; + const { model = 'gemini-2.5-flash', apiKey = process.env.GOOGLE_API_KEY, apiVersion = process.env.GOOGLE_API_VERSION, baseUrl = process.env.GOOGLE_API_BASE_URL, vertexai, vertexAi, project, location, httpOptions, googleAuthOptions, credentials, temperature = 0.5, topP = null, seed = null, thinkingBudget = null, thinkingLevel = null, maxOutputTokens = 8096, config = null, includeSystemInUser = false, supportsStructuredOutput = true, maxRetries = 5, retryableStatusCodes = [429, 500, 502, 503, 504], retryBaseDelay = 1.0, retryMaxDelay = 60.0, } = normalizedOptions; + this.model = model; + this.temperature = temperature; + this.topP = topP; + this.seed = seed; + this.thinkingBudget = thinkingBudget; + this.thinkingLevel = thinkingLevel; + this.maxOutputTokens = maxOutputTokens; + this.config = config ? { ...config } : null; + this.includeSystemInUser = includeSystemInUser; + this.supportsStructuredOutput = supportsStructuredOutput; + this.maxRetries = Math.max(1, maxRetries); + this.retryableStatusCodes = [...retryableStatusCodes]; + this.retryBaseDelay = retryBaseDelay; + this.retryMaxDelay = retryMaxDelay; + const resolvedGoogleAuthOptions = credentials == null + ? googleAuthOptions + : { + ...(googleAuthOptions ?? {}), + credentials, + }; + const resolvedVertexAi = vertexai ?? vertexAi; + const clientOptions = { + ...(apiKey != null ? { apiKey } : {}), + ...(baseUrl ? { baseUrl } : {}), + ...(apiVersion ? { apiVersion } : {}), + ...(resolvedVertexAi != null ? { vertexai: resolvedVertexAi } : {}), + ...(project ? { project } : {}), + ...(location ? { location } : {}), + ...(httpOptions ? { httpOptions } : {}), + ...(resolvedGoogleAuthOptions + ? { googleAuthOptions: resolvedGoogleAuthOptions } + : {}), + }; + this.client = new GoogleGenAI(clientOptions); + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + getUsage(result) { + const usage = result?.usageMetadata; + if (!usage) { + return null; + } + let imageTokens = 0; + const promptTokenDetails = Array.isArray(usage.promptTokensDetails) + ? usage.promptTokensDetails + : []; + for (const detail of promptTokenDetails) { + if (String(detail?.modality ?? '').toUpperCase() === 'IMAGE') { + imageTokens += Number(detail?.tokenCount ?? 0) || 0; + } + } + const completionTokens = (Number(usage.candidatesTokenCount ?? 0) || 0) + + (Number(usage.thoughtsTokenCount ?? 0) || 0); + return { + prompt_tokens: Number(usage.promptTokenCount ?? 0) || 0, + prompt_cached_tokens: usage.cachedContentTokenCount == null + ? null + : Number(usage.cachedContentTokenCount), + prompt_cache_creation_tokens: null, + prompt_image_tokens: imageTokens, + completion_tokens: completionTokens, + total_tokens: Number(usage.totalTokenCount ?? 0) || 0, + }; + } + /** + * Clean up JSON schema for Google's format + * Google API has specific requirements for responseSchema + */ + _cleanSchemaForGoogle(schema) { + if (!schema || typeof schema !== 'object') { + return schema; + } + const cleaned = {}; + for (const [key, value] of Object.entries(schema)) { + // Skip unsupported keys + if (key === '$schema' || + key === 'additionalProperties' || + key === '$ref' || + key === 'definitions') { + continue; + } + if (key === 'properties' && typeof value === 'object') { + cleaned.properties = {}; + for (const [propKey, propValue] of Object.entries(value)) { + // Align python: hide programmatic extraction schema field from LLM JSON schema. + if (propKey === 'output_schema') { + continue; + } + cleaned.properties[propKey] = this._cleanSchemaForGoogle(propValue); + } + } + else if (key === 'items' && typeof value === 'object') { + cleaned.items = this._cleanSchemaForGoogle(value); + } + else if (typeof value === 'object' && !Array.isArray(value)) { + cleaned[key] = this._cleanSchemaForGoogle(value); + } + else { + cleaned[key] = value; + } + } + const schemaType = String(cleaned.type ?? '').toUpperCase(); + if (schemaType === 'OBJECT' && + cleaned.properties && + typeof cleaned.properties === 'object' && + !Array.isArray(cleaned.properties) && + Object.keys(cleaned.properties).length === 0) { + cleaned.properties = { + _placeholder: { type: 'string' }, + }; + } + if (Array.isArray(cleaned.required) && + cleaned.properties && + typeof cleaned.properties === 'object' && + !Array.isArray(cleaned.properties)) { + const validKeys = new Set(Object.keys(cleaned.properties)); + cleaned.required = cleaned.required.filter((name) => typeof name === 'string' && validKeys.has(name)); + } + return cleaned; + } + _parseStructuredJson(text) { + let jsonText = String(text ?? '').trim(); + const fencedMatch = jsonText.match(/```(?:json)?\s*([\s\S]*?)```/i); + if (fencedMatch && fencedMatch[1]) { + jsonText = fencedMatch[1].trim(); + } + const firstBrace = jsonText.indexOf('{'); + const lastBrace = jsonText.lastIndexOf('}'); + if (firstBrace === -1 || lastBrace === -1 || lastBrace <= firstBrace) { + throw new Error(`Expected JSON response but got plain text: "${jsonText.slice(0, 50)}..."`); + } + return JSON.parse(jsonText.slice(firstBrace, lastBrace + 1)); + } + _extractStatusCode(error) { + const directStatus = Number(error?.status ?? + error?.statusCode ?? + error?.response?.status ?? + error?.response?.statusCode); + if (Number.isFinite(directStatus)) { + return directStatus; + } + const message = String(error?.message ?? error ?? '').toLowerCase(); + if (/(rate limit|resource exhausted|quota exceeded|too many requests|429)/.test(message)) { + return 429; + } + if (/(service unavailable|internal server error|bad gateway|503|502|500)/.test(message)) { + return 503; + } + if (/(forbidden|403)/.test(message)) { + return 403; + } + if (/(timeout|timed out|cancelled|canceled)/.test(message)) { + return 504; + } + return null; + } + _toModelProviderError(error) { + if (error instanceof ModelProviderError) { + return error; + } + return new ModelProviderError(error?.message ?? String(error), this._extractStatusCode(error) ?? 502, this.model); + } + async _sleep(ms) { + await new Promise((resolve) => setTimeout(resolve, ms)); + } + async ainvoke(messages, output_format, options = {}) { + const serializer = new GoogleMessageSerializer(); + const { contents, systemInstruction } = serializer.serializeWithSystem(messages, this.includeSystemInUser); + const generationConfig = this.config ? { ...this.config } : {}; + if (this.temperature !== null) { + generationConfig.temperature = this.temperature; + } + if (this.topP !== null) { + generationConfig.topP = this.topP; + } + if (this.seed !== null) { + generationConfig.seed = this.seed; + } + const isGemini3Pro = this.model.includes('gemini-3-pro'); + const isGemini3Flash = this.model.includes('gemini-3-flash'); + if (isGemini3Pro) { + let level = this.thinkingLevel ?? 'low'; + if (level === 'minimal' || level === 'medium') { + level = 'low'; + } + generationConfig.thinkingConfig = { + thinkingLevel: level.toUpperCase(), + }; + } + else if (isGemini3Flash) { + if (this.thinkingLevel !== null) { + generationConfig.thinkingConfig = { + thinkingLevel: this.thinkingLevel.toUpperCase(), + }; + } + else { + generationConfig.thinkingConfig = { + thinkingBudget: this.thinkingBudget === null ? -1 : this.thinkingBudget, + }; + } + } + else { + let budget = this.thinkingBudget; + if (budget === null && + (this.model.includes('gemini-2.5') || + this.model.includes('gemini-flash'))) { + budget = -1; + } + if (budget !== null) { + generationConfig.thinkingConfig = { thinkingBudget: budget }; + } + } + if (this.maxOutputTokens !== null) { + generationConfig.maxOutputTokens = this.maxOutputTokens; + } + // Try to get schema from output_format + const schemaForJson = (() => { + const output = output_format; + if (output && + typeof output === 'object' && + typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return output; + } + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return output.schema; + } + return null; + })(); + let cleanSchemaForJson = null; + if (schemaForJson) { + try { + const jsonSchema = zodSchemaToJsonSchema(schemaForJson); + const optimizedSchema = SchemaOptimizer.createGeminiOptimizedSchema(jsonSchema); + cleanSchemaForJson = this._cleanSchemaForGoogle(optimizedSchema); + } + catch { + cleanSchemaForJson = null; + } + } + if (cleanSchemaForJson && this.supportsStructuredOutput) { + generationConfig.responseMimeType = 'application/json'; + generationConfig.responseSchema = cleanSchemaForJson; + } + const requestContents = contents.map((entry) => ({ + ...entry, + parts: Array.isArray(entry?.parts) + ? entry.parts.map((part) => ({ ...part })) + : entry?.parts, + })); + if (output_format && cleanSchemaForJson && !this.supportsStructuredOutput) { + const jsonInstruction = '\n\nPlease respond with a valid JSON object that matches this schema: ' + + JSON.stringify(cleanSchemaForJson); + for (let i = requestContents.length - 1; i >= 0; i -= 1) { + const content = requestContents[i]; + if (content?.role === 'user' && Array.isArray(content?.parts)) { + content.parts = [...content.parts, { text: jsonInstruction }]; + break; + } + } + } + const request = { + model: this.model, + contents: requestContents, + }; + if (systemInstruction && !this.includeSystemInUser) { + request.systemInstruction = { + role: 'system', + parts: [{ text: systemInstruction }], + }; + } + if (Object.keys(generationConfig).length > 0) { + request.generationConfig = generationConfig; + } + for (let attempt = 0; attempt < this.maxRetries; attempt += 1) { + try { + const result = await this.client.models.generateContent(request, options.signal ? { signal: options.signal } : undefined); + const candidate = result.candidates?.[0]; + const textParts = candidate?.content?.parts?.filter((p) => p.text) || []; + const text = textParts.map((p) => p.text).join(''); + let completion = text; + const stopReason = result?.candidates?.[0]?.finishReason ?? null; + let parsed = text; + if (output_format && schemaForJson) { + parsed = this._parseStructuredJson(text); + } + if (output_format) { + const output = output_format; + if (schemaForJson && + output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + completion = output.schema.parse(parsed); + } + else { + completion = output.parse(parsed); + } + } + return new ChatInvokeCompletion(completion, this.getUsage(result), null, null, stopReason); + } + catch (error) { + const providerError = this._toModelProviderError(error); + const shouldRetry = this.retryableStatusCodes.includes(providerError.statusCode) && + attempt < this.maxRetries - 1; + if (!shouldRetry) { + throw providerError; + } + const delaySeconds = Math.min(this.retryBaseDelay * 2 ** attempt, this.retryMaxDelay); + const jitter = Math.random() * delaySeconds * 0.1; + await this._sleep((delaySeconds + jitter) * 1000); + } + } + throw new ModelProviderError('Retry loop completed without response', 500, this.model); + } +} diff --git a/dist/llm/google/index.d.ts b/dist/llm/google/index.d.ts new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/google/index.d.ts @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/google/index.js b/dist/llm/google/index.js new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/google/index.js @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/google/serializer.d.ts b/dist/llm/google/serializer.d.ts new file mode 100644 index 00000000..0c4de2d2 --- /dev/null +++ b/dist/llm/google/serializer.d.ts @@ -0,0 +1,14 @@ +import type { Content } from '@google/genai'; +import { type Message } from '../messages.js'; +export interface SerializedGoogleMessages { + contents: Content[]; + systemInstruction: string | null; +} +export declare class GoogleMessageSerializer { + serialize(messages: Message[]): Content[]; + serializeWithSystem(messages: Message[], includeSystemInUser?: boolean): SerializedGoogleMessages; + private serializeUserMessage; + private serializeAssistantMessage; + private serializeImagePart; + private extractMessageText; +} diff --git a/dist/llm/google/serializer.js b/dist/llm/google/serializer.js new file mode 100644 index 00000000..025d18af --- /dev/null +++ b/dist/llm/google/serializer.js @@ -0,0 +1,171 @@ +import { AssistantMessage, ContentPartImageParam, ContentPartRefusalParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js'; +export class GoogleMessageSerializer { + serialize(messages) { + return this.serializeWithSystem(messages).contents; + } + serializeWithSystem(messages, includeSystemInUser = false) { + const contents = []; + const systemParts = []; + let systemInstruction = null; + let injectedSystemIntoUser = false; + for (const message of messages) { + const role = message?.role; + if (message instanceof SystemMessage || + role === 'system' || + role === 'developer') { + const text = this.extractMessageText(message); + if (text) { + if (includeSystemInUser) { + systemParts.push(text); + } + else { + systemInstruction = text; + } + } + continue; + } + if (message instanceof UserMessage) { + const prependSystem = includeSystemInUser && + !injectedSystemIntoUser && + systemParts.length > 0 + ? systemParts.join('\n\n') + : null; + contents.push(this.serializeUserMessage(message, prependSystem)); + if (prependSystem) { + injectedSystemIntoUser = true; + } + continue; + } + if (message instanceof AssistantMessage) { + contents.push(this.serializeAssistantMessage(message)); + } + } + if (includeSystemInUser && + systemParts.length > 0 && + !injectedSystemIntoUser) { + contents.unshift({ + role: 'user', + parts: [{ text: systemParts.join('\n\n') }], + }); + } + return { + contents, + systemInstruction: includeSystemInUser ? null : systemInstruction, + }; + } + serializeUserMessage(message, prependSystem = null) { + const parts = []; + if (prependSystem) { + parts.push({ text: prependSystem }); + } + if (message instanceof UserMessage) { + if (Array.isArray(message.content)) { + for (const part of message.content) { + if (part instanceof ContentPartTextParam) { + parts.push({ text: part.text }); + } + else if (part instanceof ContentPartRefusalParam) { + parts.push({ text: `[Refusal] ${part.refusal}` }); + } + else if (part instanceof ContentPartImageParam) { + parts.push(this.serializeImagePart(part)); + } + } + } + else { + parts.push({ text: message.content }); + } + } + return { + role: 'user', + parts, + }; + } + serializeAssistantMessage(message) { + const parts = []; + if (message instanceof AssistantMessage) { + if (message.content) { + if (typeof message.content === 'string') { + parts.push({ text: message.content }); + } + else if (Array.isArray(message.content)) { + message.content.forEach((part) => { + if (part instanceof ContentPartTextParam) { + parts.push({ text: part.text }); + } + else if (part instanceof ContentPartRefusalParam) { + parts.push({ text: `[Refusal] ${part.refusal}` }); + } + else if (part instanceof ContentPartImageParam) { + parts.push(this.serializeImagePart(part)); + } + }); + } + } + if (message.tool_calls) { + message.tool_calls.forEach((toolCall) => { + let args; + try { + const parsed = JSON.parse(toolCall.functionCall.arguments); + args = + parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? parsed + : {}; + } + catch { + args = {}; + } + parts.push({ + functionCall: { + name: toolCall.functionCall.name, + args, + }, + }); + }); + } + } + return { + role: 'model', + parts: parts, + }; + } + serializeImagePart(part) { + const imageUrl = part.image_url.url; + if (!imageUrl.startsWith('data:')) { + return { text: imageUrl }; + } + const commaIndex = imageUrl.indexOf(','); + if (commaIndex === -1) { + return { text: imageUrl }; + } + const data = imageUrl.slice(commaIndex + 1); + return { + inlineData: { + mimeType: part.image_url.media_type, + data, + }, + }; + } + extractMessageText(message) { + const content = message?.content; + if (typeof content === 'string') { + return content; + } + if (!Array.isArray(content)) { + return ''; + } + const parts = []; + for (const part of content) { + if (part instanceof ContentPartTextParam) { + parts.push(part.text); + } + else if (part && + typeof part === 'object' && + part.type === 'text' && + typeof part.text === 'string') { + parts.push(part.text); + } + } + return parts.join('\n'); + } +} diff --git a/dist/llm/groq/chat.d.ts b/dist/llm/groq/chat.d.ts new file mode 100644 index 00000000..3238e983 --- /dev/null +++ b/dist/llm/groq/chat.d.ts @@ -0,0 +1,34 @@ +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import type { Message } from '../messages.js'; +import { ChatInvokeCompletion } from '../views.js'; +export interface ChatGroqOptions { + model?: string; + apiKey?: string; + baseURL?: string; + temperature?: number | null; + serviceTier?: 'auto' | 'on_demand' | 'flex' | null; + topP?: number | null; + seed?: number | null; + maxRetries?: number; + removeMinItemsFromSchema?: boolean; + removeDefaultsFromSchema?: boolean; +} +export declare class ChatGroq implements BaseChatModel { + model: string; + provider: string; + private client; + private temperature; + private serviceTier; + private topP; + private seed; + private removeMinItemsFromSchema; + private removeDefaultsFromSchema; + constructor(options?: string | ChatGroqOptions); + get name(): string; + get model_name(): string; + private getUsage; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/groq/chat.js b/dist/llm/groq/chat.js new file mode 100644 index 00000000..e79370c3 --- /dev/null +++ b/dist/llm/groq/chat.js @@ -0,0 +1,151 @@ +import Groq from 'groq-sdk'; +import { ModelProviderError, ModelRateLimitError } from '../exceptions.js'; +import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { GroqMessageSerializer } from './serializer.js'; +const JsonSchemaModels = [ + 'meta-llama/llama-4-maverick-17b-128e-instruct', + 'meta-llama/llama-4-scout-17b-16e-instruct', + 'openai/gpt-oss-20b', + 'openai/gpt-oss-120b', +]; +export class ChatGroq { + model; + provider = 'groq'; + client; + temperature; + serviceTier; + topP; + seed; + removeMinItemsFromSchema; + removeDefaultsFromSchema; + constructor(options = {}) { + const normalizedOptions = typeof options === 'string' ? { model: options } : options; + const { model = 'llama-3.1-70b-versatile', apiKey = process.env.GROQ_API_KEY, baseURL, temperature = null, serviceTier = null, topP = null, seed = null, maxRetries = 10, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions; + this.model = model; + this.temperature = temperature; + this.serviceTier = serviceTier; + this.topP = topP; + this.seed = seed; + this.removeMinItemsFromSchema = removeMinItemsFromSchema; + this.removeDefaultsFromSchema = removeDefaultsFromSchema; + this.client = new Groq({ + apiKey, + baseURL, + maxRetries, + }); + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + getUsage(response) { + if (!response?.usage) { + return null; + } + return { + prompt_tokens: response.usage.prompt_tokens, + prompt_cached_tokens: null, + prompt_cache_creation_tokens: null, + prompt_image_tokens: null, + completion_tokens: response.usage.completion_tokens, + total_tokens: response.usage.total_tokens, + }; + } + async ainvoke(messages, output_format, options = {}) { + const serializer = new GroqMessageSerializer(); + const groqMessages = serializer.serialize(messages); + const modelParams = {}; + if (this.temperature !== null) { + modelParams.temperature = this.temperature; + } + if (this.topP !== null) { + modelParams.top_p = this.topP; + } + if (this.seed !== null) { + modelParams.seed = this.seed; + } + if (this.serviceTier !== null) { + modelParams.service_tier = this.serviceTier; + } + const zodSchemaCandidate = (() => { + const output = output_format; + if (output && + typeof output === 'object' && + typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return output; + } + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return output.schema; + } + return null; + })(); + let responseFormat = undefined; + if (zodSchemaCandidate) { + if (JsonSchemaModels.includes(this.model)) { + try { + const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, { + name: 'agent_output', + target: 'jsonSchema7', + }); + const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, { + removeMinItems: this.removeMinItemsFromSchema, + removeDefaults: this.removeDefaultsFromSchema, + }); + responseFormat = { + type: 'json_schema', + json_schema: { + name: 'agent_output', + schema: optimizedJsonSchema, + }, + }; + } + catch { + responseFormat = { type: 'json_object' }; + } + } + else { + responseFormat = { type: 'json_object' }; + } + } + try { + const response = await this.client.chat.completions.create({ + model: this.model, + messages: groqMessages, + response_format: responseFormat, + ...modelParams, + }, options.signal ? { signal: options.signal } : undefined); + const content = response.choices[0].message.content || ''; + const usage = this.getUsage(response); + const stopReason = response.choices[0].finish_reason ?? null; + let completion = content; + if (output_format) { + const parsedJson = JSON.parse(content); + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + completion = output.schema.parse(parsedJson); + } + else { + completion = output.parse(parsedJson); + } + } + return new ChatInvokeCompletion(completion, usage, null, null, stopReason); + } + catch (error) { + if (error?.status === 429) { + throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model); + } + throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model); + } + } +} diff --git a/dist/llm/groq/index.d.ts b/dist/llm/groq/index.d.ts new file mode 100644 index 00000000..627e3dd9 --- /dev/null +++ b/dist/llm/groq/index.d.ts @@ -0,0 +1,3 @@ +export * from './chat.js'; +export * from './serializer.js'; +export * from './parser.js'; diff --git a/dist/llm/groq/index.js b/dist/llm/groq/index.js new file mode 100644 index 00000000..627e3dd9 --- /dev/null +++ b/dist/llm/groq/index.js @@ -0,0 +1,3 @@ +export * from './chat.js'; +export * from './serializer.js'; +export * from './parser.js'; diff --git a/dist/llm/groq/parser.d.ts b/dist/llm/groq/parser.d.ts new file mode 100644 index 00000000..b2e71b3b --- /dev/null +++ b/dist/llm/groq/parser.d.ts @@ -0,0 +1,32 @@ +import type { APIError } from 'groq-sdk'; +export declare class ParseFailedGenerationError extends Error { + constructor(message: string); +} +/** + * Extract JSON from model output, handling both plain JSON and code-block-wrapped JSON. + * This is used to parse Groq's failed_generation field when an API error occurs. + * + * @param error - The Groq API error containing failed_generation + * @param outputFormat - An object with a parse method (typically a Zod schema) + * @returns The parsed output in the expected format + * @throws ParseFailedGenerationError if the failed_generation field is missing + * @throws Error if JSON parsing fails + */ +export declare function tryParseGroqFailedGeneration(error: APIError & { + body?: { + error?: { + failed_generation?: string; + }; + }; +}, outputFormat: { + parse: (input: string) => T; +}): T; +/** + * Fix control characters in JSON string values to make them valid JSON. + * This function escapes literal control characters (newlines, tabs, etc.) that + * appear inside JSON string values, while preserving the JSON structure. + * + * @param content - The JSON string to fix + * @returns The fixed JSON string + */ +export declare function fixControlCharactersInJson(content: string): string; diff --git a/dist/llm/groq/parser.js b/dist/llm/groq/parser.js new file mode 100644 index 00000000..ae6feab1 --- /dev/null +++ b/dist/llm/groq/parser.js @@ -0,0 +1,191 @@ +import { logger } from '../../logging-config.js'; +export class ParseFailedGenerationError extends Error { + constructor(message) { + super(message); + this.name = 'ParseFailedGenerationError'; + } +} +/** + * Extract JSON from model output, handling both plain JSON and code-block-wrapped JSON. + * This is used to parse Groq's failed_generation field when an API error occurs. + * + * @param error - The Groq API error containing failed_generation + * @param outputFormat - An object with a parse method (typically a Zod schema) + * @returns The parsed output in the expected format + * @throws ParseFailedGenerationError if the failed_generation field is missing + * @throws Error if JSON parsing fails + */ +export function tryParseGroqFailedGeneration(error, outputFormat) { + try { + const failedGeneration = error.body?.error?.failed_generation; + if (!failedGeneration) { + throw new Error('No failed_generation field in error body'); + } + let content = failedGeneration; + // If content is wrapped in code blocks, extract just the JSON part + if (content.includes('```')) { + // Find the JSON content between code blocks + const parts = content.split('```'); + content = parts[1] || content; + // Remove language identifier if present (e.g., 'json\n') + if (content.includes('\n')) { + const [first, ...rest] = content.split('\n'); + // Check if first line is just a language identifier + if (first.trim().length < 20) { + content = rest.join('\n'); + } + } + } + // Remove html-like tags before the first { and after the last } + // This handles cases like <|header_start|>assistant<|header_end|> and + // Only remove content before { if content doesn't already start with { + if (!content.trim().startsWith('{')) { + content = content.replace(/^.*?(?=\{)/s, ''); + } + // Remove common HTML-like tags and patterns at the end, but be more conservative + // Look for patterns like , <|header_start|>, etc. after the JSON + content = content.replace(/\}(\s*<[^>]*>.*?$)/s, '}'); + content = content.replace(/\}(\s*<\|[^|]*\|>.*?$)/s, '}'); + // Handle extra characters after the JSON, including stray braces + // Find the position of the last } that would close the main JSON object + content = content.trim(); + if (content.endsWith('}')) { + // Try to parse and see if we get valid JSON + try { + JSON.parse(content); + } + catch { + // If parsing fails, try to find the correct end of the JSON + // by counting braces and removing anything after the balanced JSON + let braceCount = 0; + let lastValidPos = -1; + for (let i = 0; i < content.length; i++) { + const char = content[i]; + if (char === '{') { + braceCount++; + } + else if (char === '}') { + braceCount--; + if (braceCount === 0) { + lastValidPos = i + 1; + break; + } + } + } + if (lastValidPos > 0) { + content = content.substring(0, lastValidPos); + } + } + } + // Fix control characters in JSON strings before parsing + // This handles cases where literal control characters appear in JSON values + content = fixControlCharactersInJson(content); + // Parse the cleaned content + let resultDict = JSON.parse(content); + // Some models occasionally respond with a list containing one dict + if (Array.isArray(resultDict) && + resultDict.length === 1 && + typeof resultDict[0] === 'object') { + resultDict = resultDict[0]; + } + logger.debug(`Successfully parsed model output: ${JSON.stringify(resultDict)}`); + return outputFormat.parse(JSON.stringify(resultDict)); + } + catch (err) { + if (err instanceof Error && + err.message.includes('No failed_generation field')) { + throw new ParseFailedGenerationError(err.message); + } + if (err instanceof SyntaxError) { + logger.warning(`Failed to parse model output: ${err.message}`); + throw new Error(`Could not parse response. ${err.message}`, { + cause: err, + }); + } + const errorMessage = error.message || String(error); + throw new ParseFailedGenerationError(errorMessage); + } +} +/** + * Fix control characters in JSON string values to make them valid JSON. + * This function escapes literal control characters (newlines, tabs, etc.) that + * appear inside JSON string values, while preserving the JSON structure. + * + * @param content - The JSON string to fix + * @returns The fixed JSON string + */ +export function fixControlCharactersInJson(content) { + try { + // First try to parse as-is to see if it's already valid + JSON.parse(content); + return content; + } + catch { + // Continue to fix the content + } + // More sophisticated approach: only escape control characters inside string values + // while preserving JSON structure formatting + const result = []; + let i = 0; + let inString = false; + let escaped = false; + while (i < content.length) { + const char = content[i]; + if (!inString) { + // Outside of string - check if we're entering a string + if (char === '"') { + inString = true; + } + result.push(char); + } + else { + // Inside string - handle escaping and control characters + if (escaped) { + // Previous character was backslash, so this character is escaped + result.push(char); + escaped = false; + } + else if (char === '\\') { + // This is an escape character + result.push(char); + escaped = true; + } + else if (char === '"') { + // End of string + result.push(char); + inString = false; + } + else if (char === '\n') { + // Literal newline inside string - escape it + result.push('\\n'); + } + else if (char === '\r') { + // Literal carriage return inside string - escape it + result.push('\\r'); + } + else if (char === '\t') { + // Literal tab inside string - escape it + result.push('\\t'); + } + else if (char === '\b') { + // Literal backspace inside string - escape it + result.push('\\b'); + } + else if (char === '\f') { + // Literal form feed inside string - escape it + result.push('\\f'); + } + else if (char.charCodeAt(0) < 32) { + // Other control characters inside string - convert to unicode escape + const code = char.charCodeAt(0); + result.push(`\\u${code.toString(16).padStart(4, '0')}`); + } + else { + // Normal character inside string + result.push(char); + } + } + i++; + } + return result.join(''); +} diff --git a/dist/llm/groq/serializer.d.ts b/dist/llm/groq/serializer.d.ts new file mode 100644 index 00000000..5182bf2a --- /dev/null +++ b/dist/llm/groq/serializer.d.ts @@ -0,0 +1,6 @@ +import type { ChatCompletionMessageParam } from 'groq-sdk/resources/chat/completions.mjs'; +import { type Message } from '../messages.js'; +export declare class GroqMessageSerializer { + serialize(messages: Message[]): ChatCompletionMessageParam[]; + private serializeMessage; +} diff --git a/dist/llm/groq/serializer.js b/dist/llm/groq/serializer.js new file mode 100644 index 00000000..797fd5d2 --- /dev/null +++ b/dist/llm/groq/serializer.js @@ -0,0 +1,56 @@ +import { AssistantMessage, ContentPartImageParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js'; +export class GroqMessageSerializer { + serialize(messages) { + return messages.map((message) => this.serializeMessage(message)); + } + serializeMessage(message) { + if (message instanceof UserMessage) { + return { + role: 'user', + content: Array.isArray(message.content) + ? message.content.map((part) => { + if (part instanceof ContentPartTextParam) { + return { type: 'text', text: part.text }; + } + if (part instanceof ContentPartImageParam) { + return { + type: 'image_url', + image_url: { + url: part.image_url.url, + detail: part.image_url.detail, + }, + }; + } + return { type: 'text', text: '' }; + }) + : message.content, + name: message.name || undefined, + }; + } + if (message instanceof SystemMessage) { + return { + role: 'system', + content: Array.isArray(message.content) + ? message.content.map((part) => part.text).join('\n') + : message.content, + name: message.name || undefined, + }; + } + if (message instanceof AssistantMessage) { + const toolCalls = message.tool_calls?.map((toolCall) => ({ + id: toolCall.id, + type: 'function', + function: { + name: toolCall.functionCall.name, + arguments: toolCall.functionCall.arguments, + }, + })); + return { + role: 'assistant', + content: typeof message.content === 'string' ? message.content : null, + tool_calls: toolCalls, + }; + } + throw new Error(`Unknown message type: ${message.constructor.name}`); + } +} diff --git a/dist/llm/litellm/chat.d.ts b/dist/llm/litellm/chat.d.ts new file mode 100644 index 00000000..a262cec7 --- /dev/null +++ b/dist/llm/litellm/chat.d.ts @@ -0,0 +1,11 @@ +import { ChatOpenAILike } from '../openai/like.js'; +import type { ChatOpenAIOptions } from '../openai/chat.js'; +export interface ChatLiteLLMOptions extends ChatOpenAIOptions { + model?: string; + apiKey?: string; + baseURL?: string; +} +export declare class ChatLiteLLM extends ChatOpenAILike { + provider: string; + constructor(options?: string | ChatLiteLLMOptions); +} diff --git a/dist/llm/litellm/chat.js b/dist/llm/litellm/chat.js new file mode 100644 index 00000000..f7723697 --- /dev/null +++ b/dist/llm/litellm/chat.js @@ -0,0 +1,16 @@ +import { ChatOpenAILike } from '../openai/like.js'; +export class ChatLiteLLM extends ChatOpenAILike { + provider = 'litellm'; + constructor(options = {}) { + const normalizedOptions = typeof options === 'string' ? { model: options } : options; + super({ + ...normalizedOptions, + model: normalizedOptions.model ?? 'gpt-4o-mini', + apiKey: normalizedOptions.apiKey ?? process.env.LITELLM_API_KEY, + baseURL: normalizedOptions.baseURL ?? + process.env.LITELLM_API_BASE ?? + process.env.LITELLM_BASE_URL ?? + 'http://localhost:4000', + }); + } +} diff --git a/dist/llm/litellm/index.d.ts b/dist/llm/litellm/index.d.ts new file mode 100644 index 00000000..69a16816 --- /dev/null +++ b/dist/llm/litellm/index.d.ts @@ -0,0 +1 @@ +export * from './chat.js'; diff --git a/dist/llm/litellm/index.js b/dist/llm/litellm/index.js new file mode 100644 index 00000000..69a16816 --- /dev/null +++ b/dist/llm/litellm/index.js @@ -0,0 +1 @@ +export * from './chat.js'; diff --git a/dist/llm/messages.d.ts b/dist/llm/messages.d.ts new file mode 100644 index 00000000..19e16e4a --- /dev/null +++ b/dist/llm/messages.d.ts @@ -0,0 +1,77 @@ +export type SupportedImageMediaType = 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp'; +export declare class ContentPartTextParam { + text: string; + readonly type: "text"; + constructor(text: string); + toString(): string; +} +export declare class ContentPartRefusalParam { + refusal: string; + readonly type: "refusal"; + constructor(refusal: string); + toString(): string; +} +export declare class ImageURL { + url: string; + detail: 'auto' | 'low' | 'high'; + media_type: SupportedImageMediaType; + constructor(url: string, detail?: 'auto' | 'low' | 'high', media?: SupportedImageMediaType); + toString(): string; +} +export declare class ContentPartImageParam { + image_url: ImageURL; + readonly type: "image_url"; + constructor(image_url: ImageURL); + toString(): string; +} +export declare class FunctionCall { + name: string; + arguments: string; + constructor(name: string, args: string); + toString(): string; +} +export declare class ToolCall { + id: string; + functionCall: FunctionCall; + readonly type: "function"; + constructor(id: string, functionCall: FunctionCall); + toString(): string; +} +type ContentPart = ContentPartTextParam | ContentPartImageParam | ContentPartRefusalParam; +export type MessageRole = 'user' | 'system' | 'assistant'; +export declare abstract class MessageBase { + cache: boolean; + abstract role: MessageRole; + constructor(init?: Partial); +} +export declare class UserMessage extends MessageBase { + role: MessageRole; + content: string | ContentPart[]; + name: string | null; + constructor(content: string | ContentPart[], name?: string | null); + get text(): string; + toString(): string; +} +export declare class SystemMessage extends MessageBase { + role: MessageRole; + content: string | ContentPartTextParam[]; + name: string | null; + constructor(content: string | ContentPartTextParam[], name?: string | null); + get text(): string; + toString(): string; +} +export declare class AssistantMessage extends MessageBase { + role: MessageRole; + content: string | ContentPart[] | null; + tool_calls: ToolCall[] | null; + refusal: string | null; + constructor(init: { + content?: string | ContentPart[] | null; + tool_calls?: ToolCall[] | null; + refusal?: string | null; + }); + get text(): string; + toString(): string; +} +export type Message = UserMessage | SystemMessage | AssistantMessage; +export {}; diff --git a/dist/llm/messages.js b/dist/llm/messages.js new file mode 100644 index 00000000..fbec62f2 --- /dev/null +++ b/dist/llm/messages.js @@ -0,0 +1,157 @@ +const truncate = (text, maxLength = 50) => { + if (text.length <= maxLength) { + return text; + } + return `${text.slice(0, maxLength - 3)}...`; +}; +const formatImageUrl = (url, maxLength = 50) => { + if (url.startsWith('data:')) { + const mediaType = url.split(';')[0]?.split(':')[1] ?? 'image'; + return ``; + } + return truncate(url, maxLength); +}; +export class ContentPartTextParam { + text; + type = 'text'; + constructor(text) { + this.text = text; + } + toString() { + return `Text: ${truncate(this.text)}`; + } +} +export class ContentPartRefusalParam { + refusal; + type = 'refusal'; + constructor(refusal) { + this.refusal = refusal; + } + toString() { + return `Refusal: ${truncate(this.refusal)}`; + } +} +export class ImageURL { + url; + detail; + media_type; + constructor(url, detail = 'auto', media = 'image/png') { + this.url = url; + this.detail = detail; + this.media_type = media; + } + toString() { + return `🖼️ Image[${this.media_type}, detail=${this.detail}]: ${formatImageUrl(this.url)}`; + } +} +export class ContentPartImageParam { + image_url; + type = 'image_url'; + constructor(image_url) { + this.image_url = image_url; + } + toString() { + return this.image_url.toString(); + } +} +export class FunctionCall { + name; + // @ts-ignore - 'arguments' is a reserved keyword but valid as property name + arguments; + constructor(name, args) { + this.name = name; + // @ts-ignore + this.arguments = args; + } + toString() { + return `${this.name}(${truncate(this.arguments, 80)})`; + } +} +export class ToolCall { + id; + functionCall; + type = 'function'; + constructor(id, functionCall) { + this.id = id; + this.functionCall = functionCall; + } + toString() { + return `ToolCall[${this.id}]: ${this.functionCall.toString()}`; + } +} +export class MessageBase { + cache = false; + constructor(init) { + if (init?.cache !== undefined) { + this.cache = init.cache; + } + } +} +export class UserMessage extends MessageBase { + role = 'user'; + content; + name; + constructor(content, name = null) { + super(); + this.content = content; + this.name = name; + } + get text() { + if (typeof this.content === 'string') { + return this.content; + } + return this.content + .filter((part) => part instanceof ContentPartTextParam) + .map((part) => part.text) + .join('\n'); + } + toString() { + return `UserMessage(content=${this.text})`; + } +} +export class SystemMessage extends MessageBase { + role = 'system'; + content; + name; + constructor(content, name = null) { + super(); + this.content = content; + this.name = name; + } + get text() { + if (typeof this.content === 'string') { + return this.content; + } + return this.content.map((part) => part.text).join('\n'); + } + toString() { + return `SystemMessage(content=${this.text})`; + } +} +export class AssistantMessage extends MessageBase { + role = 'assistant'; + content; + tool_calls; + refusal; + constructor(init) { + super(); + this.content = init.content ?? null; + this.tool_calls = init.tool_calls ?? null; + this.refusal = init.refusal ?? null; + } + get text() { + if (typeof this.content === 'string') { + return this.content; + } + if (Array.isArray(this.content)) { + return this.content + .filter((part) => part instanceof ContentPartTextParam) + .map((part) => part.text) + .join('\n'); + } + return ''; + } + toString() { + return `AssistantMessage(content=${this.text})`; + } +} diff --git a/dist/llm/mistral/chat.d.ts b/dist/llm/mistral/chat.d.ts new file mode 100644 index 00000000..7a5aa7a0 --- /dev/null +++ b/dist/llm/mistral/chat.d.ts @@ -0,0 +1,43 @@ +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import type { Message } from '../messages.js'; +import { ChatInvokeCompletion } from '../views.js'; +export interface ChatMistralOptions { + model?: string; + apiKey?: string; + baseURL?: string; + timeout?: number | null; + defaultHeaders?: Record | null; + defaultQuery?: Record | null; + fetchImplementation?: typeof fetch; + fetchOptions?: RequestInit | null; + clientParams?: Record | null; + temperature?: number | null; + maxTokens?: number | null; + topP?: number | null; + seed?: number | null; + safePrompt?: boolean; + maxRetries?: number; + removeMinItemsFromSchema?: boolean; + removeDefaultsFromSchema?: boolean; +} +export declare class ChatMistral implements BaseChatModel { + model: string; + provider: string; + private client; + private temperature; + private maxTokens; + private topP; + private seed; + private safePrompt; + private removeMinItemsFromSchema; + private removeDefaultsFromSchema; + constructor(options?: string | ChatMistralOptions); + get name(): string; + get model_name(): string; + private getUsage; + private getSchemaCandidate; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/mistral/chat.js b/dist/llm/mistral/chat.js new file mode 100644 index 00000000..6614c484 --- /dev/null +++ b/dist/llm/mistral/chat.js @@ -0,0 +1,154 @@ +import OpenAI from 'openai'; +import { ModelProviderError, ModelRateLimitError } from '../exceptions.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { zodSchemaToJsonSchema } from '../schema.js'; +import { OpenAIMessageSerializer } from '../openai/serializer.js'; +import { MistralSchemaOptimizer } from './schema.js'; +export class ChatMistral { + model; + provider = 'mistral'; + client; + temperature; + maxTokens; + topP; + seed; + safePrompt; + removeMinItemsFromSchema; + removeDefaultsFromSchema; + constructor(options = {}) { + const normalizedOptions = typeof options === 'string' ? { model: options } : options; + const { model = 'mistral-medium-latest', apiKey = process.env.MISTRAL_API_KEY, baseURL = process.env.MISTRAL_BASE_URL || 'https://api.mistral.ai/v1', timeout = null, defaultHeaders = null, defaultQuery = null, fetchImplementation, fetchOptions = null, clientParams = null, temperature = 0.2, maxTokens = 4096, topP = null, seed = null, safePrompt = false, maxRetries = 5, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions; + this.model = model; + this.temperature = temperature; + this.maxTokens = maxTokens; + this.topP = topP; + this.seed = seed; + this.safePrompt = safePrompt; + this.removeMinItemsFromSchema = removeMinItemsFromSchema; + this.removeDefaultsFromSchema = removeDefaultsFromSchema; + this.client = new OpenAI({ + apiKey, + baseURL, + ...(timeout !== null ? { timeout } : {}), + maxRetries, + defaultHeaders: defaultHeaders ?? undefined, + defaultQuery: defaultQuery ?? undefined, + fetch: fetchImplementation, + fetchOptions: (fetchOptions ?? undefined), + ...(clientParams ?? {}), + }); + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + getUsage(response) { + if (!response.usage) { + return null; + } + return { + prompt_tokens: response.usage.prompt_tokens, + prompt_cached_tokens: response.usage.prompt_tokens_details?.cached_tokens ?? null, + prompt_cache_creation_tokens: null, + prompt_image_tokens: null, + completion_tokens: response.usage.completion_tokens, + total_tokens: response.usage.total_tokens, + }; + } + getSchemaCandidate(output_format) { + const output = output_format; + if (output && + typeof output === 'object' && + typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return output; + } + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return output.schema; + } + return null; + } + async ainvoke(messages, output_format, options = {}) { + const serializer = new OpenAIMessageSerializer(); + const mistralMessages = serializer.serialize(messages); + const modelParams = {}; + if (this.temperature !== null) { + modelParams.temperature = this.temperature; + } + if (this.maxTokens !== null) { + modelParams.max_tokens = this.maxTokens; + } + if (this.topP !== null) { + modelParams.top_p = this.topP; + } + if (this.seed !== null) { + modelParams.seed = this.seed; + } + if (this.safePrompt) { + modelParams.safe_prompt = true; + } + const zodSchemaCandidate = this.getSchemaCandidate(output_format); + let responseFormat = undefined; + if (zodSchemaCandidate) { + try { + const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, { + name: 'agent_output', + target: 'jsonSchema7', + }); + const optimizedJsonSchema = MistralSchemaOptimizer.createMistralCompatibleSchema(rawJsonSchema, { + removeMinItems: this.removeMinItemsFromSchema, + removeDefaults: this.removeDefaultsFromSchema, + }); + responseFormat = { + type: 'json_schema', + json_schema: { + name: 'agent_output', + schema: optimizedJsonSchema, + strict: true, + }, + }; + } + catch { + responseFormat = undefined; + } + } + try { + const response = await this.client.chat.completions.create({ + model: this.model, + messages: mistralMessages, + response_format: responseFormat, + ...modelParams, + }, options.signal ? { signal: options.signal } : undefined); + const content = response.choices[0].message.content || ''; + const usage = this.getUsage(response); + const stopReason = response.choices[0].finish_reason ?? null; + let completion = content; + if (output_format) { + const parsedJson = JSON.parse(content); + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + completion = output.schema.parse(parsedJson); + } + else { + completion = output.parse(parsedJson); + } + } + return new ChatInvokeCompletion(completion, usage, null, null, stopReason); + } + catch (error) { + if (error?.status === 429) { + throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model); + } + throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model); + } + } +} diff --git a/dist/llm/mistral/index.d.ts b/dist/llm/mistral/index.d.ts new file mode 100644 index 00000000..5e3a6306 --- /dev/null +++ b/dist/llm/mistral/index.d.ts @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './schema.js'; diff --git a/dist/llm/mistral/index.js b/dist/llm/mistral/index.js new file mode 100644 index 00000000..5e3a6306 --- /dev/null +++ b/dist/llm/mistral/index.js @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './schema.js'; diff --git a/dist/llm/mistral/schema.d.ts b/dist/llm/mistral/schema.d.ts new file mode 100644 index 00000000..092abd4b --- /dev/null +++ b/dist/llm/mistral/schema.d.ts @@ -0,0 +1,8 @@ +export declare class MistralSchemaOptimizer { + static readonly UNSUPPORTED_KEYWORDS: Set; + static createMistralCompatibleSchema(rawSchema: Record, options?: { + removeMinItems?: boolean; + removeDefaults?: boolean; + }): Record; + static stripUnsupportedKeywords(value: unknown): any; +} diff --git a/dist/llm/mistral/schema.js b/dist/llm/mistral/schema.js new file mode 100644 index 00000000..652906a9 --- /dev/null +++ b/dist/llm/mistral/schema.js @@ -0,0 +1,27 @@ +import { SchemaOptimizer } from '../schema.js'; +export class MistralSchemaOptimizer { + static UNSUPPORTED_KEYWORDS = new Set([ + 'minLength', + 'maxLength', + 'pattern', + 'format', + ]); + static createMistralCompatibleSchema(rawSchema, options = {}) { + const baseSchema = SchemaOptimizer.createOptimizedJsonSchema(rawSchema, { + removeMinItems: options.removeMinItems ?? false, + removeDefaults: options.removeDefaults ?? false, + }); + return this.stripUnsupportedKeywords(baseSchema); + } + static stripUnsupportedKeywords(value) { + if (Array.isArray(value)) { + return value.map((item) => this.stripUnsupportedKeywords(item)); + } + if (!value || typeof value !== 'object') { + return value; + } + return Object.fromEntries(Object.entries(value) + .filter(([key]) => !this.UNSUPPORTED_KEYWORDS.has(key)) + .map(([key, item]) => [key, this.stripUnsupportedKeywords(item)])); + } +} diff --git a/dist/llm/models.d.ts b/dist/llm/models.d.ts new file mode 100644 index 00000000..bda447d4 --- /dev/null +++ b/dist/llm/models.d.ts @@ -0,0 +1,2 @@ +import type { BaseChatModel } from './base.js'; +export declare const getLlmByName: (modelName: string) => BaseChatModel; diff --git a/dist/llm/models.js b/dist/llm/models.js new file mode 100644 index 00000000..91ef347f --- /dev/null +++ b/dist/llm/models.js @@ -0,0 +1,343 @@ +import { ChatAnthropic } from './anthropic/chat.js'; +import { ChatBedrockConverse } from './aws/chat-bedrock.js'; +import { ChatAzure } from './azure/chat.js'; +import { ChatBrowserUse } from './browser-use/chat.js'; +import { ChatCerebras } from './cerebras/chat.js'; +import { ChatDeepSeek } from './deepseek/chat.js'; +import { ChatGoogle } from './google/chat.js'; +import { ChatGroq } from './groq/chat.js'; +import { ChatLiteLLM } from './litellm/chat.js'; +import { ChatMistral } from './mistral/chat.js'; +import { ChatOCIRaw } from './oci-raw/chat.js'; +import { ChatOllama } from './ollama/chat.js'; +import { ChatOpenAI } from './openai/chat.js'; +import { ChatOpenRouter } from './openrouter/chat.js'; +import { ChatVercel } from './vercel/chat.js'; +const AVAILABLE_PROVIDERS = [ + 'openai', + 'anthropic', + 'google', + 'deepseek', + 'groq', + 'openrouter', + 'azure', + 'ollama', + 'aws', + 'browser-use', + 'mistral', + 'cerebras', + 'vercel', + 'litellm', + 'oci', +]; +const MISTRAL_ALIAS_MAP = { + large: 'mistral-large-latest', + medium: 'mistral-medium-latest', + small: 'mistral-small-latest', + codestral: 'codestral-latest', + 'pixtral-large': 'pixtral-large-latest', + pixtral_large: 'pixtral-large-latest', +}; +const convertPythonModelPart = (modelPart) => { + if (modelPart.includes('gpt_4_1_mini')) { + return modelPart.replace('gpt_4_1_mini', 'gpt-4.1-mini'); + } + if (modelPart.includes('gpt_4o_mini')) { + return modelPart.replace('gpt_4o_mini', 'gpt-4o-mini'); + } + if (modelPart.includes('gpt_4o')) { + return modelPart.replace('gpt_4o', 'gpt-4o'); + } + if (modelPart.includes('gemini_2_0')) { + return modelPart.replace('gemini_2_0', 'gemini-2.0').replace(/_/g, '-'); + } + if (modelPart.includes('gemini_2_5')) { + return modelPart.replace('gemini_2_5', 'gemini-2.5').replace(/_/g, '-'); + } + if (modelPart.includes('llama3_1')) { + return modelPart.replace('llama3_1', 'llama3.1').replace(/_/g, '-'); + } + if (modelPart.includes('llama3_3')) { + return modelPart.replace('llama3_3', 'llama-3.3').replace(/_/g, '-'); + } + if (modelPart.includes('llama_4_scout')) { + return modelPart + .replace('llama_4_scout', 'llama-4-scout') + .replace(/_/g, '-'); + } + if (modelPart.includes('llama_4_maverick')) { + return modelPart + .replace('llama_4_maverick', 'llama-4-maverick') + .replace(/_/g, '-'); + } + if (modelPart.includes('gpt_oss_120b')) { + return modelPart.replace('gpt_oss_120b', 'gpt-oss-120b'); + } + if (modelPart.includes('qwen_3_32b')) { + return modelPart.replace('qwen_3_32b', 'qwen-3-32b'); + } + if (modelPart.includes('qwen_3_235b_a22b_instruct_2507')) { + return modelPart.replace('qwen_3_235b_a22b_instruct_2507', 'qwen-3-235b-a22b-instruct-2507'); + } + if (modelPart.includes('qwen_3_235b_a22b_instruct')) { + return modelPart.replace('qwen_3_235b_a22b_instruct', 'qwen-3-235b-a22b-instruct-2507'); + } + if (modelPart.includes('qwen_3_235b_a22b_thinking_2507')) { + return modelPart.replace('qwen_3_235b_a22b_thinking_2507', 'qwen-3-235b-a22b-thinking-2507'); + } + if (modelPart.includes('qwen_3_235b_a22b_thinking')) { + return modelPart.replace('qwen_3_235b_a22b_thinking', 'qwen-3-235b-a22b-thinking-2507'); + } + if (modelPart.includes('qwen_3_coder_480b')) { + return modelPart.replace('qwen_3_coder_480b', 'qwen-3-coder-480b'); + } + return modelPart.replace(/_/g, '-'); +}; +const normalizeMistralModel = (model) => { + const key = model.trim().toLowerCase(); + return MISTRAL_ALIAS_MAP[key] ?? model; +}; +const inferProviderFromModel = (model) => { + const lower = model.toLowerCase(); + if (lower.startsWith('gpt') || + lower.startsWith('o1') || + lower.startsWith('o3') || + lower.startsWith('o4') || + lower.startsWith('gpt-5')) { + return 'openai'; + } + if (lower.startsWith('claude')) { + return 'anthropic'; + } + if (lower.startsWith('gemini')) { + return 'google'; + } + if (lower.startsWith('deepseek')) { + return 'deepseek'; + } + if (lower.startsWith('groq:')) { + return 'groq'; + } + if (lower.startsWith('openrouter:')) { + return 'openrouter'; + } + if (lower.startsWith('azure:')) { + return 'azure'; + } + if (lower.startsWith('ollama:')) { + return 'ollama'; + } + if (lower.startsWith('mistral:')) { + return 'mistral'; + } + if (lower.startsWith('cerebras:')) { + return 'cerebras'; + } + if (lower.startsWith('vercel:')) { + return 'vercel'; + } + if (lower.startsWith('litellm:')) { + return 'litellm'; + } + if (lower.startsWith('oci:')) { + return 'oci'; + } + if (lower.startsWith('mistral-') || + lower.startsWith('codestral') || + lower.startsWith('pixtral')) { + return 'mistral'; + } + if (lower.startsWith('llama3.') || + lower.startsWith('llama-4-') || + lower.startsWith('gpt-oss-') || + lower.startsWith('qwen-3-')) { + return 'cerebras'; + } + if (lower.startsWith('bedrock:')) { + return 'aws'; + } + if (lower.startsWith('anthropic.')) { + return 'aws'; + } + if (lower.startsWith('bu-') || lower.startsWith('browser-use/')) { + return 'browser-use'; + } + if (lower.includes('/') && + !lower.startsWith('http://') && + !lower.startsWith('https://')) { + return 'openrouter'; + } + return null; +}; +const normalizeModelForProvider = (provider, model) => { + const lower = model.toLowerCase(); + if (provider === 'groq' && lower.startsWith('groq:')) { + return model.slice('groq:'.length); + } + if (provider === 'openrouter' && lower.startsWith('openrouter:')) { + return model.slice('openrouter:'.length); + } + if (provider === 'azure' && lower.startsWith('azure:')) { + return model.slice('azure:'.length); + } + if (provider === 'ollama' && lower.startsWith('ollama:')) { + return model.slice('ollama:'.length); + } + if (provider === 'mistral' && lower.startsWith('mistral:')) { + return model.slice('mistral:'.length); + } + if (provider === 'cerebras' && lower.startsWith('cerebras:')) { + return model.slice('cerebras:'.length); + } + if (provider === 'vercel' && lower.startsWith('vercel:')) { + return model.slice('vercel:'.length); + } + if (provider === 'litellm' && lower.startsWith('litellm:')) { + return model.slice('litellm:'.length); + } + if (provider === 'oci' && lower.startsWith('oci:')) { + return model.slice('oci:'.length); + } + if (provider === 'aws' && lower.startsWith('bedrock:')) { + return model.slice('bedrock:'.length); + } + return model; +}; +const buildProviderModel = (provider, model) => { + switch (provider) { + case 'openai': + return new ChatOpenAI({ + model, + apiKey: process.env.OPENAI_API_KEY, + }); + case 'anthropic': + return new ChatAnthropic({ + model, + apiKey: process.env.ANTHROPIC_API_KEY, + }); + case 'google': + return new ChatGoogle({ + model, + apiKey: process.env.GOOGLE_API_KEY || '', + }); + case 'deepseek': + return new ChatDeepSeek({ + model, + apiKey: process.env.DEEPSEEK_API_KEY, + }); + case 'groq': + return new ChatGroq({ + model, + apiKey: process.env.GROQ_API_KEY, + }); + case 'openrouter': + return new ChatOpenRouter({ + model, + apiKey: process.env.OPENROUTER_API_KEY, + }); + case 'azure': + return new ChatAzure({ + model, + apiKey: process.env.AZURE_OPENAI_KEY ?? process.env.AZURE_OPENAI_API_KEY, + endpoint: process.env.AZURE_OPENAI_ENDPOINT, + }); + case 'ollama': + return new ChatOllama({ + model, + host: process.env.OLLAMA_HOST || 'http://localhost:11434', + }); + case 'mistral': + return new ChatMistral({ + model: normalizeMistralModel(model), + apiKey: process.env.MISTRAL_API_KEY, + baseURL: process.env.MISTRAL_BASE_URL, + }); + case 'cerebras': + return new ChatCerebras({ + model, + apiKey: process.env.CEREBRAS_API_KEY, + baseURL: process.env.CEREBRAS_BASE_URL, + }); + case 'vercel': + return new ChatVercel({ + model, + apiKey: process.env.VERCEL_API_KEY, + baseURL: process.env.VERCEL_BASE_URL, + }); + case 'litellm': + return new ChatLiteLLM({ + model, + apiKey: process.env.LITELLM_API_KEY, + baseURL: process.env.LITELLM_API_BASE ?? process.env.LITELLM_BASE_URL, + }); + case 'oci': + return new ChatOCIRaw({ + model, + }); + case 'aws': + return new ChatBedrockConverse({ + model, + region: process.env.AWS_REGION || 'us-east-1', + }); + case 'browser-use': + return new ChatBrowserUse({ + model, + apiKey: process.env.BROWSER_USE_API_KEY, + }); + default: + throw new Error(`Unknown provider '${provider}'. Available providers: ${AVAILABLE_PROVIDERS.join(', ')}`); + } +}; +export const getLlmByName = (modelName) => { + const normalizedName = modelName.trim(); + if (!normalizedName) { + throw new Error('Model name cannot be empty'); + } + if (normalizedName === 'bu_latest') { + return buildProviderModel('browser-use', 'bu-latest'); + } + if (normalizedName === 'bu_1_0') { + return buildProviderModel('browser-use', 'bu-1-0'); + } + if (normalizedName === 'bu_2_0') { + return buildProviderModel('browser-use', 'bu-2-0'); + } + if (normalizedName === 'mistral_large') { + return buildProviderModel('mistral', 'mistral-large-latest'); + } + if (normalizedName === 'mistral_medium') { + return buildProviderModel('mistral', 'mistral-medium-latest'); + } + if (normalizedName === 'mistral_small') { + return buildProviderModel('mistral', 'mistral-small-latest'); + } + if (normalizedName === 'codestral') { + return buildProviderModel('mistral', 'codestral-latest'); + } + if (normalizedName === 'pixtral_large') { + return buildProviderModel('mistral', 'pixtral-large-latest'); + } + const separator = normalizedName.indexOf('_'); + if (separator > 0) { + const provider = normalizedName.slice(0, separator); + const modelPart = normalizedName.slice(separator + 1); + if (provider === 'bu') { + return buildProviderModel('browser-use', `bu-${modelPart.replace(/_/g, '-')}`); + } + if (provider === 'oci') { + return buildProviderModel('oci', modelPart); + } + if (provider === 'browser-use') { + return buildProviderModel('browser-use', modelPart.replace(/_/g, '/')); + } + if (AVAILABLE_PROVIDERS.includes(provider)) { + return buildProviderModel(provider, convertPythonModelPart(modelPart)); + } + } + const inferredProvider = inferProviderFromModel(normalizedName); + if (!inferredProvider) { + throw new Error(`Invalid model name format: '${normalizedName}'. Expected provider-prefixed name like 'openai_gpt_4o' or a recognizable model prefix.`); + } + const normalizedModel = normalizeModelForProvider(inferredProvider, normalizedName); + return buildProviderModel(inferredProvider, normalizedModel); +}; diff --git a/dist/llm/oci-raw/chat.d.ts b/dist/llm/oci-raw/chat.d.ts new file mode 100644 index 00000000..9d10f3d4 --- /dev/null +++ b/dist/llm/oci-raw/chat.d.ts @@ -0,0 +1,64 @@ +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import type { Message } from '../messages.js'; +import { ChatInvokeCompletion } from '../views.js'; +export interface ChatOCIRawOptions { + model?: string; + modelId?: string; + serviceEndpoint?: string; + compartmentId?: string; + provider?: string; + temperature?: number | null; + maxTokens?: number | null; + frequencyPenalty?: number | null; + presencePenalty?: number | null; + topP?: number | null; + topK?: number | null; + authType?: 'API_KEY' | 'INSTANCE_PRINCIPAL' | 'RESOURCE_PRINCIPAL' | string; + authProfile?: string; + configFilePath?: string; + tenancyId?: string; + userId?: string; + fingerprint?: string; + privateKey?: string; + passphrase?: string | null; + responseSchemaName?: string; +} +export declare class ChatOCIRaw implements BaseChatModel { + model: string; + provider: string; + private readonly serviceEndpoint; + private readonly compartmentId; + private readonly ociProvider; + private readonly temperature; + private readonly maxTokens; + private readonly frequencyPenalty; + private readonly presencePenalty; + private readonly topP; + private readonly topK; + private readonly authType; + private readonly authProfile; + private readonly configFilePath?; + private readonly tenancyId?; + private readonly userId?; + private readonly fingerprint?; + private readonly privateKey?; + private readonly passphrase; + private readonly responseSchemaName; + private clientPromise; + constructor(options?: ChatOCIRawOptions); + get name(): string; + get model_name(): string; + private usesCohereFormat; + private createAuthProvider; + private getClient; + private getUsage; + private buildGenericRequest; + private buildCohereRequest; + private buildChatRequest; + private extractText; + private mapError; + ainvoke(messages: Message[], outputFormat?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], outputFormat: { + parse: (input: unknown) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/oci-raw/chat.js b/dist/llm/oci-raw/chat.js new file mode 100644 index 00000000..01a7e0e3 --- /dev/null +++ b/dist/llm/oci-raw/chat.js @@ -0,0 +1,350 @@ +import { ConfigFileAuthenticationDetailsProvider, InstancePrincipalsAuthenticationDetailsProviderBuilder, ResourcePrincipalAuthenticationDetailsProvider, SimpleAuthenticationDetailsProvider, } from 'oci-common'; +import { GenerativeAiInferenceClient } from 'oci-generativeaiinference'; +import { ModelProviderError, ModelRateLimitError } from '../exceptions.js'; +import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { OCIRawMessageSerializer } from './serializer.js'; +const parseStructuredJson = (text) => { + let jsonText = String(text ?? '').trim(); + if (jsonText.startsWith('```json') && jsonText.endsWith('```')) { + jsonText = jsonText.slice(7, -3).trim(); + } + else if (jsonText.startsWith('```') && jsonText.endsWith('```')) { + jsonText = jsonText.slice(3, -3).trim(); + } + return JSON.parse(jsonText); +}; +const extractZodSchema = (outputFormat) => { + const direct = outputFormat; + if (direct && + typeof direct === 'object' && + typeof direct.safeParse === 'function' && + typeof direct.parse === 'function') { + return direct; + } + const nested = outputFormat?.schema; + if (nested && + typeof nested === 'object' && + typeof nested.safeParse === 'function' && + typeof nested.parse === 'function') { + return nested; + } + return null; +}; +const parseOutput = (outputFormat, payload) => { + const maybeWrapped = outputFormat; + if (maybeWrapped && + typeof maybeWrapped === 'object' && + maybeWrapped.schema && + typeof maybeWrapped.schema.parse === 'function') { + return maybeWrapped.schema.parse(payload); + } + return outputFormat.parse(payload); +}; +const appendJsonInstruction = (messages, schema) => { + const instruction = '\n\nIMPORTANT: You must respond with ONLY a valid JSON object ' + + '(no markdown, no code blocks, no explanations)' + + (schema + ? ` that exactly matches this schema:\n${JSON.stringify(schema, null, 2)}` + : '.'); + const target = [...messages] + .reverse() + .find((message) => message.role === 'USER' || message.role === 'SYSTEM') ?? null; + if (target) { + const content = Array.isArray(target.content) + ? [...target.content] + : []; + content.push({ + type: 'TEXT', + text: instruction, + }); + target.content = content; + return messages; + } + return [ + { + role: 'SYSTEM', + content: [ + { + type: 'TEXT', + text: instruction, + }, + ], + }, + ...messages, + ]; +}; +export class ChatOCIRaw { + model; + provider = 'oci-raw'; + serviceEndpoint; + compartmentId; + ociProvider; + temperature; + maxTokens; + frequencyPenalty; + presencePenalty; + topP; + topK; + authType; + authProfile; + configFilePath; + tenancyId; + userId; + fingerprint; + privateKey; + passphrase; + responseSchemaName; + clientPromise = null; + constructor(options = {}) { + this.model = + options.model ?? + options.modelId ?? + process.env.OCI_MODEL_ID ?? + (() => { + throw new Error('ChatOCIRaw requires model or OCI_MODEL_ID'); + })(); + this.serviceEndpoint = + options.serviceEndpoint ?? + process.env.OCI_SERVICE_ENDPOINT ?? + (() => { + throw new Error('ChatOCIRaw requires serviceEndpoint or OCI_SERVICE_ENDPOINT'); + })(); + this.compartmentId = + options.compartmentId ?? + process.env.OCI_COMPARTMENT_ID ?? + (() => { + throw new Error('ChatOCIRaw requires compartmentId or OCI_COMPARTMENT_ID'); + })(); + this.ociProvider = options.provider ?? process.env.OCI_PROVIDER ?? 'meta'; + this.temperature = options.temperature ?? 1.0; + this.maxTokens = options.maxTokens ?? 600; + this.frequencyPenalty = options.frequencyPenalty ?? 0.0; + this.presencePenalty = options.presencePenalty ?? 0.0; + this.topP = options.topP ?? 0.75; + this.topK = options.topK ?? 0; + this.authType = options.authType ?? process.env.OCI_AUTH_TYPE ?? 'API_KEY'; + this.authProfile = + options.authProfile ?? + process.env.OCI_AUTH_PROFILE ?? + process.env.OCI_CONFIG_PROFILE ?? + 'DEFAULT'; + this.configFilePath = options.configFilePath ?? process.env.OCI_CONFIG_FILE; + this.tenancyId = options.tenancyId ?? process.env.OCI_TENANCY_ID; + this.userId = options.userId ?? process.env.OCI_USER_ID; + this.fingerprint = options.fingerprint ?? process.env.OCI_FINGERPRINT; + this.privateKey = options.privateKey ?? process.env.OCI_PRIVATE_KEY; + this.passphrase = + options.passphrase ?? process.env.OCI_PRIVATE_KEY_PASSPHRASE ?? null; + this.responseSchemaName = + options.responseSchemaName ?? 'browser_use_response'; + } + get name() { + if (this.model.length <= 90) { + return this.model; + } + const parts = this.model.split('.'); + if (parts.length >= 4) { + return `oci-${this.ociProvider}-${parts[3]}`; + } + return `oci-${this.ociProvider}-model`; + } + get model_name() { + return this.name; + } + usesCohereFormat() { + return this.ociProvider.toLowerCase() === 'cohere'; + } + async createAuthProvider() { + const authType = this.authType.toUpperCase(); + if (authType === 'INSTANCE_PRINCIPAL') { + return new InstancePrincipalsAuthenticationDetailsProviderBuilder().build(); + } + if (authType === 'RESOURCE_PRINCIPAL') { + return ResourcePrincipalAuthenticationDetailsProvider.builder(); + } + if (this.tenancyId && this.userId && this.fingerprint && this.privateKey) { + return new SimpleAuthenticationDetailsProvider(this.tenancyId, this.userId, this.fingerprint, this.privateKey, this.passphrase); + } + return new ConfigFileAuthenticationDetailsProvider(this.configFilePath, this.authProfile); + } + async getClient() { + if (!this.clientPromise) { + this.clientPromise = (async () => { + const authenticationDetailsProvider = await this.createAuthProvider(); + const client = new GenerativeAiInferenceClient({ + authenticationDetailsProvider, + }); + client.endpoint = this.serviceEndpoint; + return client; + })(); + } + return this.clientPromise; + } + getUsage(payload) { + const usage = payload?.usage; + if (!usage) { + return null; + } + const reasoningTokens = usage.completionTokensDetails?.reasoningTokens ?? 0; + const completionTokens = (usage.completionTokens ?? 0) + reasoningTokens; + return { + prompt_tokens: usage.promptTokens ?? 0, + prompt_cached_tokens: usage.promptTokensDetails?.cachedTokens ?? null, + prompt_cache_creation_tokens: null, + prompt_image_tokens: null, + completion_tokens: completionTokens, + total_tokens: usage.totalTokens ?? (usage.promptTokens ?? 0) + completionTokens, + }; + } + buildGenericRequest(messages, outputFormat) { + const zodSchema = extractZodSchema(outputFormat); + const serializedMessages = OCIRawMessageSerializer.serializeMessages(messages); + const optimizedSchema = zodSchema + ? SchemaOptimizer.createOptimizedJsonSchema(zodSchemaToJsonSchema(zodSchema)) + : null; + const requestMessages = outputFormat && !zodSchema + ? appendJsonInstruction(serializedMessages, null) + : serializedMessages; + const request = { + apiFormat: 'GENERIC', + messages: requestMessages, + }; + if (this.temperature !== null) { + request.temperature = this.temperature; + } + if (this.maxTokens !== null) { + request.maxTokens = this.maxTokens; + } + if (this.frequencyPenalty !== null) { + request.frequencyPenalty = this.frequencyPenalty; + } + if (this.presencePenalty !== null) { + request.presencePenalty = this.presencePenalty; + } + if (this.topP !== null) { + request.topP = this.topP; + } + if (this.topK !== null) { + request.topK = this.topK; + } + if (optimizedSchema) { + request.responseFormat = { + type: 'JSON_SCHEMA', + jsonSchema: { + name: this.responseSchemaName, + schema: optimizedSchema, + isStrict: true, + }, + }; + } + return request; + } + buildCohereRequest(messages, outputFormat) { + const zodSchema = extractZodSchema(outputFormat); + const optimizedSchema = zodSchema + ? SchemaOptimizer.createOptimizedJsonSchema(zodSchemaToJsonSchema(zodSchema)) + : null; + let conversation = OCIRawMessageSerializer.serializeMessagesForCohere(messages); + if (outputFormat) { + conversation += + '\n\nIMPORTANT: You must respond with ONLY a valid JSON object (no markdown, no code blocks, no explanations)' + + (optimizedSchema + ? ` that exactly matches this schema:\n${JSON.stringify(optimizedSchema, null, 2)}` + : '.'); + } + const request = { + apiFormat: 'COHERE', + message: conversation, + }; + if (this.temperature !== null) { + request.temperature = this.temperature; + } + if (this.maxTokens !== null) { + request.maxTokens = this.maxTokens; + } + if (this.frequencyPenalty !== null) { + request.frequencyPenalty = this.frequencyPenalty; + } + if (this.presencePenalty !== null) { + request.presencePenalty = this.presencePenalty; + } + if (this.topP !== null) { + request.topP = this.topP; + } + if (this.topK !== null) { + request.topK = this.topK; + } + return request; + } + buildChatRequest(messages, outputFormat) { + const chatRequest = this.usesCohereFormat() + ? this.buildCohereRequest(messages, outputFormat) + : this.buildGenericRequest(messages, outputFormat); + return { + chatDetails: { + compartmentId: this.compartmentId, + servingMode: { + servingType: 'ON_DEMAND', + modelId: this.model, + }, + chatRequest, + }, + }; + } + extractText(payload) { + if (!payload) { + throw new ModelProviderError('OCI response did not include chatResponse payload', 502, this.name); + } + if (payload.apiFormat === 'COHERE') { + const coherePayload = payload; + return { + text: coherePayload.text ?? '', + thinking: null, + stopReason: coherePayload.finishReason ?? null, + }; + } + const genericPayload = payload; + const choice = genericPayload.choices?.[0]; + const text = choice?.message?.content + ?.filter((part) => part.type === 'TEXT') + .map((part) => part.text ?? '') + .join('\n') + .trim(); + return { + text: text ?? '', + thinking: choice?.message?.reasoningContent ?? null, + stopReason: choice?.finishReason ?? null, + }; + } + mapError(error) { + const statusCode = typeof error?.statusCode === 'number' + ? error.statusCode + : 502; + const message = String(error?.message ?? error ?? 'OCI request failed'); + if (statusCode === 429) { + throw new ModelRateLimitError(message, statusCode, this.name); + } + throw new ModelProviderError(message, statusCode, this.name); + } + async ainvoke(messages, outputFormat, _options = {}) { + try { + const client = await this.getClient(); + const response = (await client.chat(this.buildChatRequest(messages, outputFormat))); + if (!response || typeof response !== 'object') { + throw new ModelProviderError('OCI chat request returned an empty response', 502, this.name); + } + const payload = response.chatResult?.chatResponse; + const usage = this.getUsage(payload); + const { text, thinking, stopReason } = this.extractText(payload); + if (!outputFormat) { + return new ChatInvokeCompletion(text, usage, thinking, null, stopReason); + } + const parsed = parseOutput(outputFormat, parseStructuredJson(text)); + return new ChatInvokeCompletion(parsed, usage, thinking, null, stopReason); + } + catch (error) { + this.mapError(error); + } + } +} diff --git a/dist/llm/oci-raw/index.d.ts b/dist/llm/oci-raw/index.d.ts new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/oci-raw/index.d.ts @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/oci-raw/index.js b/dist/llm/oci-raw/index.js new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/oci-raw/index.js @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/oci-raw/serializer.d.ts b/dist/llm/oci-raw/serializer.d.ts new file mode 100644 index 00000000..ba4c4056 --- /dev/null +++ b/dist/llm/oci-raw/serializer.d.ts @@ -0,0 +1,12 @@ +import { type Message } from '../messages.js'; +type OciChatContent = Record; +type OciMessage = { + role: string; + name?: string; + content: OciChatContent[]; +}; +export declare class OCIRawMessageSerializer { + static serializeMessages(messages: Message[]): OciMessage[]; + static serializeMessagesForCohere(messages: Message[]): string; +} +export {}; diff --git a/dist/llm/oci-raw/serializer.js b/dist/llm/oci-raw/serializer.js new file mode 100644 index 00000000..eb2f03e7 --- /dev/null +++ b/dist/llm/oci-raw/serializer.js @@ -0,0 +1,128 @@ +import { AssistantMessage, ContentPartImageParam, ContentPartRefusalParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js'; +const textContent = (text) => ({ + type: 'TEXT', + text, +}); +const imageContent = (url) => ({ + type: 'IMAGE', + imageUrl: { + url, + }, +}); +const contentPartsToOci = (content) => { + if (typeof content === 'string') { + return [textContent(content)]; + } + if (!Array.isArray(content)) { + return []; + } + const parts = []; + for (const part of content) { + if (part instanceof ContentPartTextParam) { + parts.push(textContent(part.text)); + continue; + } + if (part instanceof ContentPartImageParam) { + parts.push(imageContent(part.image_url.url)); + continue; + } + if (part instanceof ContentPartRefusalParam) { + parts.push(textContent(`[Refusal] ${part.refusal}`)); + } + } + return parts; +}; +const serializeRole = (message) => { + if (message instanceof SystemMessage) { + return 'SYSTEM'; + } + if (message instanceof AssistantMessage) { + return 'ASSISTANT'; + } + return 'USER'; +}; +const serializeName = (message) => { + if (message instanceof UserMessage || message instanceof SystemMessage) { + return message.name ?? undefined; + } + return undefined; +}; +export class OCIRawMessageSerializer { + static serializeMessages(messages) { + const serialized = []; + for (const message of messages) { + const content = message instanceof AssistantMessage + ? contentPartsToOci(message.content) + : contentPartsToOci(message.content); + if (content.length === 0) { + continue; + } + serialized.push({ + role: serializeRole(message), + name: serializeName(message), + content, + }); + } + return serialized; + } + static serializeMessagesForCohere(messages) { + const conversationParts = []; + for (const message of messages) { + let text = ''; + if (message instanceof UserMessage || message instanceof SystemMessage) { + const content = message.content; + if (typeof content === 'string') { + text = content; + } + else { + text = content + .map((part) => { + if (part instanceof ContentPartTextParam) { + return part.text; + } + if (part instanceof ContentPartImageParam) { + return part.image_url.url.startsWith('data:image/') + ? '[Image: base64_data]' + : '[Image: external_url]'; + } + return ''; + }) + .filter(Boolean) + .join(' '); + } + } + else if (message instanceof AssistantMessage) { + if (typeof message.content === 'string') { + text = message.content; + } + else if (Array.isArray(message.content)) { + text = message.content + .map((part) => { + if (part instanceof ContentPartTextParam) { + return part.text; + } + if (part instanceof ContentPartRefusalParam) { + return `[Refusal] ${part.refusal}`; + } + return ''; + }) + .filter(Boolean) + .join(' '); + } + else if (message.refusal) { + text = `[Refusal] ${message.refusal}`; + } + } + if (!text) { + continue; + } + const prefix = message instanceof SystemMessage + ? 'System' + : message instanceof AssistantMessage + ? 'Assistant' + : 'User'; + conversationParts.push(`${prefix}: ${text}`); + } + return conversationParts.join('\n\n'); + } +} diff --git a/dist/llm/ollama/chat.d.ts b/dist/llm/ollama/chat.d.ts new file mode 100644 index 00000000..410d7145 --- /dev/null +++ b/dist/llm/ollama/chat.d.ts @@ -0,0 +1,27 @@ +import { type Config as OllamaClientConfig, type Options as OllamaOptions } from 'ollama'; +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import { ChatInvokeCompletion } from '../views.js'; +import type { Message } from '../messages.js'; +export interface ChatOllamaOptions { + model?: string; + host?: string; + timeout?: number | null; + clientParams?: Partial | null; + ollamaOptions?: Partial | null; +} +export declare class ChatOllama implements BaseChatModel { + model: string; + provider: string; + private client; + private ollamaOptions; + constructor(modelOrOptions?: string | ChatOllamaOptions, host?: string); + get name(): string; + get model_name(): string; + private getZodSchemaCandidate; + private parseOutput; + private createTimeoutFetch; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/ollama/chat.js b/dist/llm/ollama/chat.js new file mode 100644 index 00000000..01523e04 --- /dev/null +++ b/dist/llm/ollama/chat.js @@ -0,0 +1,168 @@ +import { Ollama, } from 'ollama'; +import { ModelProviderError } from '../exceptions.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { zodSchemaToJsonSchema } from '../schema.js'; +import { OllamaMessageSerializer } from './serializer.js'; +export class ChatOllama { + model; + provider = 'ollama'; + client; + ollamaOptions; + constructor(modelOrOptions = 'qwen2.5:latest', host = 'http://localhost:11434') { + const normalizedOptions = typeof modelOrOptions === 'string' + ? { model: modelOrOptions, host } + : modelOrOptions; + const { model = 'qwen2.5:latest', host: ollamaHost = 'http://localhost:11434', timeout = null, clientParams = null, ollamaOptions = null, } = normalizedOptions; + this.model = model; + this.ollamaOptions = ollamaOptions; + const baseFetch = clientParams?.fetch; + let fetchWithTimeout = baseFetch; + if (timeout !== null && timeout !== undefined) { + const timeoutMs = Number(timeout); + if (Number.isFinite(timeoutMs) && timeoutMs > 0) { + fetchWithTimeout = this.createTimeoutFetch(baseFetch, timeoutMs); + } + } + this.client = new Ollama({ + host: ollamaHost, + ...(clientParams ?? {}), + ...(fetchWithTimeout ? { fetch: fetchWithTimeout } : {}), + }); + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + getZodSchemaCandidate(output_format) { + const output = output_format; + if (output && + typeof output === 'object' && + typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return output; + } + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return output.schema; + } + return null; + } + parseOutput(output_format, payload) { + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + return output.schema.parse(payload); + } + return output.parse(payload); + } + createTimeoutFetch(baseFetch, timeoutMs) { + return async (input, init) => { + const fetchImpl = baseFetch ?? fetch; + const timeoutController = new AbortController(); + const timeoutHandle = setTimeout(() => timeoutController.abort(), timeoutMs); + const externalSignal = init?.signal; + const onAbort = () => timeoutController.abort(); + try { + if (externalSignal) { + if (externalSignal.aborted) { + timeoutController.abort(); + } + else { + externalSignal.addEventListener('abort', onAbort, { once: true }); + } + } + return await fetchImpl(input, { + ...init, + signal: timeoutController.signal, + }); + } + finally { + clearTimeout(timeoutHandle); + externalSignal?.removeEventListener('abort', onAbort); + } + }; + } + async ainvoke(messages, output_format, options = {}) { + const serializer = new OllamaMessageSerializer(); + const ollamaMessages = serializer.serialize(messages); + const zodSchemaCandidate = this.getZodSchemaCandidate(output_format); + let format = undefined; + if (zodSchemaCandidate) { + format = zodSchemaToJsonSchema(zodSchemaCandidate, { + name: 'Response', + target: 'jsonSchema7', + }); + } + else if (output_format) { + format = 'json'; + } + const requestPromise = this.client.chat({ + model: this.model, + messages: ollamaMessages, + format: format, + options: this.ollamaOptions ?? undefined, + stream: false, + }); + const abortSignal = options.signal; + const response = abortSignal + ? await new Promise((resolve, reject) => { + const onAbort = () => { + cleanup(); + const error = new Error('Operation aborted'); + error.name = 'AbortError'; + reject(error); + }; + const cleanup = () => { + abortSignal.removeEventListener('abort', onAbort); + }; + if (abortSignal.aborted) { + onAbort(); + return; + } + abortSignal.addEventListener('abort', onAbort, { once: true }); + requestPromise + .then((result) => { + cleanup(); + resolve(result); + }) + .catch((error) => { + cleanup(); + reject(error); + }); + }) + : await requestPromise; + try { + const content = response.message.content; + let completion = content; + if (output_format) { + if (zodSchemaCandidate) { + completion = this.parseOutput(output_format, JSON.parse(content)); + } + else { + try { + completion = this.parseOutput(output_format, JSON.parse(content)); + } + catch { + completion = this.parseOutput(output_format, content); + } + } + } + const stopReason = response.done_reason ?? null; + return new ChatInvokeCompletion(completion, { + prompt_tokens: response.prompt_eval_count ?? 0, + completion_tokens: response.eval_count ?? 0, + total_tokens: (response.prompt_eval_count ?? 0) + (response.eval_count ?? 0), + }, null, null, stopReason); + } + catch (error) { + throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 502, this.model); + } + } +} diff --git a/dist/llm/ollama/index.d.ts b/dist/llm/ollama/index.d.ts new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/ollama/index.d.ts @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/ollama/index.js b/dist/llm/ollama/index.js new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/ollama/index.js @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/ollama/serializer.d.ts b/dist/llm/ollama/serializer.d.ts new file mode 100644 index 00000000..272fceb2 --- /dev/null +++ b/dist/llm/ollama/serializer.d.ts @@ -0,0 +1,7 @@ +import type { Message as OllamaMessage } from 'ollama'; +import { type Message } from '../messages.js'; +export declare class OllamaMessageSerializer { + serialize(messages: Message[]): OllamaMessage[]; + private extractTextContent; + private serializeMessage; +} diff --git a/dist/llm/ollama/serializer.js b/dist/llm/ollama/serializer.js new file mode 100644 index 00000000..0d1ffd86 --- /dev/null +++ b/dist/llm/ollama/serializer.js @@ -0,0 +1,75 @@ +import { AssistantMessage, ContentPartImageParam, ContentPartRefusalParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js'; +export class OllamaMessageSerializer { + serialize(messages) { + return messages.map((message) => this.serializeMessage(message)); + } + extractTextContent(content) { + if (content === null || content === undefined) { + return ''; + } + if (typeof content === 'string') { + return content; + } + if (!Array.isArray(content)) { + return ''; + } + const parts = []; + for (const part of content) { + if (part instanceof ContentPartTextParam) { + parts.push(part.text); + } + else if (part instanceof ContentPartRefusalParam) { + parts.push(`[Refusal] ${part.refusal}`); + } + } + return parts.join('\n'); + } + serializeMessage(message) { + if (message instanceof UserMessage) { + const images = []; + const content = this.extractTextContent(message.content); + if (Array.isArray(message.content)) { + message.content.forEach((part) => { + if (part instanceof ContentPartImageParam) { + // Ollama expects base64 string without header + const data = part.image_url.url.split(',')[1]; + if (data) + images.push(data); + } + }); + } + return { + role: 'user', + content: content, + images: images.length > 0 ? images : undefined, + }; + } + if (message instanceof SystemMessage) { + return { + role: 'system', + content: this.extractTextContent(message.content), + }; + } + if (message instanceof AssistantMessage) { + const toolCalls = message.tool_calls?.map((toolCall) => ({ + function: { + name: toolCall.functionCall.name, + arguments: (() => { + try { + return JSON.parse(toolCall.functionCall.arguments); + } + catch { + return { arguments: toolCall.functionCall.arguments }; + } + })(), + }, + })); + return { + role: 'assistant', + content: this.extractTextContent(message.content), + tool_calls: toolCalls, + }; + } + throw new Error(`Unknown message type: ${message.constructor.name}`); + } +} diff --git a/dist/llm/openai/chat.d.ts b/dist/llm/openai/chat.d.ts new file mode 100644 index 00000000..caa6a9e1 --- /dev/null +++ b/dist/llm/openai/chat.d.ts @@ -0,0 +1,54 @@ +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import { ChatInvokeCompletion } from '../views.js'; +import type { Message } from '../messages.js'; +export interface ChatOpenAIOptions { + model?: string; + apiKey?: string; + organization?: string; + project?: string; + baseURL?: string; + timeout?: number | null; + temperature?: number | null; + frequencyPenalty?: number | null; + reasoningEffort?: 'low' | 'medium' | 'high'; + serviceTier?: 'auto' | 'default' | 'flex' | 'priority' | 'scale' | null; + maxCompletionTokens?: number | null; + maxRetries?: number; + defaultHeaders?: Record | null; + defaultQuery?: Record | null; + fetchImplementation?: typeof fetch; + fetchOptions?: RequestInit | null; + seed?: number | null; + topP?: number | null; + addSchemaToSystemPrompt?: boolean; + dontForceStructuredOutput?: boolean; + removeMinItemsFromSchema?: boolean; + removeDefaultsFromSchema?: boolean; + reasoningModels?: string[] | null; +} +export declare class ChatOpenAI implements BaseChatModel { + model: string; + provider: string; + private client; + private temperature; + private frequencyPenalty; + private reasoningEffort; + private serviceTier; + private maxCompletionTokens; + private seed; + private topP; + private addSchemaToSystemPrompt; + private dontForceStructuredOutput; + private removeMinItemsFromSchema; + private removeDefaultsFromSchema; + private reasoningModels; + constructor(options?: ChatOpenAIOptions); + get name(): string; + get model_name(): string; + private isReasoningModel; + private getUsage; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/openai/chat.js b/dist/llm/openai/chat.js new file mode 100644 index 00000000..292bde0d --- /dev/null +++ b/dist/llm/openai/chat.js @@ -0,0 +1,224 @@ +import OpenAI from 'openai'; +import { ChatInvokeCompletion } from '../views.js'; +import { OpenAIMessageSerializer } from './serializer.js'; +import { ModelProviderError, ModelRateLimitError } from '../exceptions.js'; +import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js'; +// Reasoning models that support reasoning_effort parameter +const DEFAULT_REASONING_MODELS = [ + 'o4-mini', + 'o3', + 'o3-mini', + 'o1', + 'o1-pro', + 'o3-pro', + 'gpt-5', + 'gpt-5-mini', + 'gpt-5-nano', +]; +export class ChatOpenAI { + model; + provider = 'openai'; + client; + temperature; + frequencyPenalty; + reasoningEffort; + serviceTier; + maxCompletionTokens; + seed; + topP; + addSchemaToSystemPrompt; + dontForceStructuredOutput; + removeMinItemsFromSchema; + removeDefaultsFromSchema; + reasoningModels; + constructor(options = {}) { + const { model = 'gpt-4o', apiKey, organization, project, baseURL, timeout = null, temperature = 0.2, frequencyPenalty = 0.3, reasoningEffort = 'low', serviceTier = null, maxCompletionTokens = 4096, maxRetries = 5, defaultHeaders = null, defaultQuery = null, fetchImplementation, fetchOptions = null, seed = null, topP = null, addSchemaToSystemPrompt = false, dontForceStructuredOutput = false, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, reasoningModels = DEFAULT_REASONING_MODELS, } = options; + this.model = model; + this.temperature = temperature; + this.frequencyPenalty = frequencyPenalty; + this.reasoningEffort = reasoningEffort; + this.serviceTier = serviceTier; + this.maxCompletionTokens = maxCompletionTokens; + this.seed = seed; + this.topP = topP; + this.addSchemaToSystemPrompt = addSchemaToSystemPrompt; + this.dontForceStructuredOutput = dontForceStructuredOutput; + this.removeMinItemsFromSchema = removeMinItemsFromSchema; + this.removeDefaultsFromSchema = removeDefaultsFromSchema; + this.reasoningModels = reasoningModels + ? [...reasoningModels] + : reasoningModels; + this.client = new OpenAI({ + apiKey, + organization, + project, + baseURL, + timeout: timeout ?? undefined, + maxRetries, + defaultHeaders: defaultHeaders ?? undefined, + defaultQuery: defaultQuery ?? undefined, + fetch: fetchImplementation, + fetchOptions: (fetchOptions ?? undefined), + }); + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + isReasoningModel() { + return (this.reasoningModels ?? []).some((m) => this.model.toLowerCase().includes(m.toLowerCase())); + } + getUsage(response) { + if (!response.usage) + return null; + let completionTokens = response.usage.completion_tokens; + const details = response.usage.completion_tokens_details; + if (details?.reasoning_tokens) { + completionTokens += details.reasoning_tokens; + } + return { + prompt_tokens: response.usage.prompt_tokens, + prompt_cached_tokens: response.usage.prompt_tokens_details?.cached_tokens ?? null, + prompt_cache_creation_tokens: null, + prompt_image_tokens: null, + completion_tokens: completionTokens, + total_tokens: response.usage.total_tokens, + }; + } + async ainvoke(messages, output_format, options = {}) { + const serializer = new OpenAIMessageSerializer(); + const openaiMessages = serializer.serialize(messages); + // Build model parameters + const modelParams = {}; + if (!this.isReasoningModel()) { + // Regular models support temperature and frequency_penalty + if (this.temperature !== null) { + modelParams.temperature = this.temperature; + } + if (this.frequencyPenalty !== null) { + modelParams.frequency_penalty = this.frequencyPenalty; + } + } + else { + // Reasoning models use reasoning_effort instead + modelParams.reasoning_effort = this.reasoningEffort; + } + if (this.maxCompletionTokens !== null) { + modelParams.max_completion_tokens = this.maxCompletionTokens; + } + if (this.seed !== null) { + modelParams.seed = this.seed; + } + if (this.topP !== null) { + modelParams.top_p = this.topP; + } + if (this.serviceTier !== null) { + modelParams.service_tier = this.serviceTier; + } + const zodSchemaCandidate = (() => { + const output = output_format; + if (output && + typeof output === 'object' && + typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return output; + } + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return output.schema; + } + return null; + })(); + let responseFormat = undefined; + if (zodSchemaCandidate) { + try { + const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, { + name: 'agent_output', + target: 'jsonSchema7', + }); + const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, { + removeMinItems: this.removeMinItemsFromSchema, + removeDefaults: this.removeDefaultsFromSchema, + }); + const responseJsonSchema = { + name: 'agent_output', + schema: optimizedJsonSchema, + strict: true, + }; + if (this.addSchemaToSystemPrompt && openaiMessages.length > 0) { + const firstMessage = openaiMessages[0]; + const schemaText = `\n\n` + + `${JSON.stringify(responseJsonSchema, null, 2)}\n` + + ``; + if (firstMessage?.role === 'system') { + if (typeof firstMessage.content === 'string') { + firstMessage.content = + (firstMessage.content ?? '') + schemaText; + } + else if (Array.isArray(firstMessage.content)) { + firstMessage.content = [ + ...firstMessage.content, + { type: 'text', text: schemaText }, + ]; + } + } + } + if (!this.dontForceStructuredOutput) { + responseFormat = { + type: 'json_schema', + json_schema: responseJsonSchema, + }; + } + } + catch { + responseFormat = undefined; + } + } + try { + const response = await this.client.chat.completions.create({ + model: this.model, + messages: openaiMessages, + response_format: responseFormat, + ...modelParams, + }, options.signal ? { signal: options.signal } : undefined); + const content = response.choices[0].message.content || ''; + const usage = this.getUsage(response); + const stopReason = response.choices[0].finish_reason ?? null; + let completion = content; + if (output_format) { + if (zodSchemaCandidate) { + const parsedJson = JSON.parse(content); + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + completion = output.schema.parse(parsedJson); + } + else { + completion = output_format.parse(parsedJson); + } + } + else { + completion = output_format.parse(content); + } + } + return new ChatInvokeCompletion(completion, usage, null, null, stopReason); + } + catch (error) { + // Handle OpenAI-specific errors + if (error?.status === 429) { + throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model); + } + if (error?.status >= 500) { + throw new ModelProviderError(error?.message ?? 'Server error', error.status, this.model); + } + throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model); + } + } +} diff --git a/dist/llm/openai/index.d.ts b/dist/llm/openai/index.d.ts new file mode 100644 index 00000000..279f525a --- /dev/null +++ b/dist/llm/openai/index.d.ts @@ -0,0 +1,3 @@ +export * from './chat.js'; +export * from './serializer.js'; +export * from './like.js'; diff --git a/dist/llm/openai/index.js b/dist/llm/openai/index.js new file mode 100644 index 00000000..279f525a --- /dev/null +++ b/dist/llm/openai/index.js @@ -0,0 +1,3 @@ +export * from './chat.js'; +export * from './serializer.js'; +export * from './like.js'; diff --git a/dist/llm/openai/like.d.ts b/dist/llm/openai/like.d.ts new file mode 100644 index 00000000..a4cb5a56 --- /dev/null +++ b/dist/llm/openai/like.d.ts @@ -0,0 +1,19 @@ +import { ChatOpenAI, type ChatOpenAIOptions } from './chat.js'; +/** + * A class to interact with any provider using the OpenAI API schema. + * + * This allows using any OpenAI-compatible API provider by specifying a custom model name. + * + * @example + * ```typescript + * const llm = new ChatOpenAILike('custom-model-name'); + * ``` + */ +export declare class ChatOpenAILike extends ChatOpenAI { + /** + * @param options - A model name or ChatOpenAI-compatible options + */ + constructor(options: string | (ChatOpenAIOptions & { + model: string; + })); +} diff --git a/dist/llm/openai/like.js b/dist/llm/openai/like.js new file mode 100644 index 00000000..b50e4247 --- /dev/null +++ b/dist/llm/openai/like.js @@ -0,0 +1,23 @@ +import { ChatOpenAI } from './chat.js'; +/** + * A class to interact with any provider using the OpenAI API schema. + * + * This allows using any OpenAI-compatible API provider by specifying a custom model name. + * + * @example + * ```typescript + * const llm = new ChatOpenAILike('custom-model-name'); + * ``` + */ +export class ChatOpenAILike extends ChatOpenAI { + /** + * @param options - A model name or ChatOpenAI-compatible options + */ + constructor(options) { + if (typeof options === 'string') { + super({ model: options }); + return; + } + super(options); + } +} diff --git a/dist/llm/openai/responses-serializer.d.ts b/dist/llm/openai/responses-serializer.d.ts new file mode 100644 index 00000000..1be57486 --- /dev/null +++ b/dist/llm/openai/responses-serializer.d.ts @@ -0,0 +1,18 @@ +import { type Message } from '../messages.js'; +type ResponsesInputPart = { + type: 'input_text'; + text: string; +} | { + type: 'input_image'; + image_url: string; + detail?: 'auto' | 'low' | 'high'; +}; +export type ResponsesInputMessage = { + role: 'user' | 'system' | 'assistant'; + content: string | ResponsesInputPart[]; +}; +export declare class ResponsesAPIMessageSerializer { + serialize(messages: Message[]): ResponsesInputMessage[]; + private serializeMessage; +} +export {}; diff --git a/dist/llm/openai/responses-serializer.js b/dist/llm/openai/responses-serializer.js new file mode 100644 index 00000000..0c4c1b50 --- /dev/null +++ b/dist/llm/openai/responses-serializer.js @@ -0,0 +1,72 @@ +import { AssistantMessage, ContentPartImageParam, ContentPartRefusalParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js'; +export class ResponsesAPIMessageSerializer { + serialize(messages) { + return messages.map((message) => this.serializeMessage(message)); + } + serializeMessage(message) { + if (message instanceof UserMessage) { + if (typeof message.content === 'string') { + return { role: 'user', content: message.content }; + } + const content = message.content + .map((part) => { + if (part instanceof ContentPartTextParam) { + return { type: 'input_text', text: part.text }; + } + if (part instanceof ContentPartImageParam) { + return { + type: 'input_image', + image_url: part.image_url.url, + detail: part.image_url.detail, + }; + } + return null; + }) + .filter((part) => part !== null); + return { role: 'user', content }; + } + if (message instanceof SystemMessage) { + if (typeof message.content === 'string') { + return { role: 'system', content: message.content }; + } + return { + role: 'system', + content: message.content.map((part) => ({ + type: 'input_text', + text: part.text, + })), + }; + } + if (message instanceof AssistantMessage) { + if (message.content == null) { + if (Array.isArray(message.tool_calls) && + message.tool_calls.length > 0) { + const toolCallText = message.tool_calls + .map((toolCall) => `[Tool call: ${toolCall.functionCall.name}(${toolCall.functionCall.arguments})]`) + .join('\n'); + return { role: 'assistant', content: toolCallText }; + } + return { role: 'assistant', content: '' }; + } + if (typeof message.content === 'string') { + return { role: 'assistant', content: message.content }; + } + const content = message.content + .map((part) => { + if (part instanceof ContentPartTextParam) { + return { type: 'input_text', text: part.text }; + } + if (part instanceof ContentPartRefusalParam) { + return { + type: 'input_text', + text: `[Refusal: ${part.refusal}]`, + }; + } + return null; + }) + .filter((part) => part !== null); + return { role: 'assistant', content }; + } + throw new Error(`Unknown message type: ${message?.constructor?.name ?? typeof message}`); + } +} diff --git a/dist/llm/openai/serializer.d.ts b/dist/llm/openai/serializer.d.ts new file mode 100644 index 00000000..339310a3 --- /dev/null +++ b/dist/llm/openai/serializer.d.ts @@ -0,0 +1,6 @@ +import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs'; +import { type Message } from '../messages.js'; +export declare class OpenAIMessageSerializer { + serialize(messages: Message[]): ChatCompletionMessageParam[]; + private serializeMessage; +} diff --git a/dist/llm/openai/serializer.js b/dist/llm/openai/serializer.js new file mode 100644 index 00000000..63556452 --- /dev/null +++ b/dist/llm/openai/serializer.js @@ -0,0 +1,57 @@ +import { AssistantMessage, ContentPartImageParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js'; +export class OpenAIMessageSerializer { + serialize(messages) { + return messages.map((message) => this.serializeMessage(message)); + } + serializeMessage(message) { + if (message instanceof UserMessage) { + return { + role: 'user', + content: Array.isArray(message.content) + ? message.content.map((part) => { + if (part instanceof ContentPartTextParam) { + return { type: 'text', text: part.text }; + } + if (part instanceof ContentPartImageParam) { + return { + type: 'image_url', + image_url: { + url: part.image_url.url, + detail: part.image_url.detail, + }, + }; + } + return { type: 'text', text: '' }; // Fallback + }) + : message.content, + name: message.name || undefined, + }; + } + if (message instanceof SystemMessage) { + return { + role: 'system', + content: Array.isArray(message.content) + ? message.content.map((part) => part.text).join('\n') + : message.content, + name: message.name || undefined, + }; + } + if (message instanceof AssistantMessage) { + const toolCalls = message.tool_calls?.map((toolCall) => ({ + id: toolCall.id, + type: 'function', + function: { + name: toolCall.functionCall.name, + arguments: toolCall.functionCall.arguments, + }, + })); + return { + role: 'assistant', + content: typeof message.content === 'string' ? message.content : null, // OpenAI expects string or null for content in assistant message if tool_calls are present + tool_calls: toolCalls, + refusal: message.refusal || undefined, + }; + } + throw new Error(`Unknown message type: ${message.constructor.name}`); + } +} diff --git a/dist/llm/openrouter/chat.d.ts b/dist/llm/openrouter/chat.d.ts new file mode 100644 index 00000000..0d30515f --- /dev/null +++ b/dist/llm/openrouter/chat.d.ts @@ -0,0 +1,41 @@ +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import type { Message } from '../messages.js'; +import { ChatInvokeCompletion } from '../views.js'; +export interface ChatOpenRouterOptions { + model?: string; + apiKey?: string; + baseURL?: string; + timeout?: number | null; + temperature?: number | null; + topP?: number | null; + seed?: number | null; + maxRetries?: number; + defaultHeaders?: Record | null; + defaultQuery?: Record | null; + fetchImplementation?: typeof fetch; + fetchOptions?: RequestInit | null; + httpReferer?: string | null; + extraBody?: Record | null; + removeMinItemsFromSchema?: boolean; + removeDefaultsFromSchema?: boolean; +} +export declare class ChatOpenRouter implements BaseChatModel { + model: string; + provider: string; + private client; + private temperature; + private topP; + private seed; + private httpReferer; + private extraBody; + private removeMinItemsFromSchema; + private removeDefaultsFromSchema; + constructor(options?: string | ChatOpenRouterOptions); + get name(): string; + get model_name(): string; + private getUsage; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/openrouter/chat.js b/dist/llm/openrouter/chat.js new file mode 100644 index 00000000..2c45ac43 --- /dev/null +++ b/dist/llm/openrouter/chat.js @@ -0,0 +1,160 @@ +import OpenAI from 'openai'; +import { ModelProviderError } from '../exceptions.js'; +import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { OpenRouterMessageSerializer } from './serializer.js'; +export class ChatOpenRouter { + model; + provider = 'openrouter'; + client; + temperature; + topP; + seed; + httpReferer; + extraBody; + removeMinItemsFromSchema; + removeDefaultsFromSchema; + constructor(options = {}) { + const normalizedOptions = typeof options === 'string' ? { model: options } : options; + const { model = 'openai/gpt-4o', apiKey = process.env.OPENROUTER_API_KEY, baseURL = 'https://openrouter.ai/api/v1', timeout = null, temperature = null, topP = null, seed = null, maxRetries = 10, defaultHeaders = null, defaultQuery = null, fetchImplementation, fetchOptions = null, httpReferer = null, extraBody = null, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions; + this.model = model; + this.temperature = temperature; + this.topP = topP; + this.seed = seed; + this.httpReferer = httpReferer; + this.extraBody = extraBody; + this.removeMinItemsFromSchema = removeMinItemsFromSchema; + this.removeDefaultsFromSchema = removeDefaultsFromSchema; + this.client = new OpenAI({ + apiKey, + baseURL, + timeout: timeout ?? undefined, + maxRetries, + defaultHeaders: defaultHeaders ?? undefined, + defaultQuery: defaultQuery ?? undefined, + fetch: fetchImplementation, + fetchOptions: (fetchOptions ?? undefined), + }); + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + getUsage(response) { + if (!response.usage) { + return null; + } + return { + prompt_tokens: response.usage.prompt_tokens, + prompt_cached_tokens: response.usage.prompt_tokens_details?.cached_tokens ?? null, + prompt_cache_creation_tokens: null, + prompt_image_tokens: null, + completion_tokens: response.usage.completion_tokens, + total_tokens: response.usage.total_tokens, + }; + } + async ainvoke(messages, output_format, options = {}) { + const serializer = new OpenRouterMessageSerializer(); + const openRouterMessages = serializer.serialize(messages); + const modelParams = {}; + if (this.temperature !== null) { + modelParams.temperature = this.temperature; + } + if (this.topP !== null) { + modelParams.top_p = this.topP; + } + if (this.seed !== null) { + modelParams.seed = this.seed; + } + const zodSchemaCandidate = (() => { + const output = output_format; + if (output && + typeof output === 'object' && + typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return output; + } + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return output.schema; + } + return null; + })(); + let responseFormat = undefined; + if (zodSchemaCandidate) { + try { + const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, { + name: 'agent_output', + target: 'jsonSchema7', + }); + const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, { + removeMinItems: this.removeMinItemsFromSchema, + removeDefaults: this.removeDefaultsFromSchema, + }); + responseFormat = { + type: 'json_schema', + json_schema: { + name: 'agent_output', + schema: optimizedJsonSchema, + strict: true, + }, + }; + } + catch { + responseFormat = undefined; + } + } + const request = { + model: this.model, + messages: openRouterMessages, + response_format: responseFormat, + ...modelParams, + ...(this.extraBody ?? {}), + }; + if (this.httpReferer) { + request.extra_headers = { + 'HTTP-Referer': this.httpReferer, + }; + } + try { + const response = await this.client.chat.completions.create(request, options.signal ? { signal: options.signal } : undefined); + const content = response.choices[0].message.content || ''; + const usage = this.getUsage(response); + const stopReason = response.choices[0].finish_reason ?? null; + let completion = content; + if (output_format) { + if (zodSchemaCandidate) { + const parsedJson = JSON.parse(content); + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + completion = output.schema.parse(parsedJson); + } + else { + completion = output_format.parse(parsedJson); + } + } + else { + completion = output_format.parse(content); + } + } + return new ChatInvokeCompletion(completion, usage, null, null, stopReason); + } + catch (error) { + if (error?.status === 429) { + throw new ModelProviderError(error?.message ?? 'Rate limit exceeded', 429, this.model); + } + if (error?.status >= 500) { + throw new ModelProviderError(error?.message ?? 'Server error', error.status, this.model); + } + throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model); + } + } +} diff --git a/dist/llm/openrouter/index.d.ts b/dist/llm/openrouter/index.d.ts new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/openrouter/index.d.ts @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/openrouter/index.js b/dist/llm/openrouter/index.js new file mode 100644 index 00000000..b05bb150 --- /dev/null +++ b/dist/llm/openrouter/index.js @@ -0,0 +1,2 @@ +export * from './chat.js'; +export * from './serializer.js'; diff --git a/dist/llm/openrouter/serializer.d.ts b/dist/llm/openrouter/serializer.d.ts new file mode 100644 index 00000000..34782843 --- /dev/null +++ b/dist/llm/openrouter/serializer.d.ts @@ -0,0 +1,3 @@ +import { OpenAIMessageSerializer } from '../openai/serializer.js'; +export declare class OpenRouterMessageSerializer extends OpenAIMessageSerializer { +} diff --git a/dist/llm/openrouter/serializer.js b/dist/llm/openrouter/serializer.js new file mode 100644 index 00000000..ad475982 --- /dev/null +++ b/dist/llm/openrouter/serializer.js @@ -0,0 +1,3 @@ +import { OpenAIMessageSerializer } from '../openai/serializer.js'; +export class OpenRouterMessageSerializer extends OpenAIMessageSerializer { +} diff --git a/dist/llm/schema.d.ts b/dist/llm/schema.d.ts new file mode 100644 index 00000000..ad34bb71 --- /dev/null +++ b/dist/llm/schema.d.ts @@ -0,0 +1,16 @@ +type JsonSchema = Record; +interface ZodJsonSchemaOptions { + name?: string; + target?: string; + [key: string]: unknown; +} +export declare const zodSchemaToJsonSchema: (schema: unknown, options?: ZodJsonSchemaOptions) => JsonSchema; +export declare class SchemaOptimizer { + static createOptimizedJsonSchema(schema: JsonSchema, options?: { + removeMinItems?: boolean; + removeDefaults?: boolean; + }): JsonSchema; + static createGeminiOptimizedSchema(schema: JsonSchema): JsonSchema; + static makeStrictCompatible(schema: any): void; +} +export {}; diff --git a/dist/llm/schema.js b/dist/llm/schema.js new file mode 100644 index 00000000..e984a496 --- /dev/null +++ b/dist/llm/schema.js @@ -0,0 +1,182 @@ +import { z } from 'zod'; +import { zodToJsonSchema } from 'zod-to-json-schema'; +export const zodSchemaToJsonSchema = (schema, options = {}) => { + try { + const toJSONSchema = z?.toJSONSchema; + if (typeof toJSONSchema === 'function') { + const converted = toJSONSchema(schema); + if (converted && typeof converted === 'object') { + return converted; + } + } + } + catch { + // Fall back to zod-to-json-schema below. + } + return zodToJsonSchema(schema, options); +}; +export class SchemaOptimizer { + static createOptimizedJsonSchema(schema, options = {}) { + const defsLookup = schema.$defs ?? {}; + const removeMinItems = options.removeMinItems ?? false; + const removeDefaults = options.removeDefaults ?? false; + const optimize = (obj, inProperties = false) => { + if (Array.isArray(obj)) { + return obj.map((item) => optimize(item, inProperties)); + } + if (obj && typeof obj === 'object') { + let flattenedRef = null; + const optimized = {}; + for (const [key, value] of Object.entries(obj)) { + if (key === '$defs') + continue; + // Some OpenAI-compatible providers reject this JSON Schema keyword. + if (key === 'propertyNames') + continue; + if (key === 'title' && !inProperties) + continue; + if (removeMinItems && (key === 'minItems' || key === 'min_items')) { + continue; + } + if (removeDefaults && key === 'default') + continue; + if (key === '$ref' && typeof value === 'string') { + const refName = value.split('/').pop(); + if (defsLookup[refName]) { + flattenedRef = optimize(defsLookup[refName], inProperties); + } + continue; + } + if (key === 'additionalProperties') { + if (value && typeof value === 'object' && !Array.isArray(value)) { + const optimizedAdditional = optimize(value, inProperties); + optimized[key] = + Object.keys(optimizedAdditional).length === 0 + ? true + : optimizedAdditional; + } + else { + optimized[key] = value; + } + continue; + } + if (key === 'properties') { + optimized[key] = optimize(value, true); + continue; + } + if (typeof value === 'object' && value !== null) { + optimized[key] = optimize(value, inProperties); + continue; + } + optimized[key] = value; + } + const result = flattenedRef + ? { ...flattenedRef, ...optimized } + : optimized; + const hasExplicitProperties = result.properties && + typeof result.properties === 'object' && + !Array.isArray(result.properties); + if (result.type === 'object' && + result.additionalProperties === undefined && + hasExplicitProperties) { + result.additionalProperties = false; + } + return result; + } + return obj; + }; + const optimizedSchema = optimize(schema); + const ensureAdditionalProperties = (obj) => { + if (Array.isArray(obj)) { + obj.forEach(ensureAdditionalProperties); + return; + } + if (obj && typeof obj === 'object') { + const hasExplicitProperties = obj.properties && + typeof obj.properties === 'object' && + !Array.isArray(obj.properties); + if (obj.type === 'object' && + obj.additionalProperties === undefined && + hasExplicitProperties) { + obj.additionalProperties = false; + } + Object.values(obj).forEach(ensureAdditionalProperties); + } + }; + const stripStructuredDoneSuccess = (obj) => { + if (Array.isArray(obj)) { + obj.forEach(stripStructuredDoneSuccess); + return; + } + if (!obj || typeof obj !== 'object') { + return; + } + const properties = obj.properties; + if (obj.type === 'object' && + properties && + typeof properties === 'object' && + !Array.isArray(properties)) { + const dataSchema = properties.data; + const successSchema = properties.success; + const looksLikeStructuredDone = dataSchema && + successSchema && + successSchema.type === 'boolean' && + successSchema.description === + 'True if user_request completed successfully'; + if (looksLikeStructuredDone) { + delete properties.success; + if (Array.isArray(obj.required)) { + obj.required = obj.required.filter((name) => name !== 'success'); + } + } + } + Object.values(obj).forEach(stripStructuredDoneSuccess); + }; + const stripExtractOutputSchema = (obj, parentKey = null) => { + if (Array.isArray(obj)) { + obj.forEach((item) => stripExtractOutputSchema(item, parentKey)); + return; + } + if (!obj || typeof obj !== 'object') { + return; + } + const isExtractActionSchema = parentKey === 'extract_structured_data' || parentKey === 'extract'; + if (isExtractActionSchema && obj.type === 'object') { + const props = obj.properties; + if (props && typeof props === 'object' && !Array.isArray(props)) { + delete props.output_schema; + } + if (Array.isArray(obj.required)) { + obj.required = obj.required.filter((name) => name !== 'output_schema'); + } + } + for (const [key, value] of Object.entries(obj)) { + stripExtractOutputSchema(value, key); + } + }; + ensureAdditionalProperties(optimizedSchema); + stripStructuredDoneSuccess(optimizedSchema); + stripExtractOutputSchema(optimizedSchema); + SchemaOptimizer.makeStrictCompatible(optimizedSchema); + return optimizedSchema; + } + static createGeminiOptimizedSchema(schema) { + return SchemaOptimizer.createOptimizedJsonSchema(schema); + } + static makeStrictCompatible(schema) { + if (Array.isArray(schema)) { + schema.forEach(SchemaOptimizer.makeStrictCompatible); + return; + } + if (schema && typeof schema === 'object') { + for (const [key, value] of Object.entries(schema)) { + if (key !== 'required' && value && typeof value === 'object') { + SchemaOptimizer.makeStrictCompatible(value); + } + } + if (schema.type === 'object' && schema.properties) { + schema.required = Object.keys(schema.properties); + } + } + } +} diff --git a/dist/llm/vercel/chat.d.ts b/dist/llm/vercel/chat.d.ts new file mode 100644 index 00000000..1dee10ae --- /dev/null +++ b/dist/llm/vercel/chat.d.ts @@ -0,0 +1,50 @@ +import type { BaseChatModel, ChatInvokeOptions } from '../base.js'; +import type { Message } from '../messages.js'; +import { ChatInvokeCompletion } from '../views.js'; +export interface ChatVercelOptions { + model?: string; + apiKey?: string; + baseURL?: string; + timeout?: number | null; + temperature?: number | null; + maxTokens?: number | null; + topP?: number | null; + seed?: number | null; + maxRetries?: number; + defaultHeaders?: Record | null; + defaultQuery?: Record | null; + fetchImplementation?: typeof fetch; + fetchOptions?: RequestInit | null; + reasoningModels?: string[] | null; + providerOptions?: Record | null; + extraBody?: Record | null; + removeMinItemsFromSchema?: boolean; + removeDefaultsFromSchema?: boolean; +} +export declare class ChatVercel implements BaseChatModel { + model: string; + provider: string; + private client; + private temperature; + private maxTokens; + private topP; + private seed; + private reasoningModels; + private providerOptions; + private extraBody; + private removeMinItemsFromSchema; + private removeDefaultsFromSchema; + constructor(options?: string | ChatVercelOptions); + get name(): string; + get model_name(): string; + private getUsage; + private getExtraBodyPayload; + private cloneMessages; + private appendJsonInstructionToMessages; + private parseStructuredJson; + private parseOutput; + ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise>; + ainvoke(messages: Message[], output_format: { + parse: (input: string) => T; + } | undefined, options?: ChatInvokeOptions): Promise>; +} diff --git a/dist/llm/vercel/chat.js b/dist/llm/vercel/chat.js new file mode 100644 index 00000000..9ed29593 --- /dev/null +++ b/dist/llm/vercel/chat.js @@ -0,0 +1,276 @@ +import OpenAI from 'openai'; +import { ModelProviderError, ModelRateLimitError } from '../exceptions.js'; +import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js'; +import { ChatInvokeCompletion } from '../views.js'; +import { VercelMessageSerializer } from './serializer.js'; +const DEFAULT_REASONING_MODELS = [ + 'o1', + 'o3', + 'o4', + 'gpt-oss', + 'deepseek-r1', + 'qwen3-next-80b-a3b-thinking', +]; +export class ChatVercel { + model; + provider = 'vercel'; + client; + temperature; + maxTokens; + topP; + seed; + reasoningModels; + providerOptions; + extraBody; + removeMinItemsFromSchema; + removeDefaultsFromSchema; + constructor(options = {}) { + const normalizedOptions = typeof options === 'string' ? { model: options } : options; + const { model = 'openai/gpt-4o', apiKey = process.env.VERCEL_API_KEY, baseURL = process.env.VERCEL_BASE_URL || + 'https://ai-gateway.vercel.sh/v1', timeout = null, temperature = null, maxTokens = null, topP = null, seed = null, maxRetries = 5, defaultHeaders = null, defaultQuery = null, fetchImplementation, fetchOptions = null, reasoningModels = [...DEFAULT_REASONING_MODELS], providerOptions = null, extraBody = null, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions; + this.model = model; + this.temperature = temperature; + this.maxTokens = maxTokens; + this.topP = topP; + this.seed = seed; + this.reasoningModels = reasoningModels ? [...reasoningModels] : null; + this.providerOptions = providerOptions; + this.extraBody = extraBody; + this.removeMinItemsFromSchema = removeMinItemsFromSchema; + this.removeDefaultsFromSchema = removeDefaultsFromSchema; + this.client = new OpenAI({ + apiKey, + baseURL, + timeout: timeout ?? undefined, + maxRetries, + defaultHeaders: defaultHeaders ?? undefined, + defaultQuery: defaultQuery ?? undefined, + fetch: fetchImplementation, + fetchOptions: (fetchOptions ?? undefined), + }); + } + get name() { + return this.model; + } + get model_name() { + return this.model; + } + getUsage(response) { + if (!response.usage) { + return null; + } + return { + prompt_tokens: response.usage.prompt_tokens, + prompt_cached_tokens: response.usage.prompt_tokens_details?.cached_tokens ?? null, + prompt_cache_creation_tokens: null, + prompt_image_tokens: null, + completion_tokens: response.usage.completion_tokens, + total_tokens: response.usage.total_tokens, + }; + } + getExtraBodyPayload() { + const payload = { + ...(this.extraBody ?? {}), + }; + if (this.providerOptions) { + payload.providerOptions = this.providerOptions; + } + return Object.keys(payload).length > 0 ? payload : undefined; + } + cloneMessages(messages) { + return messages.map((message) => ({ + ...message, + content: Array.isArray(message?.content) + ? message.content.map((part) => ({ ...part })) + : message?.content, + })); + } + appendJsonInstructionToMessages(messages, schema) { + const cloned = this.cloneMessages(messages); + const instruction = '\n\nIMPORTANT: You must respond with ONLY a valid JSON object ' + + '(no markdown, no code blocks, no explanations) that exactly matches this schema:\n' + + `${JSON.stringify(schema, null, 2)}`; + if (cloned.length > 0 && cloned[0]?.role === 'system') { + if (typeof cloned[0].content === 'string') { + cloned[0].content = `${cloned[0].content}${instruction}`; + } + else if (Array.isArray(cloned[0].content)) { + cloned[0].content = [ + ...cloned[0].content, + { type: 'text', text: instruction }, + ]; + } + else { + cloned[0].content = instruction; + } + return cloned; + } + for (let i = cloned.length - 1; i >= 0; i -= 1) { + if (cloned[i]?.role === 'user') { + if (typeof cloned[i].content === 'string') { + cloned[i].content = `${cloned[i].content}${instruction}`; + } + else if (Array.isArray(cloned[i].content)) { + cloned[i].content = [ + ...cloned[i].content, + { type: 'text', text: instruction }, + ]; + } + else { + cloned[i].content = instruction; + } + return cloned; + } + } + cloned.unshift({ + role: 'system', + content: instruction, + }); + return cloned; + } + parseStructuredJson(text) { + let jsonText = String(text ?? '').trim(); + if (jsonText.startsWith('```json') && jsonText.endsWith('```')) { + jsonText = jsonText.slice(7, -3).trim(); + } + else if (jsonText.startsWith('```') && jsonText.endsWith('```')) { + jsonText = jsonText.slice(3, -3).trim(); + } + return JSON.parse(jsonText); + } + parseOutput(output_format, payload) { + const output = output_format; + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.parse === 'function') { + return output.schema.parse(payload); + } + return output.parse(payload); + } + async ainvoke(messages, output_format, options = {}) { + const serializer = new VercelMessageSerializer(); + const vercelMessages = serializer.serialize(messages); + const modelParams = {}; + if (this.temperature !== null) { + modelParams.temperature = this.temperature; + } + if (this.maxTokens !== null) { + modelParams.max_tokens = this.maxTokens; + } + if (this.topP !== null) { + modelParams.top_p = this.topP; + } + if (this.seed !== null) { + modelParams.seed = this.seed; + } + const zodSchemaCandidate = (() => { + const output = output_format; + if (output && + typeof output === 'object' && + typeof output.safeParse === 'function' && + typeof output.parse === 'function') { + return output; + } + if (output && + typeof output === 'object' && + output.schema && + typeof output.schema.safeParse === 'function' && + typeof output.schema.parse === 'function') { + return output.schema; + } + return null; + })(); + const extraBodyPayload = this.getExtraBodyPayload(); + const isGoogleModel = this.model.startsWith('google/'); + const isAnthropicModel = this.model.startsWith('anthropic/'); + const isReasoningModel = (this.reasoningModels ?? []).some((pattern) => String(this.model).toLowerCase().includes(String(pattern).toLowerCase())); + if (output_format && + zodSchemaCandidate && + (isGoogleModel || isAnthropicModel || isReasoningModel)) { + try { + const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, { + name: 'agent_output', + target: 'jsonSchema7', + }); + const optimizedJsonSchema = SchemaOptimizer.createGeminiOptimizedSchema(rawJsonSchema); + const requestMessages = this.appendJsonInstructionToMessages(vercelMessages, optimizedJsonSchema); + const request = { + model: this.model, + messages: requestMessages, + ...modelParams, + }; + if (extraBodyPayload) { + request.extra_body = extraBodyPayload; + } + const response = await this.client.chat.completions.create(request, options.signal ? { signal: options.signal } : undefined); + const content = response.choices[0].message.content || ''; + const usage = this.getUsage(response); + const stopReason = response.choices[0].finish_reason ?? null; + const completion = this.parseOutput(output_format, this.parseStructuredJson(content)); + return new ChatInvokeCompletion(completion, usage, null, null, stopReason); + } + catch (error) { + throw new ModelProviderError(`Failed to parse JSON response: ${error?.message ?? String(error)}`, 500, this.model); + } + } + let responseFormat = undefined; + if (zodSchemaCandidate) { + try { + const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, { + name: 'agent_output', + target: 'jsonSchema7', + }); + const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, { + removeMinItems: this.removeMinItemsFromSchema, + removeDefaults: this.removeDefaultsFromSchema, + }); + responseFormat = { + type: 'json_schema', + json_schema: { + name: 'agent_output', + schema: optimizedJsonSchema, + strict: true, + }, + }; + } + catch { + responseFormat = undefined; + } + } + const request = { + model: this.model, + messages: vercelMessages, + response_format: responseFormat, + ...modelParams, + }; + if (extraBodyPayload) { + request.extra_body = extraBodyPayload; + } + try { + const response = await this.client.chat.completions.create(request, options.signal ? { signal: options.signal } : undefined); + const content = response.choices[0].message.content || ''; + const usage = this.getUsage(response); + const stopReason = response.choices[0].finish_reason ?? null; + let completion = content; + if (output_format) { + if (zodSchemaCandidate) { + completion = this.parseOutput(output_format, JSON.parse(content)); + } + else { + completion = this.parseOutput(output_format, content); + } + } + return new ChatInvokeCompletion(completion, usage, null, null, stopReason); + } + catch (error) { + if (error?.status === 429) { + throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model); + } + if (error?.status >= 500) { + throw new ModelProviderError(error?.message ?? 'Server error', error.status, this.model); + } + throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model); + } + } +} diff --git a/dist/llm/vercel/index.d.ts b/dist/llm/vercel/index.d.ts new file mode 100644 index 00000000..69a16816 --- /dev/null +++ b/dist/llm/vercel/index.d.ts @@ -0,0 +1 @@ +export * from './chat.js'; diff --git a/dist/llm/vercel/index.js b/dist/llm/vercel/index.js new file mode 100644 index 00000000..69a16816 --- /dev/null +++ b/dist/llm/vercel/index.js @@ -0,0 +1 @@ +export * from './chat.js'; diff --git a/dist/llm/vercel/serializer.d.ts b/dist/llm/vercel/serializer.d.ts new file mode 100644 index 00000000..1dce6fd7 --- /dev/null +++ b/dist/llm/vercel/serializer.d.ts @@ -0,0 +1,5 @@ +import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs'; +import type { Message } from '../messages.js'; +export declare class VercelMessageSerializer { + serialize(messages: Message[]): ChatCompletionMessageParam[]; +} diff --git a/dist/llm/vercel/serializer.js b/dist/llm/vercel/serializer.js new file mode 100644 index 00000000..048c1453 --- /dev/null +++ b/dist/llm/vercel/serializer.js @@ -0,0 +1,7 @@ +import { OpenAIMessageSerializer } from '../openai/serializer.js'; +export class VercelMessageSerializer { + serialize(messages) { + const serializer = new OpenAIMessageSerializer(); + return serializer.serialize(messages); + } +} diff --git a/dist/llm/views.d.ts b/dist/llm/views.d.ts new file mode 100644 index 00000000..48e4721a --- /dev/null +++ b/dist/llm/views.d.ts @@ -0,0 +1,16 @@ +export interface ChatInvokeUsage { + prompt_tokens: number; + prompt_cached_tokens?: number | null; + prompt_cache_creation_tokens?: number | null; + prompt_image_tokens?: number | null; + completion_tokens: number; + total_tokens: number; +} +export declare class ChatInvokeCompletion { + completion: T; + usage: ChatInvokeUsage | null; + thinking: string | null; + redacted_thinking: string | null; + stop_reason: string | null; + constructor(completion: T, usage?: ChatInvokeUsage | null, thinking?: string | null, redacted_thinking?: string | null, stop_reason?: string | null); +} diff --git a/dist/llm/views.js b/dist/llm/views.js new file mode 100644 index 00000000..4e9e1127 --- /dev/null +++ b/dist/llm/views.js @@ -0,0 +1,14 @@ +export class ChatInvokeCompletion { + completion; + usage; + thinking; + redacted_thinking; + stop_reason; + constructor(completion, usage = null, thinking = null, redacted_thinking = null, stop_reason = null) { + this.completion = completion; + this.usage = usage; + this.thinking = thinking; + this.redacted_thinking = redacted_thinking; + this.stop_reason = stop_reason; + } +} diff --git a/dist/logging-config.d.ts b/dist/logging-config.d.ts new file mode 100644 index 00000000..a80574f5 --- /dev/null +++ b/dist/logging-config.d.ts @@ -0,0 +1,27 @@ +import { Writable } from 'node:stream'; +export type LogLevel = 'debug' | 'info' | 'result' | 'warning' | 'error'; +interface SetupLoggingOptions { + stream?: Writable; + logLevel?: LogLevel; + forceSetup?: boolean; + debugLogFile?: string | null; + infoLogFile?: string | null; +} +export declare class Logger { + private readonly name; + constructor(name: string); + private shouldLog; + get level(): LogLevel; + private emit; + debug(message: string, ...args: unknown[]): void; + info(message: string, ...args: unknown[]): void; + result(message: string, ...args: unknown[]): void; + warning(message: string, ...args: unknown[]): void; + warn(message: string, ...args: unknown[]): void; + error(message: string, ...args: unknown[]): void; + child(suffix: string): Logger; +} +export declare const createLogger: (name: string) => Logger; +export declare const setupLogging: (options?: SetupLoggingOptions) => Logger; +export declare const logger: Logger; +export {}; diff --git a/dist/logging-config.js b/dist/logging-config.js new file mode 100644 index 00000000..15eee1c7 --- /dev/null +++ b/dist/logging-config.js @@ -0,0 +1,142 @@ +import fs from 'node:fs'; +import path from 'node:path'; +const LEVEL_PRIORITY = { + debug: 10, + info: 20, + result: 25, + warning: 30, + error: 40, +}; +let configured = false; +let consoleLevel = process.env.BROWSER_USE_LOGGING_LEVEL || 'info'; +let consoleStream = process.stderr; +let debugLogStream = null; +let infoLogStream = null; +const normalizeLogLevel = (candidate, fallback) => { + if (!candidate) { + return fallback; + } + const normalized = candidate.toLowerCase(); + return normalized in LEVEL_PRIORITY ? normalized : fallback; +}; +const formatMessage = (level, name, message) => { + if (level === 'result') { + return message; + } + const paddedLevel = level.toUpperCase().padEnd(7, ' '); + return `${paddedLevel} [${name}] ${message}`; +}; +const formatFileMessage = (level, name, message) => `${new Date().toISOString()} ${formatMessage(level, name, message)}`; +const writePayload = (stream, level, payload) => { + if ('write' in stream) { + stream.write(`${payload}\n`); + return; + } + switch (level) { + case 'error': + console.error(payload); + break; + case 'warning': + console.warn(payload); + break; + default: + console.log(payload); + break; + } +}; +const ensureFilePathReady = (filePath) => { + fs.mkdirSync(path.dirname(path.resolve(filePath)), { recursive: true }); +}; +const closeFileStreams = () => { + if (debugLogStream) { + debugLogStream.end(); + debugLogStream = null; + } + if (infoLogStream) { + infoLogStream.end(); + infoLogStream = null; + } +}; +export class Logger { + name; + constructor(name) { + this.name = name; + } + shouldLog(level, threshold) { + return LEVEL_PRIORITY[level] >= LEVEL_PRIORITY[threshold]; + } + get level() { + return consoleLevel; + } + emit(level, message, ...args) { + const argsPayload = args.length + ? args + .map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))) + .join(' ') + : ''; + const formatted = formatMessage(level, this.name, message); + const payload = argsPayload ? `${formatted} ${argsPayload}` : formatted; + if (this.shouldLog(level, consoleLevel)) { + writePayload(consoleStream, level, payload); + } + if (debugLogStream && this.shouldLog(level, 'debug')) { + const filePayload = formatFileMessage(level, this.name, message); + debugLogStream.write(`${argsPayload ? `${filePayload} ${argsPayload}` : filePayload}\n`); + } + if (infoLogStream && this.shouldLog(level, 'info')) { + const filePayload = formatFileMessage(level, this.name, message); + infoLogStream.write(`${argsPayload ? `${filePayload} ${argsPayload}` : filePayload}\n`); + } + } + debug(message, ...args) { + this.emit('debug', message, ...args); + } + info(message, ...args) { + this.emit('info', message, ...args); + } + result(message, ...args) { + this.emit('result', message, ...args); + } + warning(message, ...args) { + this.emit('warning', message, ...args); + } + // Alias for compatibility + warn(message, ...args) { + this.warning(message, ...args); + } + error(message, ...args) { + this.emit('error', message, ...args); + } + child(suffix) { + return new Logger(`${this.name}.${suffix}`); + } +} +export const createLogger = (name) => new Logger(name); +export const setupLogging = (options = {}) => { + if (configured && !options.forceSetup) { + return createLogger('browser_use'); + } + closeFileStreams(); + consoleLevel = normalizeLogLevel(options.logLevel ?? process.env.BROWSER_USE_LOGGING_LEVEL, 'info'); + consoleStream = options.stream || process.stderr; + const debugLogFile = options.debugLogFile ?? process.env.BROWSER_USE_DEBUG_LOG_FILE ?? null; + if (debugLogFile && debugLogFile.trim().length > 0) { + ensureFilePathReady(debugLogFile); + debugLogStream = fs.createWriteStream(path.resolve(debugLogFile), { + flags: 'a', + encoding: 'utf-8', + }); + } + const infoLogFile = options.infoLogFile ?? process.env.BROWSER_USE_INFO_LOG_FILE ?? null; + if (infoLogFile && infoLogFile.trim().length > 0) { + ensureFilePathReady(infoLogFile); + infoLogStream = fs.createWriteStream(path.resolve(infoLogFile), { + flags: 'a', + encoding: 'utf-8', + }); + } + configured = true; + return createLogger('browser_use'); +}; +setupLogging(); +export const logger = createLogger('browser_use'); diff --git a/dist/mcp/client.d.ts b/dist/mcp/client.d.ts new file mode 100644 index 00000000..6e49c4eb --- /dev/null +++ b/dist/mcp/client.d.ts @@ -0,0 +1,147 @@ +/** + * MCP (Model Context Protocol) client integration for browser-use. + * + * This module provides integration between external MCP servers and browser-use's action registry. + * MCP tools are dynamically discovered and registered as browser-use actions. + * + * Example usage: + * import { Tools } from './tools/service.js'; + * import { MCPClient } from './mcp/client.js'; + * + * const tools = new Tools(); + * + * // Connect to an MCP server + * const mcpClient = new MCPClient( + * 'my-server', + * 'npx', + * ['@mycompany/mcp-server@latest'] + * ); + * + * // Register all MCP tools as browser-use actions + * await mcpClient.registerToTools(tools); + * + * // Now use with Agent as normal - MCP tools are available as actions + */ +import { type Tool, type Prompt } from '@modelcontextprotocol/sdk/types.js'; +import type { Controller } from '../controller/service.js'; +import type { Tools } from '../tools/service.js'; +export interface MCPClientOptions { + /** Maximum number of connection retry attempts (default: 3) */ + maxRetries?: number; + /** Connection timeout in seconds (default: 30) */ + connectionTimeout?: number; + /** Tool call timeout in seconds (default: 60) */ + toolCallTimeout?: number; + /** Enable auto-reconnect on connection loss (default: true) */ + autoReconnect?: boolean; + /** Health check interval in seconds (default: 30, 0 = disabled) */ + healthCheckInterval?: number; +} +export declare class MCPClient { + private client; + private command; + private args; + private env?; + private serverName; + private _tools; + private _prompts; + private _registeredActions; + private _connected; + private _connecting; + private _toolCallCount; + private _errorCount; + private _lastConnectTime?; + private _lastHealthCheck?; + private _healthCheckInterval?; + private maxRetries; + private connectionTimeout; + private toolCallTimeout; + private autoReconnect; + private healthCheckIntervalSeconds; + constructor(serverName: string, command: string, args?: string[], env?: Record, options?: MCPClientOptions); + /** + * Connect to the MCP server and discover available tools + */ + connect(timeout?: number): Promise; + private _connectWithTimeout; + /** + * Disconnect from the MCP server + */ + disconnect(): Promise; + /** + * List all available tools from the MCP server + */ + listTools(): Promise; + /** + * Call a tool on the MCP server + */ + callTool(name: string, args: any): Promise; + /** + * Register MCP tools as actions in browser-use tools. + * + * @param tools - Browser-use tools to register actions to + * @param toolFilter - Optional list of tool names to register (undefined = all tools) + * @param prefix - Optional prefix to add to action names (e.g., "playwright_") + */ + registerToTools(tools: Pick, toolFilter?: string[], prefix?: string): Promise; + /** + * @deprecated Use `registerToTools` instead. + */ + registerToController(controller: Controller, toolFilter?: string[], prefix?: string): Promise; + private _registerToolAsAction; + private _convertToolSchemaToParamModel; + private _jsonSchemaToZod; + private _toJsonSchemaObject; + private _toLiteralUnion; + private _isLiteralPrimitive; + private _applyDefault; + private _applySchemaMetadata; + private _formatMcpResult; + /** + * List available prompts from the MCP server + */ + listPrompts(): Promise; + /** + * Get a prompt with arguments + */ + getPrompt(name: string, args?: Record): Promise; + /** + * Start health check monitoring + */ + private _startHealthCheck; + /** + * Stop health check monitoring + */ + private _stopHealthCheck; + /** + * Perform health check by listing tools + */ + private _performHealthCheck; + /** + * Attempt to reconnect to the server + */ + private _attemptReconnect; + /** + * Get client statistics + */ + getStats(): { + serverName: string; + connected: boolean; + toolsDiscovered: number; + promptsDiscovered: number; + toolCallCount: number; + errorCount: number; + successRate: number; + uptime?: number; + lastHealthCheck?: number; + }; + /** + * Check if client is connected + */ + isConnected(): boolean; + /** + * Reset statistics + */ + resetStats(): void; + [Symbol.asyncDispose](): Promise; +} diff --git a/dist/mcp/client.js b/dist/mcp/client.js new file mode 100644 index 00000000..e9779d40 --- /dev/null +++ b/dist/mcp/client.js @@ -0,0 +1,644 @@ +/** + * MCP (Model Context Protocol) client integration for browser-use. + * + * This module provides integration between external MCP servers and browser-use's action registry. + * MCP tools are dynamically discovered and registered as browser-use actions. + * + * Example usage: + * import { Tools } from './tools/service.js'; + * import { MCPClient } from './mcp/client.js'; + * + * const tools = new Tools(); + * + * // Connect to an MCP server + * const mcpClient = new MCPClient( + * 'my-server', + * 'npx', + * ['@mycompany/mcp-server@latest'] + * ); + * + * // Register all MCP tools as browser-use actions + * await mcpClient.registerToTools(tools); + * + * // Now use with Agent as normal - MCP tools are available as actions + */ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; +import { CallToolRequestSchema, ListToolsRequestSchema, ListPromptsRequestSchema, GetPromptRequestSchema, } from '@modelcontextprotocol/sdk/types.js'; +import { z } from 'zod'; +import { createLogger } from '../logging-config.js'; +import { ActionResult } from '../agent/views.js'; +import { productTelemetry } from '../telemetry/service.js'; +import { MCPClientTelemetryEvent } from '../telemetry/views.js'; +import { get_browser_use_version, retryAsync } from '../utils.js'; +const logger = createLogger('browser_use.mcp.client'); +export class MCPClient { + client; + command; + args; + env; + serverName; + _tools = new Map(); + _prompts = new Map(); + _registeredActions = new Set(); + _connected = false; + _connecting = false; + _toolCallCount = 0; + _errorCount = 0; + _lastConnectTime; + _lastHealthCheck; + _healthCheckInterval; + // Options + maxRetries; + connectionTimeout; + toolCallTimeout; + autoReconnect; + healthCheckIntervalSeconds; + constructor(serverName, command, args = [], env, options = {}) { + this.serverName = serverName; + this.command = command; + this.args = args; + this.env = env; + // Set options with defaults + this.maxRetries = options.maxRetries ?? 3; + this.connectionTimeout = options.connectionTimeout ?? 30; + this.toolCallTimeout = options.toolCallTimeout ?? 60; + this.autoReconnect = options.autoReconnect ?? true; + this.healthCheckIntervalSeconds = options.healthCheckInterval ?? 30; + this.client = new Client({ + name: 'browser-use', + version: get_browser_use_version(), + }, { + capabilities: {}, + }); + } + /** + * Connect to the MCP server and discover available tools + */ + async connect(timeout) { + if (this._connected) { + logger.debug(`Already connected to ${this.serverName}`); + return; + } + if (this._connecting) { + logger.debug(`Connection already in progress for ${this.serverName}`); + return; + } + this._connecting = true; + const actualTimeout = timeout ?? this.connectionTimeout; + const startTime = Date.now() / 1000; + let errorMsg = null; + try { + logger.info(`🔌 Connecting to MCP server '${this.serverName}': ${this.command} ${this.args.join(' ')}`); + // Use retry logic for connection + await retryAsync(async () => { + // Create transport with environment variables + const transport = new StdioClientTransport({ + command: this.command, + args: this.args, + env: this.env, + }); + // Connect with timeout + await this._connectWithTimeout(transport, actualTimeout); + }, { + maxAttempts: this.maxRetries, + delayMs: 1000, + backoffMultiplier: 2, + onRetry: (error, attempt, delay) => { + logger.warning(`Connection attempt ${attempt} failed for '${this.serverName}': ${error.message}. Retrying in ${delay}ms...`); + }, + }); + this._connected = true; + this._lastConnectTime = Date.now() / 1000; + // Discover available tools + const result = await this.client.request(ListToolsRequestSchema, {}); + this._tools = new Map(result.tools.map((tool) => [tool.name, tool])); + // Try to discover prompts (optional) + try { + await this.listPrompts(); + } + catch { + // Prompts are optional, ignore failures + } + logger.info(`📦 Discovered ${this._tools.size} tools${this._prompts.size > 0 ? ` and ${this._prompts.size} prompts` : ''} from '${this.serverName}'`); + // Start health checks + this._startHealthCheck(); + } + catch (error) { + errorMsg = error instanceof Error ? error.message : String(error); + this._connected = false; + throw error; + } + finally { + this._connecting = false; + // Capture telemetry for connect action + const duration = Date.now() / 1000 - startTime; + productTelemetry.capture(new MCPClientTelemetryEvent({ + server_name: this.serverName, + command: this.command, + tools_discovered: this._tools.size, + version: get_browser_use_version(), + action: 'connect', + duration_seconds: duration, + error_message: errorMsg, + })); + } + } + async _connectWithTimeout(transport, timeoutSeconds) { + return new Promise((resolve, reject) => { + const timeoutHandle = setTimeout(() => { + reject(new Error(`Failed to connect to MCP server '${this.serverName}' after ${timeoutSeconds} seconds`)); + }, timeoutSeconds * 1000); + this.client + .connect(transport) + .then(() => { + clearTimeout(timeoutHandle); + resolve(); + }) + .catch((error) => { + clearTimeout(timeoutHandle); + reject(error); + }); + }); + } + /** + * Disconnect from the MCP server + */ + async disconnect() { + if (!this._connected) { + return; + } + const startTime = Date.now() / 1000; + let errorMsg = null; + try { + logger.info(`🔌 Disconnecting from MCP server '${this.serverName}'`); + // Stop health checks + this._stopHealthCheck(); + await this.client.close(); + this._connected = false; + this._tools.clear(); + this._prompts.clear(); + this._registeredActions.clear(); + const stats = this.getStats(); + logger.info(`Disconnected from '${this.serverName}' (${stats.toolCallCount} tool calls, ${(stats.successRate * 100).toFixed(1)}% success rate)`); + } + catch (error) { + errorMsg = error instanceof Error ? error.message : String(error); + logger.error(`Error disconnecting from MCP server: ${errorMsg}`); + } + finally { + // Capture telemetry for disconnect action + const duration = Date.now() / 1000 - startTime; + productTelemetry.capture(new MCPClientTelemetryEvent({ + server_name: this.serverName, + command: this.command, + tools_discovered: 0, // Tools cleared on disconnect + version: get_browser_use_version(), + action: 'disconnect', + duration_seconds: duration, + error_message: errorMsg, + })); + productTelemetry.flush(); + } + } + /** + * List all available tools from the MCP server + */ + async listTools() { + if (!this._connected) { + await this.connect(); + } + return Array.from(this._tools.values()); + } + /** + * Call a tool on the MCP server + */ + async callTool(name, args) { + if (!this._connected) { + throw new Error(`MCP server '${this.serverName}' not connected`); + } + const startTime = Date.now() / 1000; + let errorMsg = null; + try { + logger.debug(`🔧 Calling MCP tool '${name}' with params: ${JSON.stringify(args)}`); + this._toolCallCount++; + const result = await this.client.request(CallToolRequestSchema, { + name, + arguments: args, + }); + return result.content; + } + catch (error) { + this._errorCount++; + errorMsg = error instanceof Error ? error.message : String(error); + logger.error(`MCP tool '${name}' failed: ${errorMsg}`); + throw error; + } + finally { + // Capture telemetry for tool call + const duration = Date.now() / 1000 - startTime; + productTelemetry.capture(new MCPClientTelemetryEvent({ + server_name: this.serverName, + command: this.command, + tools_discovered: this._tools.size, + version: get_browser_use_version(), + action: 'tool_call', + tool_name: name, + duration_seconds: duration, + error_message: errorMsg, + })); + } + } + /** + * Register MCP tools as actions in browser-use tools. + * + * @param tools - Browser-use tools to register actions to + * @param toolFilter - Optional list of tool names to register (undefined = all tools) + * @param prefix - Optional prefix to add to action names (e.g., "playwright_") + */ + async registerToTools(tools, toolFilter, prefix) { + if (!this._connected) { + await this.connect(); + } + const registry = tools.registry; + for (const [toolName, tool] of this._tools.entries()) { + // Skip if not in filter + if (toolFilter && !toolFilter.includes(toolName)) { + continue; + } + // Apply prefix if specified + const actionName = prefix ? `${prefix}${toolName}` : toolName; + // Skip if already registered + if (this._registeredActions.has(actionName)) { + continue; + } + // Register the tool as an action + this._registerToolAsAction(registry, actionName, tool); + this._registeredActions.add(actionName); + } + logger.info(`✅ Registered ${this._registeredActions.size} MCP tools from '${this.serverName}' as browser-use actions`); + } + /** + * @deprecated Use `registerToTools` instead. + */ + async registerToController(controller, toolFilter, prefix) { + await this.registerToTools(controller, toolFilter, prefix); + } + _registerToolAsAction(registry, actionName, tool) { + /** + * Register a single MCP tool as a browser-use action + */ + // Create async wrapper function for the MCP tool + const mcpActionWrapper = async (params) => { + if (!this._connected) { + return new ActionResult({ + error: `MCP server '${this.serverName}' not connected`, + success: false, + }); + } + const startTime = Date.now() / 1000; + try { + // Call the MCP tool + const result = await this.callTool(tool.name, params || {}); + // Convert MCP result to ActionResult + const extractedContent = this._formatMcpResult(result); + return new ActionResult({ + extracted_content: extractedContent, + long_term_memory: `Used MCP tool '${tool.name}' from ${this.serverName}`, + }); + } + catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + logger.error(`MCP tool '${tool.name}' failed: ${errorMsg}`); + return new ActionResult({ + error: `MCP tool '${tool.name}' failed: ${errorMsg}`, + success: false, + }); + } + }; + // Set function metadata for better debugging + Object.defineProperty(mcpActionWrapper, 'name', { value: actionName }); + // Register the action with browser-use + const description = tool.description || `MCP tool from ${this.serverName}: ${tool.name}`; + const paramModel = this._convertToolSchemaToParamModel(tool.inputSchema); + // Use the registry's action decorator + registry.action(description, { + param_model: paramModel, + })(mcpActionWrapper); + logger.debug(`✅ Registered MCP tool '${tool.name}' as action '${actionName}'`); + } + _convertToolSchemaToParamModel(inputSchema) { + if (!inputSchema || typeof inputSchema !== 'object') { + return z.object({}).strict(); + } + const schemaObject = inputSchema; + if (Object.keys(schemaObject).length === 0) { + return z.object({}).strict(); + } + const converted = this._jsonSchemaToZod(schemaObject, 'input'); + if (converted instanceof z.ZodObject) { + return converted.strict(); + } + return z.any(); + } + _jsonSchemaToZod(schema, path) { + const typeValue = schema.type; + const enumValues = Array.isArray(schema.enum) ? schema.enum : null; + if ('const' in schema) { + if (this._isLiteralPrimitive(schema.const)) { + return this._applySchemaMetadata(z.literal(schema.const), schema); + } + return this._applySchemaMetadata(z.any(), schema); + } + if (enumValues && enumValues.length > 0) { + const enumSchema = this._toLiteralUnion(enumValues); + return this._applySchemaMetadata(enumSchema, schema); + } + if (Array.isArray(schema.oneOf) && schema.oneOf.length > 0) { + const options = schema.oneOf.map((option, index) => this._jsonSchemaToZod(this._toJsonSchemaObject(option), `${path}.oneOf[${index}]`)); + const union = options.length === 1 ? options[0] : z.union(options); + return this._applySchemaMetadata(union, schema); + } + if (Array.isArray(schema.anyOf) && schema.anyOf.length > 0) { + const options = schema.anyOf.map((option, index) => this._jsonSchemaToZod(this._toJsonSchemaObject(option), `${path}.anyOf[${index}]`)); + const union = options.length === 1 ? options[0] : z.union(options); + return this._applySchemaMetadata(union, schema); + } + if (Array.isArray(typeValue)) { + const hasNull = typeValue.includes('null'); + const nonNullTypes = typeValue.filter((entry) => entry !== 'null'); + if (nonNullTypes.length === 1) { + const base = this._jsonSchemaToZod({ ...schema, type: nonNullTypes[0] }, path); + return hasNull ? base.nullable() : base; + } + if (nonNullTypes.length > 1) { + const options = nonNullTypes.map((entry) => this._jsonSchemaToZod({ ...schema, type: entry }, path)); + const union = options.length === 1 ? options[0] : z.union(options); + return hasNull ? union.nullable() : union; + } + } + let zodSchema; + if (typeValue === 'object' || schema.properties) { + const properties = schema.properties && typeof schema.properties === 'object' + ? schema.properties + : {}; + const required = new Set(Array.isArray(schema.required) + ? schema.required.filter((entry) => typeof entry === 'string') + : []); + const shape = {}; + for (const [key, value] of Object.entries(properties)) { + const propertySchema = this._toJsonSchemaObject(value); + const hasDefault = Object.prototype.hasOwnProperty.call(propertySchema, 'default'); + let field = this._jsonSchemaToZod(propertySchema, `${path}.${key}`); + if (hasDefault) { + field = this._applyDefault(field, propertySchema.default); + } + else if (!required.has(key)) { + field = field.optional(); + } + shape[key] = field; + } + zodSchema = z.object(shape); + } + else if (typeValue === 'array') { + const itemsSchema = this._toJsonSchemaObject(schema.items); + zodSchema = z.array(this._jsonSchemaToZod(itemsSchema, `${path}[]`)); + } + else if (typeValue === 'string') { + zodSchema = z.string(); + } + else if (typeValue === 'integer') { + zodSchema = z.number().int(); + } + else if (typeValue === 'number') { + zodSchema = z.number(); + } + else if (typeValue === 'boolean') { + zodSchema = z.boolean(); + } + else if (typeValue === 'null') { + zodSchema = z.null(); + } + else { + zodSchema = z.any(); + } + return this._applySchemaMetadata(zodSchema, schema); + } + _toJsonSchemaObject(input) { + if (!input || typeof input !== 'object') { + return {}; + } + return input; + } + _toLiteralUnion(values) { + const literalValues = values.filter((value) => this._isLiteralPrimitive(value)); + if (!literalValues.length) { + return z.any(); + } + if (literalValues.length === 1) { + return z.literal(literalValues[0]); + } + const literals = literalValues.map((value) => z.literal(value)); + return z.union(literals); + } + _isLiteralPrimitive(value) { + return (typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' || + value === null); + } + _applyDefault(schema, value) { + try { + return schema.default(value); + } + catch { + return schema; + } + } + _applySchemaMetadata(schema, sourceSchema) { + let result = schema; + if (sourceSchema.nullable === true) { + result = result.nullable(); + } + if (typeof sourceSchema.description === 'string') { + result = result.describe(sourceSchema.description); + } + return result; + } + _formatMcpResult(result) { + /** + * Format MCP tool result into a string for ActionResult + */ + // Handle different MCP result formats + if (result && typeof result === 'object') { + if (Array.isArray(result)) { + // List of content items + const parts = []; + for (const item of result) { + if (item && typeof item === 'object' && 'text' in item) { + parts.push(String(item.text)); + } + else { + parts.push(String(item)); + } + } + return parts.join('\n'); + } + else if ('text' in result) { + return String(result.text); + } + else if ('content' in result) { + // Structured content response + if (Array.isArray(result.content)) { + const parts = []; + for (const item of result.content) { + if (item && typeof item === 'object' && 'text' in item) { + parts.push(String(item.text)); + } + else { + parts.push(String(item)); + } + } + return parts.join('\n'); + } + else { + return String(result.content); + } + } + } + // Direct result or unknown format + return String(result); + } + /** + * List available prompts from the MCP server + */ + async listPrompts() { + if (!this._connected) { + await this.connect(); + } + try { + const result = await this.client.request(ListPromptsRequestSchema, {}); + this._prompts = new Map(result.prompts.map((prompt) => [prompt.name, prompt])); + return Array.from(this._prompts.values()); + } + catch (error) { + logger.debug(`Server '${this.serverName}' does not support prompts`); + return []; + } + } + /** + * Get a prompt with arguments + */ + async getPrompt(name, args) { + if (!this._connected) { + await this.connect(); + } + try { + const result = await this.client.request(GetPromptRequestSchema, { + name, + arguments: args, + }); + return result; + } + catch (error) { + logger.error(`Failed to get prompt '${name}': ${error}`); + throw error; + } + } + /** + * Start health check monitoring + */ + _startHealthCheck() { + if (this.healthCheckIntervalSeconds <= 0 || this._healthCheckInterval) { + return; + } + this._healthCheckInterval = setInterval(async () => { + try { + await this._performHealthCheck(); + } + catch (error) { + logger.warning(`Health check failed for '${this.serverName}': ${error}`); + if (this.autoReconnect) { + await this._attemptReconnect(); + } + } + }, this.healthCheckIntervalSeconds * 1000); + } + /** + * Stop health check monitoring + */ + _stopHealthCheck() { + if (this._healthCheckInterval) { + clearInterval(this._healthCheckInterval); + this._healthCheckInterval = undefined; + } + } + /** + * Perform health check by listing tools + */ + async _performHealthCheck() { + if (!this._connected) { + return; + } + try { + await this.client.request(ListToolsRequestSchema, {}); + this._lastHealthCheck = Date.now() / 1000; + } + catch (error) { + this._connected = false; + throw error; + } + } + /** + * Attempt to reconnect to the server + */ + async _attemptReconnect() { + logger.info(`Attempting to reconnect to '${this.serverName}'...`); + this._connected = false; + try { + await this.connect(this.connectionTimeout); + logger.info(`✅ Reconnected to '${this.serverName}'`); + } + catch (error) { + logger.error(`Failed to reconnect to '${this.serverName}': ${error}`); + } + } + /** + * Get client statistics + */ + getStats() { + const uptime = this._lastConnectTime + ? Date.now() / 1000 - this._lastConnectTime + : undefined; + const successRate = this._toolCallCount > 0 ? 1 - this._errorCount / this._toolCallCount : 1; + return { + serverName: this.serverName, + connected: this._connected, + toolsDiscovered: this._tools.size, + promptsDiscovered: this._prompts.size, + toolCallCount: this._toolCallCount, + errorCount: this._errorCount, + successRate, + uptime, + lastHealthCheck: this._lastHealthCheck, + }; + } + /** + * Check if client is connected + */ + isConnected() { + return this._connected; + } + /** + * Reset statistics + */ + resetStats() { + this._toolCallCount = 0; + this._errorCount = 0; + logger.debug(`Reset statistics for '${this.serverName}'`); + } + // Async context manager support + async [Symbol.asyncDispose]() { + await this.disconnect(); + } +} diff --git a/dist/mcp/controller.d.ts b/dist/mcp/controller.d.ts new file mode 100644 index 00000000..60ba823e --- /dev/null +++ b/dist/mcp/controller.d.ts @@ -0,0 +1,45 @@ +import type { Registry } from '../tools/registry/service.js'; +import type { Tools } from '../tools/service.js'; +import { MCPClient, type MCPClientOptions } from './client.js'; +export interface MCPToolWrapperOptions { + serverName?: string; + env?: Record; + clientOptions?: MCPClientOptions; +} +export declare class MCPToolWrapper { + private registry; + private client; + constructor(registry: Registry, mcpCommand: string, mcpArgs?: string[], options?: MCPToolWrapperOptions); + connect(toolFilter?: string[], prefix?: string, tools?: Pick): Promise; + disconnect(): Promise; + getClient(): MCPClient; + getStats(): { + serverName: string; + connected: boolean; + toolsDiscovered: number; + promptsDiscovered: number; + toolCallCount: number; + errorCount: number; + successRate: number; + uptime?: number; + lastHealthCheck?: number; + }; +} +export declare function registerMcpTools(registry: Registry, mcpCommand: string, mcpArgs?: string[], options?: MCPToolWrapperOptions): Promise; +export interface AddMCPServerOptions { + serverName?: string; + env?: Record; + clientOptions?: MCPClientOptions; + toolFilter?: string[]; + prefix?: string; + tools?: Pick; +} +export declare class MCPController { + private clients; + private tools; + constructor(tools?: Pick | null); + setTools(tools: Pick): void; + addServer(command: string, args?: string[], options?: AddMCPServerOptions): Promise; + getClients(): MCPClient[]; + disconnectAll(): Promise; +} diff --git a/dist/mcp/controller.js b/dist/mcp/controller.js new file mode 100644 index 00000000..0a2dd8e3 --- /dev/null +++ b/dist/mcp/controller.js @@ -0,0 +1,63 @@ +import { createLogger } from '../logging-config.js'; +import { MCPClient } from './client.js'; +const logger = createLogger('browser_use.mcp.controller'); +export class MCPToolWrapper { + registry; + client; + constructor(registry, mcpCommand, mcpArgs = [], options = {}) { + this.registry = registry; + this.client = new MCPClient(options.serverName ?? 'browser-use-mcp-tools', mcpCommand, mcpArgs, options.env, options.clientOptions); + } + async connect(toolFilter, prefix, tools) { + if (!this.client.isConnected()) { + await this.client.connect(); + } + const targetTools = tools ?? { registry: this.registry }; + await this.client.registerToTools(targetTools, toolFilter, prefix); + } + async disconnect() { + await this.client.disconnect(); + } + getClient() { + return this.client; + } + getStats() { + return this.client.getStats(); + } +} +export async function registerMcpTools(registry, mcpCommand, mcpArgs = [], options = {}) { + const wrapper = new MCPToolWrapper(registry, mcpCommand, mcpArgs, options); + await wrapper.connect(); + return wrapper; +} +// Backward-compatible helper that can manage multiple MCP clients. +export class MCPController { + clients = []; + tools; + constructor(tools = null) { + this.tools = tools; + } + setTools(tools) { + this.tools = tools; + } + async addServer(command, args = [], options = {}) { + const client = new MCPClient(options.serverName ?? `browser-use-client-${this.clients.length + 1}`, command, args, options.env, options.clientOptions); + await client.connect(); + const targetTools = options.tools ?? this.tools; + if (targetTools) { + await client.registerToTools(targetTools, options.toolFilter, options.prefix); + } + else { + logger.warning('MCPController.addServer connected but skipped tool registration because no Tools instance was provided'); + } + this.clients.push(client); + return client; + } + getClients() { + return [...this.clients]; + } + async disconnectAll() { + await Promise.all(this.clients.map((client) => client.disconnect())); + this.clients = []; + } +} diff --git a/dist/mcp/index.d.ts b/dist/mcp/index.d.ts new file mode 100644 index 00000000..80e61143 --- /dev/null +++ b/dist/mcp/index.d.ts @@ -0,0 +1,3 @@ +export * from './server.js'; +export * from './client.js'; +export * from './controller.js'; diff --git a/dist/mcp/index.js b/dist/mcp/index.js new file mode 100644 index 00000000..80e61143 --- /dev/null +++ b/dist/mcp/index.js @@ -0,0 +1,3 @@ +export * from './server.js'; +export * from './client.js'; +export * from './controller.js'; diff --git a/dist/mcp/server.d.ts b/dist/mcp/server.d.ts new file mode 100644 index 00000000..ef8ccb49 --- /dev/null +++ b/dist/mcp/server.d.ts @@ -0,0 +1,150 @@ +/** + * MCP Server for browser-use - exposes browser automation capabilities via Model Context Protocol. + * + * This server provides tools for: + * - Running autonomous browser tasks with an AI agent + * - Direct browser control (navigation, clicking, typing, etc.) + * - Content extraction from web pages + * - File system operations + * + * Usage: + * npx browser-use --mcp + * + * Or as an MCP server in Claude Desktop or other MCP clients: + * { + * "mcpServers": { + * "browser-use": { + * "command": "npx", + * "args": ["browser-use", "--mcp"], + * "env": { + * "OPENAI_API_KEY": "sk-proj-1234567890" + * } + * } + * } + * } + */ +import { z } from 'zod'; +import type { Controller } from '../controller/service.js'; +import { BrowserSession } from '../browser/session.js'; +export interface MCPPromptTemplate { + name: string; + description: string; + arguments?: Array<{ + name: string; + description?: string; + required?: boolean; + }>; + template: (args: Record) => string; +} +export declare class MCPServer { + private server; + private tools; + private prompts; + private config; + private browserSession; + private controller; + private llm; + private fileSystem; + private startTime; + private isRunning; + private toolExecutionCount; + private errorCount; + private abortController; + private activeSessions; + private sessionTimeoutMinutes; + private sessionCleanupInterval; + constructor(name: string, version: string); + private resolvePath; + private getDefaultProfileConfig; + private getDefaultLlmConfig; + private isPlaceholderOpenAiApiKey; + private seedOpenAiApiKeyFromConfig; + private createLlmFromModelName; + private sanitizeProfileConfig; + private buildDirectSessionProfile; + private buildRetryProfile; + private initializeLlmForDirectTools; + private initializeFileSystem; + private formatRetryResult; + private trackSession; + private updateSessionActivity; + private serializeTrackedSessions; + private shutdownSession; + private closeSessionById; + private closeAllTrackedSessions; + private cleanupExpiredSessions; + private startSessionCleanupLoop; + private stopSessionCleanupLoop; + private formatToolResult; + private setupHandlers; + private ensureController; + private ensureBrowserSession; + private executeControllerAction; + private registerCoreBrowserTools; + /** + * Register default prompts for common browser automation tasks + */ + private registerDefaultPrompts; + /** + * Register a tool with the MCP server + */ + registerTool(name: string, description: string, inputSchema: z.ZodType | Record, handler: (args: any) => Promise): void; + /** + * Register all Controller actions as MCP tools + */ + registerControllerActions(controller: Controller): Promise; + /** + * Initialize the browser session + */ + initBrowserSession(browserSession: BrowserSession): Promise; + /** + * Start the MCP server + */ + start(): Promise; + /** + * Stop the MCP server and cleanup resources + */ + stop(): Promise; + /** + * Register a prompt template + */ + registerPrompt(prompt: MCPPromptTemplate): void; + /** + * Get the number of registered tools + */ + getToolCount(): number; + /** + * Get the number of registered prompts + */ + getPromptCount(): number; + /** + * Get server health status + */ + getHealth(): { + status: 'healthy' | 'degraded' | 'unhealthy'; + uptime: number; + toolExecutionCount: number; + errorCount: number; + errorRate: number; + browserSessionActive: boolean; + }; + /** + * Get server statistics + */ + getStats(): { + toolsRegistered: number; + promptsRegistered: number; + uptime: number; + executionCount: number; + errorCount: number; + successRate: number; + }; + /** + * Reset statistics + */ + resetStats(): void; + /** + * Check if server is running + */ + isServerRunning(): boolean; +} diff --git a/dist/mcp/server.js b/dist/mcp/server.js new file mode 100644 index 00000000..a05fdd32 --- /dev/null +++ b/dist/mcp/server.js @@ -0,0 +1,1020 @@ +/** + * MCP Server for browser-use - exposes browser automation capabilities via Model Context Protocol. + * + * This server provides tools for: + * - Running autonomous browser tasks with an AI agent + * - Direct browser control (navigation, clicking, typing, etc.) + * - Content extraction from web pages + * - File system operations + * + * Usage: + * npx browser-use --mcp + * + * Or as an MCP server in Claude Desktop or other MCP clients: + * { + * "mcpServers": { + * "browser-use": { + * "command": "npx", + * "args": ["browser-use", "--mcp"], + * "env": { + * "OPENAI_API_KEY": "sk-proj-1234567890" + * } + * } + * } + * } + */ +import os from 'node:os'; +import path from 'node:path'; +import { Server } from '@modelcontextprotocol/sdk/server/index.js'; +import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; +import { CallToolRequestSchema, ListToolsRequestSchema, ListPromptsRequestSchema, GetPromptRequestSchema, } from '@modelcontextprotocol/sdk/types.js'; +import { z } from 'zod'; +import { createLogger } from '../logging-config.js'; +import { Controller as DefaultController } from '../controller/service.js'; +import { Agent } from '../agent/service.js'; +import { BrowserSession } from '../browser/session.js'; +import { BrowserStateRequestEvent } from '../browser/events.js'; +import { BrowserProfile } from '../browser/profile.js'; +import { FileSystem } from '../filesystem/file-system.js'; +import { getLlmByName } from '../llm/models.js'; +import { load_browser_use_config, get_default_llm, get_default_profile, } from '../config.js'; +import { zodSchemaToJsonSchema } from '../llm/schema.js'; +import { productTelemetry } from '../telemetry/service.js'; +import { MCPServerTelemetryEvent } from '../telemetry/views.js'; +import { get_browser_use_version } from '../utils.js'; +// Redirect console logs to stderr to prevent JSON-RPC interference +const originalLog = console.log; +const originalInfo = console.info; +const originalWarn = console.warn; +const originalError = console.error; +console.log = (...args) => console.error(...args); +console.info = (...args) => console.error(...args); +console.warn = (...args) => console.error(...args); +const logger = createLogger('browser_use.mcp.server'); +export class MCPServer { + server; + tools = {}; + prompts = new Map(); + config; + browserSession = null; + controller = null; + llm = null; + fileSystem = null; + startTime; + isRunning = false; + toolExecutionCount = 0; + errorCount = 0; + abortController = null; + activeSessions = new Map(); + sessionTimeoutMinutes = 10; + sessionCleanupInterval = null; + constructor(name, version) { + this.server = new Server({ + name, + version, + }, { + capabilities: { + tools: {}, + prompts: {}, + }, + }); + this.config = load_browser_use_config(); + const configuredTimeout = Number(process.env.BROWSER_USE_MCP_SESSION_TIMEOUT_MINUTES ?? '10'); + this.sessionTimeoutMinutes = + Number.isFinite(configuredTimeout) && configuredTimeout > 0 + ? configuredTimeout + : 10; + this.startTime = Date.now() / 1000; + this.setupHandlers(); + this.registerDefaultPrompts(); + this.controller = new DefaultController(); + this.registerControllerActions(this.controller); + this.registerCoreBrowserTools(); + } + resolvePath(input) { + const expanded = input.replace(/^~(?=$|\/|\\)/, os.homedir()); + return path.resolve(expanded); + } + getDefaultProfileConfig() { + const profile = get_default_profile(this.config); + return profile && typeof profile === 'object' ? { ...profile } : {}; + } + getDefaultLlmConfig() { + const llm = get_default_llm(this.config); + return llm && typeof llm === 'object' ? { ...llm } : {}; + } + isPlaceholderOpenAiApiKey(apiKey) { + const normalized = apiKey.trim().toLowerCase(); + return (normalized === 'your-openai-api-key-here' || + normalized === 'your-openai-api-key'); + } + seedOpenAiApiKeyFromConfig(llmConfig) { + if (typeof process.env.OPENAI_API_KEY === 'string' && + process.env.OPENAI_API_KEY.trim()) { + return; + } + const configuredApiKey = typeof llmConfig.api_key === 'string' ? llmConfig.api_key.trim() : ''; + if (!configuredApiKey || this.isPlaceholderOpenAiApiKey(configuredApiKey)) { + return; + } + process.env.OPENAI_API_KEY = configuredApiKey; + } + createLlmFromModelName(modelName, llmConfig) { + this.seedOpenAiApiKeyFromConfig(llmConfig); + return getLlmByName(modelName); + } + sanitizeProfileConfig(profileConfig) { + const sanitized = { ...profileConfig }; + delete sanitized.id; + delete sanitized.default; + delete sanitized.created_at; + return sanitized; + } + buildDirectSessionProfile(profileConfig) { + const merged = { + downloads_path: '~/Downloads/browser-use-mcp', + wait_between_actions: 0.1, + keep_alive: true, + user_data_dir: '~/.config/browseruse/profiles/default', + is_mobile: false, + device_scale_factor: 1.0, + disable_security: false, + headless: false, + ...this.sanitizeProfileConfig(profileConfig), + }; + if (typeof merged.user_data_dir === 'string') { + merged.user_data_dir = this.resolvePath(merged.user_data_dir); + } + if (typeof merged.downloads_path === 'string') { + merged.downloads_path = this.resolvePath(merged.downloads_path); + } + if (Array.isArray(merged.allowed_domains)) { + merged.allowed_domains = merged.allowed_domains + .map((entry) => String(entry).trim()) + .filter(Boolean); + } + if (Array.isArray(merged.prohibited_domains)) { + merged.prohibited_domains = merged.prohibited_domains + .map((entry) => String(entry).trim()) + .filter(Boolean); + } + return new BrowserProfile(merged); + } + buildRetryProfile(profileConfig, allowedDomains) { + const merged = { + ...this.sanitizeProfileConfig(profileConfig), + }; + if (allowedDomains !== undefined) { + merged.allowed_domains = allowedDomains; + } + if (merged.keep_alive == null) { + merged.keep_alive = false; + } + if (typeof merged.user_data_dir === 'string') { + merged.user_data_dir = this.resolvePath(merged.user_data_dir); + } + if (typeof merged.downloads_path === 'string') { + merged.downloads_path = this.resolvePath(merged.downloads_path); + } + if (Array.isArray(merged.allowed_domains)) { + merged.allowed_domains = merged.allowed_domains + .map((entry) => String(entry).trim()) + .filter(Boolean); + } + if (Array.isArray(merged.prohibited_domains)) { + merged.prohibited_domains = merged.prohibited_domains + .map((entry) => String(entry).trim()) + .filter(Boolean); + } + return new BrowserProfile(merged); + } + initializeLlmForDirectTools() { + if (this.llm) { + return; + } + const llmConfig = this.getDefaultLlmConfig(); + const model = typeof llmConfig.model === 'string' && llmConfig.model.trim() + ? llmConfig.model.trim() + : 'gpt-4o-mini'; + try { + this.llm = this.createLlmFromModelName(model, llmConfig); + } + catch (error) { + logger.debug(`Skipping MCP direct-tools LLM initialization for model "${model}": ${error instanceof Error ? error.message : String(error)}`); + } + } + initializeFileSystem(profileConfig) { + if (this.fileSystem) { + return; + } + const configuredPath = typeof profileConfig.file_system_path === 'string' + ? profileConfig.file_system_path + : '~/.browser-use-mcp'; + this.fileSystem = new FileSystem(this.resolvePath(configuredPath)); + } + formatRetryResult(history) { + const results = []; + const steps = Array.isArray(history?.history) || + typeof history?.number_of_steps === 'function' + ? typeof history?.number_of_steps === 'function' + ? history.number_of_steps() + : history.history.length + : 0; + results.push(`Task completed in ${steps} steps`); + results.push(`Success: ${String(history?.is_successful?.())}`); + const finalResult = history?.final_result?.(); + if (finalResult) { + results.push(`\nFinal result:\n${finalResult}`); + } + const errors = Array.isArray(history?.errors?.()) + ? history.errors().filter((entry) => entry != null) + : []; + if (errors.length > 0) { + results.push(`\nErrors encountered:\n${JSON.stringify(errors, null, 2)}`); + } + const urls = Array.isArray(history?.urls?.()) + ? history + .urls() + .filter((entry) => entry != null) + .map((entry) => String(entry)) + : []; + if (urls.length > 0) { + results.push(`\nURLs visited: ${urls.join(', ')}`); + } + return results.join('\n'); + } + trackSession(session) { + const now = Date.now() / 1000; + const existing = this.activeSessions.get(session.id); + if (existing) { + existing.session = session; + existing.last_activity = now; + return; + } + this.activeSessions.set(session.id, { + session, + created_at: now, + last_activity: now, + }); + } + updateSessionActivity(session) { + if (!session) { + return; + } + const tracked = this.activeSessions.get(session.id); + if (!tracked) { + this.trackSession(session); + return; + } + tracked.last_activity = Date.now() / 1000; + } + serializeTrackedSessions() { + const now = Date.now() / 1000; + return Array.from(this.activeSessions.entries()).map(([session_id, tracked]) => ({ + session_id, + created_at: tracked.created_at, + last_activity: tracked.last_activity, + age_minutes: (now - tracked.created_at) / 60, + active: Boolean(tracked.session?.initialized), + current_session: this.browserSession?.id === session_id, + })); + } + async shutdownSession(session) { + const withKill = session; + if (typeof withKill.kill === 'function') { + await withKill.kill(); + return; + } + if (typeof session.stop === 'function') { + await session.stop(); + } + } + async closeSessionById(sessionId) { + const tracked = this.activeSessions.get(sessionId); + if (!tracked) { + return { + session_id: sessionId, + closed: false, + message: `Session ${sessionId} not found`, + }; + } + try { + await this.shutdownSession(tracked.session); + this.activeSessions.delete(sessionId); + if (this.browserSession?.id === sessionId) { + this.browserSession = null; + } + return { + session_id: sessionId, + closed: true, + message: `Closed session ${sessionId}`, + }; + } + catch (error) { + return { + session_id: sessionId, + closed: false, + message: `Failed to close session ${sessionId}: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } + async closeAllTrackedSessions() { + const sessionIds = Array.from(this.activeSessions.keys()); + const results = await Promise.all(sessionIds.map((sessionId) => this.closeSessionById(sessionId))); + const closedCount = results.filter((result) => result.closed).length; + return { + closed_count: closedCount, + total_count: sessionIds.length, + results, + }; + } + async cleanupExpiredSessions() { + const now = Date.now() / 1000; + const timeoutSeconds = this.sessionTimeoutMinutes * 60; + const expiredSessionIds = []; + for (const [sessionId, tracked] of this.activeSessions.entries()) { + if (now - tracked.last_activity > timeoutSeconds) { + expiredSessionIds.push(sessionId); + } + } + for (const sessionId of expiredSessionIds) { + const result = await this.closeSessionById(sessionId); + if (!result.closed) { + logger.warning(result.message); + } + } + } + startSessionCleanupLoop() { + if (this.sessionCleanupInterval) { + return; + } + this.sessionCleanupInterval = setInterval(() => { + this.cleanupExpiredSessions().catch((error) => { + logger.warning(`MCP session cleanup failed: ${error instanceof Error ? error.message : String(error)}`); + }); + }, 120_000); + // Do not keep the Node process alive solely because of the cleanup loop. + this.sessionCleanupInterval.unref?.(); + } + stopSessionCleanupLoop() { + if (!this.sessionCleanupInterval) { + return; + } + clearInterval(this.sessionCleanupInterval); + this.sessionCleanupInterval = null; + } + formatToolResult(toolName, result) { + if (toolName === 'browser_get_state' && + result && + typeof result === 'object' && + !Array.isArray(result)) { + const payload = { + ...result, + }; + const screenshot = typeof payload.screenshot === 'string' && payload.screenshot.trim() + ? payload.screenshot + : null; + delete payload.screenshot; + const pageInfo = payload.page_info && + typeof payload.page_info === 'object' && + !Array.isArray(payload.page_info) + ? payload.page_info + : null; + const viewportWidth = typeof pageInfo?.viewport_width === 'number' + ? pageInfo.viewport_width + : null; + const viewportHeight = typeof pageInfo?.viewport_height === 'number' + ? pageInfo.viewport_height + : null; + if (screenshot && + viewportWidth !== null && + viewportHeight !== null && + payload.screenshot_dimensions == null) { + payload.screenshot_dimensions = { + width: viewportWidth, + height: viewportHeight, + }; + } + const content = [ + { + type: 'text', + text: JSON.stringify(payload, null, 2), + }, + ]; + if (screenshot) { + content.push({ + type: 'image', + data: screenshot, + mimeType: 'image/png', + }); + } + return { content }; + } + return { + content: [ + { + type: 'text', + text: typeof result === 'string' + ? result + : JSON.stringify(result, null, 2), + }, + ], + }; + } + setupHandlers() { + // List available tools + this.server.setRequestHandler(ListToolsRequestSchema, async () => { + return { + tools: Object.entries(this.tools).map(([name, tool]) => ({ + name, + description: tool.description, + inputSchema: tool.inputSchema, + })), + }; + }); + // Execute tool + this.server.setRequestHandler(CallToolRequestSchema, async (request) => { + const startTime = Date.now() / 1000; + let errorMsg = null; + try { + const tool = this.tools[request.params.name]; + if (!tool) { + throw new Error(`Tool not found: ${request.params.name}`); + } + logger.debug(`Executing tool: ${request.params.name}`); + this.toolExecutionCount++; + const result = await tool.handler(request.params.arguments || {}); + return this.formatToolResult(request.params.name, result); + } + catch (error) { + this.errorCount++; + errorMsg = error instanceof Error ? error.message : String(error); + logger.error(`Tool execution failed: ${errorMsg}`); + return { + content: [ + { + type: 'text', + text: `Error: ${errorMsg}`, + }, + ], + isError: true, + }; + } + finally { + // Capture telemetry for tool calls + const duration = Date.now() / 1000 - startTime; + productTelemetry.capture(new MCPServerTelemetryEvent({ + version: get_browser_use_version(), + action: 'tool_call', + tool_name: request.params.name, + duration_seconds: duration, + error_message: errorMsg, + })); + } + }); + // List available prompts + this.server.setRequestHandler(ListPromptsRequestSchema, async () => { + return { + prompts: Array.from(this.prompts.values()).map((prompt) => ({ + name: prompt.name, + description: prompt.description, + arguments: prompt.arguments, + })), + }; + }); + // Get prompt with arguments + this.server.setRequestHandler(GetPromptRequestSchema, async (request) => { + const prompt = this.prompts.get(request.params.name); + if (!prompt) { + throw new Error(`Prompt not found: ${request.params.name}`); + } + const args = request.params.arguments || {}; + const message = prompt.template(args); + return { + messages: [ + { + role: 'user', + content: { + type: 'text', + text: message, + }, + }, + ], + }; + }); + } + async ensureController() { + if (!this.controller) { + this.controller = new DefaultController(); + this.registerControllerActions(this.controller); + } + return this.controller; + } + async ensureBrowserSession() { + if (!this.browserSession) { + const profileConfig = this.getDefaultProfileConfig(); + const profile = this.buildDirectSessionProfile(profileConfig); + this.browserSession = new BrowserSession({ browser_profile: profile }); + this.trackSession(this.browserSession); + this.initializeLlmForDirectTools(); + this.initializeFileSystem(profileConfig); + } + if (!this.browserSession.initialized) { + await this.browserSession.start(); + } + this.trackSession(this.browserSession); + this.updateSessionActivity(this.browserSession); + return this.browserSession; + } + async executeControllerAction(actionName, args) { + const controller = await this.ensureController(); + const browserSession = await this.ensureBrowserSession(); + this.updateSessionActivity(browserSession); + if (actionName === 'extract_structured_data' && !this.llm) { + throw new Error('LLM not initialized. Set provider API key env vars and configure BROWSER_USE_LLM_MODEL/DEFAULT_LLM to a supported model.'); + } + return await controller.registry.execute_action(actionName, args, { + browser_session: browserSession, + page_extraction_llm: this.llm, + file_system: this.fileSystem, + available_file_paths: Array.isArray(browserSession.downloaded_files) + ? [...browserSession.downloaded_files] + : null, + context: undefined, + }); + } + registerCoreBrowserTools() { + this.registerTool('browser_navigate', 'Navigate to a URL in the browser', z.object({ + url: z.string(), + new_tab: z.boolean().default(false), + }), async (args) => this.executeControllerAction('go_to_url', { + url: String(args?.url ?? ''), + new_tab: Boolean(args?.new_tab), + })); + this.registerTool('browser_click', 'Click an element on the page by index from browser_get_state', z.object({ + index: z.number().int(), + new_tab: z.boolean().optional().default(false), + }), async (args) => { + const browserSession = await this.ensureBrowserSession(); + const index = Number(args?.index); + const openInNewTab = Boolean(args?.new_tab); + if (!openInNewTab) { + return this.executeControllerAction('click_element_by_index', { + index, + }); + } + const element = await browserSession.get_dom_element_by_index(index); + if (!element) { + throw new Error(`Element with index ${index} not found`); + } + const href = element?.attributes?.href; + if (typeof href === 'string' && href.trim()) { + const currentPage = await browserSession.get_current_page(); + const currentUrl = typeof currentPage?.url === 'function' ? currentPage.url() : ''; + let targetUrl = href.trim(); + try { + if (currentUrl) { + targetUrl = new URL(targetUrl, currentUrl).toString(); + } + } + catch { + // Keep the original href if URL resolution fails. + } + await browserSession.create_new_tab(targetUrl); + const tabIndex = typeof browserSession.active_tab_index === 'number' + ? browserSession.active_tab_index + : null; + if (tabIndex !== null) { + return `Clicked element ${index} and opened in new tab #${tabIndex}: ${targetUrl}`; + } + return `Clicked element ${index} and opened new tab: ${targetUrl}`; + } + const locator = typeof browserSession.get_locate_element === 'function' + ? await browserSession.get_locate_element(element) + : null; + if (locator && typeof locator.click === 'function') { + const modifier = process.platform === 'darwin' ? 'Meta' : 'Control'; + await locator.click({ modifiers: [modifier] }); + await new Promise((resolve) => setTimeout(resolve, 500)); + return `Clicked element ${index} with ${modifier} key (new tab if supported)`; + } + // Fallback: if no href exists, perform a normal click. + return this.executeControllerAction('click_element_by_index', { + index, + }); + }); + this.registerTool('browser_type', 'Type text into an input field by index from browser_get_state', z.object({ + index: z.number().int(), + text: z.string(), + }), async (args) => this.executeControllerAction('input_text', { + index: Number(args?.index), + text: String(args?.text ?? ''), + })); + this.registerTool('browser_get_state', 'Get the current state of the page including interactive elements', z + .object({ + include_screenshot: z.boolean().default(false), + include_recent_events: z.boolean().default(false), + }) + .default({ include_screenshot: false, include_recent_events: false }), async (args) => { + const browserSession = await this.ensureBrowserSession(); + let state = null; + if (typeof browserSession.dispatch_browser_event === 'function') { + const dispatchResult = await browserSession.dispatch_browser_event(new BrowserStateRequestEvent({ + include_dom: true, + include_screenshot: Boolean(args?.include_screenshot), + include_recent_events: Boolean(args?.include_recent_events), + })); + state = dispatchResult?.event?.event_result ?? null; + } + if (!state) { + state = await browserSession.get_browser_state_with_recovery({ + include_screenshot: Boolean(args?.include_screenshot), + include_recent_events: Boolean(args?.include_recent_events), + cache_clickable_elements_hashes: true, + }); + } + return { + url: state.url, + title: state.title, + tabs: state.tabs, + page_info: state.page_info, + pixels_above: state.pixels_above, + pixels_below: state.pixels_below, + browser_errors: state.browser_errors, + loading_status: state.loading_status, + recent_events: state.recent_events, + pending_network_requests: state.pending_network_requests, + pagination_buttons: state.pagination_buttons, + closed_popup_messages: state.closed_popup_messages, + screenshot: state.screenshot, + interactive_elements: state.element_tree.clickable_elements_to_string(), + interactive_count: Object.keys(state.selector_map ?? {}).length, + }; + }); + this.registerTool('browser_extract_content', 'Extract structured content from the current page', z.object({ + query: z.string(), + extract_links: z.boolean().default(false), + }), async (args) => this.executeControllerAction('extract_structured_data', { + query: String(args?.query ?? ''), + extract_links: Boolean(args?.extract_links), + })); + this.registerTool('browser_scroll', 'Scroll the page up or down', z + .object({ + direction: z.enum(['up', 'down']).default('down'), + }) + .default({ direction: 'down' }), async (args) => this.executeControllerAction('scroll', { + down: (args?.direction ?? 'down') !== 'up', + num_pages: 1, + })); + this.registerTool('browser_go_back', 'Go back to the previous page', z.object({}).strict(), async () => this.executeControllerAction('go_back', {})); + this.registerTool('browser_list_tabs', 'List all open tabs', z.object({}).strict(), async () => { + const browserSession = await this.ensureBrowserSession(); + return browserSession.get_tabs_info(); + }); + this.registerTool('browser_switch_tab', 'Switch to a tab by tab_id (or page_id/tab_index for compatibility)', z + .object({ + tab_id: z.string().trim().length(4).optional(), + page_id: z.number().int().optional(), + tab_index: z.number().int().optional(), + }) + .refine((value) => value.tab_id != null || + value.page_id != null || + value.tab_index != null, { message: 'Provide tab_id, page_id, or tab_index' }), async (args) => { + const tabId = typeof args?.tab_id === 'string' && args.tab_id.trim() + ? args.tab_id.trim() + : null; + if (tabId) { + return this.executeControllerAction('switch_tab', { tab_id: tabId }); + } + const pageId = typeof args?.page_id === 'number' && Number.isFinite(args.page_id) + ? Number(args.page_id) + : Number(args?.tab_index); + return this.executeControllerAction('switch_tab', { + page_id: pageId, + }); + }); + this.registerTool('browser_close_tab', 'Close a tab by tab_id (or page_id/tab_index for compatibility)', z + .object({ + tab_id: z.string().trim().length(4).optional(), + page_id: z.number().int().optional(), + tab_index: z.number().int().optional(), + }) + .refine((value) => value.tab_id != null || + value.page_id != null || + value.tab_index != null, { message: 'Provide tab_id, page_id, or tab_index' }), async (args) => { + const tabId = typeof args?.tab_id === 'string' && args.tab_id.trim() + ? args.tab_id.trim() + : null; + if (tabId) { + return this.executeControllerAction('close_tab', { tab_id: tabId }); + } + const pageId = typeof args?.page_id === 'number' && Number.isFinite(args.page_id) + ? Number(args.page_id) + : Number(args?.tab_index); + return this.executeControllerAction('close_tab', { + page_id: pageId, + }); + }); + this.registerTool('browser_list_sessions', 'List active browser sessions managed by this MCP server', z.object({}).strict(), async () => { + return this.serializeTrackedSessions(); + }); + this.registerTool('browser_close_session', 'Close a specific browser session by session_id', z.object({ + session_id: z.string().trim().min(1), + }), async (args) => this.closeSessionById(String(args?.session_id ?? ''))); + this.registerTool('browser_close_all', 'Close all active browser sessions managed by this MCP server', z.object({}).strict(), async () => this.closeAllTrackedSessions()); + this.registerTool('retry_with_browser_use_agent', 'Retry a complex task with the browser-use autonomous agent', z.object({ + task: z.string(), + max_steps: z.number().int().optional().default(100), + model: z.string().optional(), + allowed_domains: z.array(z.string()).optional().default([]), + use_vision: z.boolean().optional().default(true), + }), async (args) => { + const task = String(args?.task ?? '').trim(); + if (!task) { + throw new Error('task is required'); + } + const requestedModel = typeof args?.model === 'string' ? args.model.trim() : ''; + const maxSteps = Number(args?.max_steps ?? 100); + const useVision = Boolean(args?.use_vision ?? true); + const allowedDomains = Array.isArray(args?.allowed_domains) + ? args.allowed_domains + .map((entry) => String(entry).trim()) + .filter(Boolean) + : []; + const llmConfig = this.getDefaultLlmConfig(); + const configuredModel = typeof llmConfig.model === 'string' && llmConfig.model.trim() + ? llmConfig.model.trim() + : 'gpt-4o'; + const llmModel = requestedModel || configuredModel; + let llm; + try { + llm = this.createLlmFromModelName(llmModel, llmConfig); + } + catch (error) { + return `Error: Failed to initialize LLM "${llmModel}": ${error instanceof Error ? error.message : String(error)}`; + } + const profileConfig = this.getDefaultProfileConfig(); + const profile = this.buildRetryProfile(profileConfig, allowedDomains); + const retryBrowserSession = new BrowserSession({ + browser_profile: profile, + }); + const agent = new Agent({ + task, + llm, + browser_session: retryBrowserSession, + use_vision: useVision, + }); + try { + const history = await agent.run(maxSteps); + return this.formatRetryResult(history); + } + catch (error) { + const message = error instanceof Error ? error.message : String(error); + return `Agent task failed: ${message}`; + } + finally { + await agent.close(); + } + }); + } + /** + * Register default prompts for common browser automation tasks + */ + registerDefaultPrompts() { + // Scrape data prompt + this.registerPrompt({ + name: 'scrape_data', + description: 'Extract structured data from a website', + arguments: [ + { name: 'url', description: 'URL to scrape', required: true }, + { + name: 'data_type', + description: 'Type of data to extract', + required: true, + }, + ], + template: (args) => `Use browser_navigate to go to ${args.url}, then use browser_extract_content to extract ${args.data_type}. If the page requires interaction, use browser_get_state to find elements and browser_click/browser_type as needed.`, + }); + // Fill form prompt + this.registerPrompt({ + name: 'fill_form', + description: 'Fill out and submit a web form', + arguments: [ + { name: 'url', description: 'URL of the form', required: true }, + { + name: 'field_data', + description: 'JSON object with field values', + required: true, + }, + ], + template: (args) => `Navigate to ${args.url}, use browser_get_state to identify form fields, then use browser_type to fill in: ${args.field_data}. Finally, click the submit button.`, + }); + // Multi-step task prompt + this.registerPrompt({ + name: 'multi_step_task', + description: 'Execute a complex multi-step task', + arguments: [ + { + name: 'task_description', + description: 'Detailed description of the task', + required: true, + }, + { + name: 'max_steps', + description: 'Maximum number of steps (default: 100)', + required: false, + }, + ], + template: (args) => `Use retry_with_browser_use_agent with task: '${args.task_description}'. Set max_steps=${args.max_steps || '100'} and use_vision=true for better understanding.`, + }); + // Research topic prompt + this.registerPrompt({ + name: 'research_topic', + description: 'Research a topic across multiple websites', + arguments: [ + { name: 'topic', description: 'Topic to research', required: true }, + { + name: 'sites', + description: 'Comma-separated list of websites', + required: true, + }, + ], + template: (args) => `Open multiple tabs using browser_navigate with new_tab=true for sites: ${args.sites}. Use browser_extract_content on each to gather information about ${args.topic}. Switch between tabs with browser_switch_tab.`, + }); + } + /** + * Register a tool with the MCP server + */ + registerTool(name, description, inputSchema, handler) { + this.tools[name] = { + description, + inputSchema: inputSchema instanceof z.ZodType + ? zodSchemaToJsonSchema(inputSchema) + : inputSchema, + handler, + }; + logger.debug(`Registered tool: ${name}`); + } + /** + * Register all Controller actions as MCP tools + */ + async registerControllerActions(controller) { + this.controller = controller; + // Get all registered actions from the controller + const actions = controller.registry.get_all_actions(); + for (const [actionName, actionInfo] of actions.entries()) { + // Create a wrapper for the action + const handler = async (args) => { + return this.executeControllerAction(actionName, args || {}); + }; + // Register the action as a tool + this.registerTool(actionName, actionInfo.description || `Execute ${actionName} action`, actionInfo.paramSchema ?? z.object({}).strict(), handler); + } + logger.info(`✅ Registered ${actions.size} controller actions as MCP tools`); + } + /** + * Initialize the browser session + */ + async initBrowserSession(browserSession) { + this.browserSession = browserSession; + this.trackSession(browserSession); + await this.browserSession.start(); + this.trackSession(this.browserSession); + this.updateSessionActivity(this.browserSession); + logger.info('Browser session initialized'); + } + /** + * Start the MCP server + */ + async start() { + if (this.isRunning) { + logger.warning('MCP Server is already running'); + return; + } + // Capture telemetry for server start + productTelemetry.capture(new MCPServerTelemetryEvent({ + version: get_browser_use_version(), + action: 'start', + })); + try { + const transport = new StdioServerTransport(); + await this.server.connect(transport); + this.isRunning = true; + this.startSessionCleanupLoop(); + logger.info(`🔌 MCP Server started (${this.getToolCount()} tools, ${this.getPromptCount()} prompts registered)`); + } + catch (error) { + this.isRunning = false; + logger.error(`Failed to start MCP server: ${error}`); + throw error; + } + } + /** + * Stop the MCP server and cleanup resources + */ + async stop() { + if (!this.isRunning) { + logger.warning('MCP Server is not running'); + return; + } + try { + this.isRunning = false; + this.stopSessionCleanupLoop(); + // Cancel any pending operations + if (this.abortController) { + this.abortController.abort(); + this.abortController = null; + } + // Close browser session if active + if (this.activeSessions.size > 0) { + await this.closeAllTrackedSessions(); + this.browserSession = null; + logger.info('Browser session closed'); + } + // Capture telemetry for server stop + const duration = Date.now() / 1000 - this.startTime; + productTelemetry.capture(new MCPServerTelemetryEvent({ + version: get_browser_use_version(), + action: 'stop', + duration_seconds: duration, + })); + productTelemetry.flush(); + const stats = this.getStats(); + logger.info(`🔌 MCP Server stopped (uptime: ${Math.floor(stats.uptime)}s, executions: ${stats.executionCount}, success rate: ${(stats.successRate * 100).toFixed(1)}%)`); + } + catch (error) { + logger.error(`Error stopping MCP server: ${error}`); + } + } + /** + * Register a prompt template + */ + registerPrompt(prompt) { + this.prompts.set(prompt.name, prompt); + logger.debug(`Registered prompt: ${prompt.name}`); + } + /** + * Get the number of registered tools + */ + getToolCount() { + return Object.keys(this.tools).length; + } + /** + * Get the number of registered prompts + */ + getPromptCount() { + return this.prompts.size; + } + /** + * Get server health status + */ + getHealth() { + const uptime = Date.now() / 1000 - this.startTime; + const errorRate = this.toolExecutionCount > 0 + ? this.errorCount / this.toolExecutionCount + : 0; + let status = 'healthy'; + if (errorRate > 0.5) { + status = 'unhealthy'; + } + else if (errorRate > 0.2) { + status = 'degraded'; + } + return { + status, + uptime, + toolExecutionCount: this.toolExecutionCount, + errorCount: this.errorCount, + errorRate, + browserSessionActive: this.browserSession !== null, + }; + } + /** + * Get server statistics + */ + getStats() { + const health = this.getHealth(); + return { + toolsRegistered: this.getToolCount(), + promptsRegistered: this.getPromptCount(), + uptime: health.uptime, + executionCount: this.toolExecutionCount, + errorCount: this.errorCount, + successRate: health.toolExecutionCount > 0 ? 1 - health.errorRate : 1, + }; + } + /** + * Reset statistics + */ + resetStats() { + this.toolExecutionCount = 0; + this.errorCount = 0; + logger.info('Statistics reset'); + } + /** + * Check if server is running + */ + isServerRunning() { + return this.isRunning; + } +} diff --git a/dist/observability-decorators.d.ts b/dist/observability-decorators.d.ts new file mode 100644 index 00000000..e480eb0e --- /dev/null +++ b/dist/observability-decorators.d.ts @@ -0,0 +1,158 @@ +/** + * Observability Decorators and Utilities + * + * Provides debugging and performance tracking capabilities + * for browser automation operations. + * + * Note: TypeScript decorators work differently than Python. + * These are implemented as wrapper functions that can be used + * in a similar way to decorators. + */ +import { createLogger } from './logging-config.js'; +/** + * Debug observation configuration + */ +export interface ObserveDebugOptions { + /** Enable detailed logging */ + verbose?: boolean; + /** Log function arguments */ + logArgs?: boolean; + /** Log return values */ + logResult?: boolean; + /** Log execution time */ + logTime?: boolean; + /** Custom logger instance */ + logger?: ReturnType; +} +/** + * Observe and debug async function execution + * Wraps an async function to add logging and debugging capabilities + * + * @example + * const debuggedFn = observeDebug(myAsyncFn, { verbose: true, logArgs: true }); + * await debuggedFn(arg1, arg2); + */ +export declare function observeDebug Promise>(fn: T, options?: ObserveDebugOptions): T; +/** + * Method decorator for observing debug (TypeScript experimental decorators) + * This requires "experimentalDecorators": true in tsconfig.json + * + * @example + * class MyClass { + * @observeDebugMethod({ verbose: true }) + * async myMethod(arg: string) { + * // method implementation + * } + * } + */ +export declare function observeDebugMethod(options?: ObserveDebugOptions): (target: any, propertyKey: string, descriptor: PropertyDescriptor) => PropertyDescriptor; +/** + * Performance tracking for async functions + * Combines time execution tracking with debug observation + * + * @example + * const trackedFn = trackPerformance(myAsyncFn, 'MyOperation'); + * await trackedFn(arg1, arg2); + */ +export declare function trackPerformance Promise>(fn: T, operationName?: string): T; +/** + * Comprehensive observability wrapper + * Combines debugging, performance tracking, and error handling + * + * @example + * const observedFn = withObservability(myAsyncFn, { + * name: 'CriticalOperation', + * debug: true, + * trackPerformance: true, + * onError: (error) => console.error('Operation failed:', error) + * }); + */ +export declare function withObservability Promise>(fn: T, options?: { + name?: string; + debug?: boolean; + debugOptions?: ObserveDebugOptions; + trackPerformance?: boolean; + onError?: (error: Error) => void; + onSuccess?: (result: any) => void; +}): T; +/** + * Create a debug trace for a series of operations + * Useful for tracking complex workflows + */ +type TraceOperation = { + name: string; + startTime: number; + endTime?: number; + duration?: number; + status: 'pending' | 'success' | 'error'; + error?: Error; +}; +export declare class OperationTrace { + private operations; + private logger; + private traceName; + constructor(traceName: string, customLogger?: ReturnType); + /** + * Start tracking an operation + */ + startOperation(name: string): void; + /** + * Mark an operation as completed successfully + */ + completeOperation(name: string): void; + /** + * Mark an operation as failed + */ + failOperation(name: string, error: Error): void; + /** + * Get trace summary + */ + getSummary(): { + traceName: string; + totalOperations: number; + successCount: number; + errorCount: number; + pendingCount: number; + totalDuration: number; + operations: TraceOperation[]; + }; + /** + * Log trace summary + */ + logSummary(): void; +} +/** + * Simple performance counter for tracking operation metrics + */ +export declare class PerformanceCounter { + private counters; + private logger; + constructor(customLogger?: ReturnType); + /** + * Record an operation execution + */ + record(operationName: string, durationMs: number): void; + /** + * Get statistics for an operation + */ + getStats(operationName: string): { + count: number; + avgTime: number; + minTime: number; + maxTime: number; + totalTime: number; + } | null; + /** + * Get all statistics + */ + getAllStats(): Map>; + /** + * Log statistics summary + */ + logSummary(): void; + /** + * Reset all counters + */ + reset(): void; +} +export {}; diff --git a/dist/observability-decorators.js b/dist/observability-decorators.js new file mode 100644 index 00000000..125d8740 --- /dev/null +++ b/dist/observability-decorators.js @@ -0,0 +1,286 @@ +/** + * Observability Decorators and Utilities + * + * Provides debugging and performance tracking capabilities + * for browser automation operations. + * + * Note: TypeScript decorators work differently than Python. + * These are implemented as wrapper functions that can be used + * in a similar way to decorators. + */ +import { createLogger } from './logging-config.js'; +import { time_execution_async } from './utils.js'; +const logger = createLogger('browser_use.observability'); +/** + * Observe and debug async function execution + * Wraps an async function to add logging and debugging capabilities + * + * @example + * const debuggedFn = observeDebug(myAsyncFn, { verbose: true, logArgs: true }); + * await debuggedFn(arg1, arg2); + */ +export function observeDebug(fn, options = {}) { + const { verbose = false, logArgs = true, logResult = false, logTime = true, logger: customLogger = logger, } = options; + const wrappedFn = async function (...args) { + const fnName = fn.name || 'anonymous'; + const startTime = Date.now(); + try { + // Log function entry + if (verbose) { + customLogger.debug(`→ Entering ${fnName}`); + } + // Log arguments + if (logArgs && args.length > 0) { + customLogger.debug(`${fnName} arguments:`, args); + } + // Execute function + const result = await fn.apply(this, args); + // Log execution time + if (logTime) { + const duration = Date.now() - startTime; + customLogger.debug(`${fnName} completed in ${duration}ms`); + } + // Log result + if (logResult) { + customLogger.debug(`${fnName} result:`, result); + } + // Log function exit + if (verbose) { + customLogger.debug(`← Exiting ${fnName}`); + } + return result; + } + catch (error) { + const duration = Date.now() - startTime; + customLogger.error(`${fnName} failed after ${duration}ms: ${error.message}`); + customLogger.debug(`${fnName} error stack:`, error.stack); + throw error; + } + }; + // Preserve function name + Object.defineProperty(wrappedFn, 'name', { + value: fn.name, + configurable: true, + }); + return wrappedFn; +} +/** + * Method decorator for observing debug (TypeScript experimental decorators) + * This requires "experimentalDecorators": true in tsconfig.json + * + * @example + * class MyClass { + * @observeDebugMethod({ verbose: true }) + * async myMethod(arg: string) { + * // method implementation + * } + * } + */ +export function observeDebugMethod(options = {}) { + return function (target, propertyKey, descriptor) { + const originalMethod = descriptor.value; + descriptor.value = observeDebug(originalMethod, { + ...options, + logger: options.logger || + createLogger(`${target.constructor.name}.${propertyKey}`), + }); + return descriptor; + }; +} +/** + * Performance tracking for async functions + * Combines time execution tracking with debug observation + * + * @example + * const trackedFn = trackPerformance(myAsyncFn, 'MyOperation'); + * await trackedFn(arg1, arg2); + */ +export function trackPerformance(fn, operationName) { + const name = operationName || fn.name || 'anonymous'; + return time_execution_async(name)(fn); +} +/** + * Comprehensive observability wrapper + * Combines debugging, performance tracking, and error handling + * + * @example + * const observedFn = withObservability(myAsyncFn, { + * name: 'CriticalOperation', + * debug: true, + * trackPerformance: true, + * onError: (error) => console.error('Operation failed:', error) + * }); + */ +export function withObservability(fn, options = {}) { + const { name = fn.name, debug = false, debugOptions = {}, trackPerformance: shouldTrackPerformance = true, onError, onSuccess, } = options; + let wrappedFn = fn; + // Apply debug observation if requested + if (debug) { + wrappedFn = observeDebug(wrappedFn, debugOptions); + } + // Apply performance tracking if requested + if (shouldTrackPerformance) { + wrappedFn = trackPerformance(wrappedFn, name); + } + // Add error/success callbacks + if (onError || onSuccess) { + const callbackFn = async function (...args) { + try { + const result = await wrappedFn.apply(this, args); + if (onSuccess) { + onSuccess(result); + } + return result; + } + catch (error) { + if (onError) { + onError(error); + } + throw error; + } + }; + wrappedFn = callbackFn; + } + return wrappedFn; +} +export class OperationTrace { + operations = []; + logger; + traceName; + constructor(traceName, customLogger) { + this.traceName = traceName; + this.logger = customLogger || createLogger(`trace.${traceName}`); + } + /** + * Start tracking an operation + */ + startOperation(name) { + this.operations.push({ + name, + startTime: Date.now(), + status: 'pending', + }); + this.logger.debug(`Started operation: ${name}`); + } + /** + * Mark an operation as completed successfully + */ + completeOperation(name) { + const op = this.operations.find((o) => o.name === name && o.status === 'pending'); + if (op) { + op.endTime = Date.now(); + op.duration = op.endTime - op.startTime; + op.status = 'success'; + this.logger.debug(`Completed operation: ${name} (${op.duration}ms)`); + } + } + /** + * Mark an operation as failed + */ + failOperation(name, error) { + const op = this.operations.find((o) => o.name === name && o.status === 'pending'); + if (op) { + op.endTime = Date.now(); + op.duration = op.endTime - op.startTime; + op.status = 'error'; + op.error = error; + this.logger.error(`Failed operation: ${name} (${op.duration}ms) - ${error.message}`); + } + } + /** + * Get trace summary + */ + getSummary() { + const successCount = this.operations.filter((o) => o.status === 'success').length; + const errorCount = this.operations.filter((o) => o.status === 'error').length; + const pendingCount = this.operations.filter((o) => o.status === 'pending').length; + const totalDuration = this.operations.reduce((sum, op) => sum + (op.duration || 0), 0); + return { + traceName: this.traceName, + totalOperations: this.operations.length, + successCount, + errorCount, + pendingCount, + totalDuration, + operations: this.operations, + }; + } + /** + * Log trace summary + */ + logSummary() { + const summary = this.getSummary(); + this.logger.info(`Trace "${this.traceName}" completed: ${summary.successCount} succeeded, ${summary.errorCount} failed, ${summary.pendingCount} pending (total: ${summary.totalDuration}ms)`); + } +} +/** + * Simple performance counter for tracking operation metrics + */ +export class PerformanceCounter { + counters = new Map(); + logger; + constructor(customLogger) { + this.logger = customLogger || createLogger('performance_counter'); + } + /** + * Record an operation execution + */ + record(operationName, durationMs) { + const current = this.counters.get(operationName) || { + count: 0, + totalTime: 0, + minTime: Infinity, + maxTime: 0, + }; + this.counters.set(operationName, { + count: current.count + 1, + totalTime: current.totalTime + durationMs, + minTime: Math.min(current.minTime, durationMs), + maxTime: Math.max(current.maxTime, durationMs), + }); + } + /** + * Get statistics for an operation + */ + getStats(operationName) { + const counter = this.counters.get(operationName); + if (!counter) { + return null; + } + return { + count: counter.count, + avgTime: counter.totalTime / counter.count, + minTime: counter.minTime, + maxTime: counter.maxTime, + totalTime: counter.totalTime, + }; + } + /** + * Get all statistics + */ + getAllStats() { + const stats = new Map(); + for (const [name] of this.counters) { + stats.set(name, this.getStats(name)); + } + return stats; + } + /** + * Log statistics summary + */ + logSummary() { + this.logger.info('Performance Counter Summary:'); + for (const [name, stats] of this.getAllStats()) { + if (stats) { + this.logger.info(` ${name}: count=${stats.count}, avg=${stats.avgTime.toFixed(2)}ms, min=${stats.minTime.toFixed(2)}ms, max=${stats.maxTime.toFixed(2)}ms`); + } + } + } + /** + * Reset all counters + */ + reset() { + this.counters.clear(); + this.logger.debug('Performance counters reset'); + } +} diff --git a/dist/observability.d.ts b/dist/observability.d.ts new file mode 100644 index 00000000..89cfda6e --- /dev/null +++ b/dist/observability.d.ts @@ -0,0 +1,23 @@ +type SpanType = 'DEFAULT' | 'LLM' | 'TOOL'; +export interface ObserveOptions { + name?: string | null; + ignoreInput?: boolean; + ignoreOutput?: boolean; + metadata?: Record | null; + spanType?: SpanType; + [key: string]: unknown; +} +type AnyFunc = (...args: any[]) => any; +type Decorator = (fn: T) => T; +export declare const observe: (options?: ObserveOptions) => Decorator; +export declare const observeDebug: (options?: ObserveOptions) => Decorator; +export declare const observe_debug: (options?: ObserveOptions) => Decorator; +export declare const isLmnrAvailable: () => boolean; +export declare const isDebugMode: () => boolean; +export declare const getObservabilityStatus: () => { + lmnrAvailable: boolean; + debugMode: boolean; + observeActive: boolean; + observeDebugActive: boolean; +}; +export {}; diff --git a/dist/observability.js b/dist/observability.js new file mode 100644 index 00000000..93e72241 --- /dev/null +++ b/dist/observability.js @@ -0,0 +1,64 @@ +import { createRequire } from 'node:module'; +import { config as loadEnv } from 'dotenv'; +import { createLogger } from './logging-config.js'; +loadEnv(); +const require = createRequire(import.meta.url); +const logger = createLogger('browser_use.observability'); +let lmnrObserve = null; +let lmnrAvailable = false; +try { + const lmnr = require('lmnr'); + if (typeof lmnr?.observe === 'function') { + lmnrObserve = (options) => lmnr.observe(options); + lmnrAvailable = true; + if (process.env.BROWSER_USE_VERBOSE_OBSERVABILITY?.toLowerCase() === 'true') { + logger.debug('Lmnr is available for observability'); + } + } +} +catch (error) { + lmnrObserve = null; + lmnrAvailable = false; + if (process.env.BROWSER_USE_VERBOSE_OBSERVABILITY?.toLowerCase() === 'true') { + logger.debug(`Lmnr is not available for observability (${error.message})`); + } +} +const isDebugModeEnv = () => process.env.LMNR_LOGGING_LEVEL?.toLowerCase() === 'debug'; +const createNoopDecorator = () => (fn) => fn; +const normalizeOptions = (options = {}) => ({ + name: options.name ?? null, + ignoreInput: options.ignoreInput ?? false, + ignoreOutput: options.ignoreOutput ?? false, + metadata: options.metadata ?? null, + spanType: options.spanType ?? 'DEFAULT', + ...options, +}); +export const observe = (options = {}) => { + const normalized = normalizeOptions({ + tags: ['observe', 'observe_debug'], + ...options, + }); + if (lmnrAvailable && lmnrObserve) { + return lmnrObserve(normalized); + } + return createNoopDecorator(); +}; +export const observeDebug = (options = {}) => { + const normalized = normalizeOptions({ + tags: ['observe_debug'], + ...options, + }); + if (lmnrAvailable && lmnrObserve && isDebugModeEnv()) { + return lmnrObserve(normalized); + } + return createNoopDecorator(); +}; +export const observe_debug = observeDebug; +export const isLmnrAvailable = () => lmnrAvailable; +export const isDebugMode = () => isDebugModeEnv(); +export const getObservabilityStatus = () => ({ + lmnrAvailable, + debugMode: isDebugModeEnv(), + observeActive: lmnrAvailable, + observeDebugActive: lmnrAvailable && isDebugModeEnv(), +}); diff --git a/dist/sandbox/index.d.ts b/dist/sandbox/index.d.ts new file mode 100644 index 00000000..a0938d57 --- /dev/null +++ b/dist/sandbox/index.d.ts @@ -0,0 +1,2 @@ +export * from './views.js'; +export * from './sandbox.js'; diff --git a/dist/sandbox/index.js b/dist/sandbox/index.js new file mode 100644 index 00000000..a0938d57 --- /dev/null +++ b/dist/sandbox/index.js @@ -0,0 +1,2 @@ +export * from './views.js'; +export * from './sandbox.js'; diff --git a/dist/sandbox/sandbox.d.ts b/dist/sandbox/sandbox.d.ts new file mode 100644 index 00000000..ad73ff46 --- /dev/null +++ b/dist/sandbox/sandbox.d.ts @@ -0,0 +1,19 @@ +import { BrowserCreatedData, ErrorData, LogData, ResultData, SandboxError } from './views.js'; +export interface SandboxOptions { + api_key?: string | null; + server_url?: string | null; + log_level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR' | string; + quiet?: boolean; + headers?: Record; + cloud_profile_id?: string | null; + cloud_proxy_country_code?: string | null; + cloud_timeout?: number | null; + fetch_impl?: typeof fetch; + on_browser_created?: (event: BrowserCreatedData) => void | Promise; + on_instance_ready?: () => void | Promise; + on_log?: (event: LogData) => void | Promise; + on_result?: (event: ResultData) => void | Promise; + on_error?: (event: ErrorData) => void | Promise; +} +export declare const sandbox: (options?: SandboxOptions) => (fn: (...args: TArgs) => Promise | TResult) => (...args: TArgs) => Promise; +export { SandboxError }; diff --git a/dist/sandbox/sandbox.js b/dist/sandbox/sandbox.js new file mode 100644 index 00000000..438d1e4e --- /dev/null +++ b/dist/sandbox/sandbox.js @@ -0,0 +1,140 @@ +import { createLogger } from '../logging-config.js'; +import { BrowserCreatedData, ErrorData, LogData, ResultData, SandboxError, SSEEvent, SSEEventType, } from './views.js'; +const logger = createLogger('browser_use.sandbox'); +const defaultServerUrl = 'https://sandbox.api.browser-use.com/sandbox-stream'; +const maybeInvoke = async (callback, data) => { + if (!callback) { + return; + } + await callback(data); +}; +const parseSSEChunks = async (response, onEvent) => { + const processLine = async (line) => { + if (!line.startsWith('data:')) { + return; + } + const jsonPayload = line.slice(5).trim(); + if (!jsonPayload) { + return; + } + try { + await onEvent(SSEEvent.from_json(jsonPayload)); + } + catch { + // Ignore malformed SSE entries. + } + }; + if (!response.body) { + const text = await response.text(); + for (const line of text.split(/\r?\n/)) { + await processLine(line); + } + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split(/\r?\n/); + buffer = lines.pop() ?? ''; + for (const line of lines) { + await processLine(line); + } + } + if (buffer) { + await processLine(buffer); + } +}; +const shouldUseRemoteSandbox = (options) => Boolean(options.server_url || + options.api_key || + options.cloud_profile_id || + options.cloud_proxy_country_code || + options.cloud_timeout); +export const sandbox = (options = {}) => (fn) => async (...args) => { + const remoteMode = shouldUseRemoteSandbox(options); + if (!remoteMode) { + return await fn(...args); + } + const apiKey = options.api_key?.trim() || process.env.BROWSER_USE_API_KEY?.trim(); + if (!apiKey) { + throw new SandboxError('BROWSER_USE_API_KEY is required for remote sandbox execution'); + } + const fetch_impl = options.fetch_impl ?? fetch; + const server_url = options.server_url ?? defaultServerUrl; + const payload = { + code: Buffer.from(String(fn)).toString('base64'), + args: Buffer.from(JSON.stringify(args)).toString('base64'), + env: { + LOG_LEVEL: String(options.log_level ?? 'INFO').toUpperCase(), + }, + }; + if (options.cloud_profile_id != null) { + payload.cloud_profile_id = options.cloud_profile_id; + } + if (options.cloud_proxy_country_code != null) { + payload.cloud_proxy_country_code = options.cloud_proxy_country_code; + } + if (options.cloud_timeout != null) { + payload.cloud_timeout = options.cloud_timeout; + } + const response = await fetch_impl(server_url, { + method: 'POST', + headers: { + 'X-API-Key': apiKey, + 'Content-Type': 'application/json', + ...(options.headers ?? {}), + }, + body: JSON.stringify(payload), + }); + if (!response.ok) { + throw new SandboxError(`Sandbox request failed with status ${response.status}`); + } + let executionResult = null; + let hasResult = false; + await parseSSEChunks(response, async (event) => { + if (event.type === SSEEventType.BROWSER_CREATED && + event.data instanceof BrowserCreatedData) { + await maybeInvoke(options.on_browser_created, event.data); + if (!options.quiet && event.data.live_url) { + logger.info(`🔗 Live URL: ${event.data.live_url}`); + } + return; + } + if (event.type === SSEEventType.INSTANCE_READY) { + await maybeInvoke(options.on_instance_ready, undefined); + return; + } + if (event.type === SSEEventType.LOG && event.data instanceof LogData) { + await maybeInvoke(options.on_log, event.data); + if (!options.quiet) { + logger.info(event.data.message); + } + return; + } + if (event.type === SSEEventType.RESULT && + event.data instanceof ResultData) { + await maybeInvoke(options.on_result, event.data); + if (!event.data.execution_response.success) { + throw new SandboxError(`Execution failed: ${event.data.execution_response.error ?? 'unknown error'}`); + } + executionResult = event.data.execution_response.result; + hasResult = true; + return; + } + if (event.type === SSEEventType.ERROR && + event.data instanceof ErrorData) { + await maybeInvoke(options.on_error, event.data); + throw new SandboxError(`Execution failed: ${event.data.error || 'unknown error'}`); + } + }); + if (!hasResult) { + throw new SandboxError('No result received from sandbox execution'); + } + return executionResult; +}; +export { SandboxError }; diff --git a/dist/sandbox/views.d.ts b/dist/sandbox/views.d.ts new file mode 100644 index 00000000..1cdb3191 --- /dev/null +++ b/dist/sandbox/views.d.ts @@ -0,0 +1,67 @@ +export declare class SandboxError extends Error { + constructor(message: string); +} +export declare enum SSEEventType { + BROWSER_CREATED = "browser_created", + INSTANCE_CREATED = "instance_created", + INSTANCE_READY = "instance_ready", + LOG = "log", + RESULT = "result", + ERROR = "error", + STREAM_COMPLETE = "stream_complete" +} +export declare class BrowserCreatedData { + session_id: string; + live_url: string; + status: string; + constructor(init: { + session_id: string; + live_url: string; + status: string; + }); +} +export declare class LogData { + message: string; + level: string; + constructor(init: { + message: string; + level?: string; + }); +} +export interface ExecutionResponse { + success: boolean; + result?: unknown; + error?: string | null; + traceback?: string | null; +} +export declare class ResultData { + execution_response: ExecutionResponse; + constructor(init: { + execution_response: ExecutionResponse; + }); +} +export declare class ErrorData { + error: string; + traceback: string | null; + status_code: number; + constructor(init: { + error: string; + traceback?: string | null; + status_code?: number; + }); +} +export declare class SSEEvent { + type: SSEEventType; + data: BrowserCreatedData | LogData | ResultData | ErrorData | Record; + timestamp: string | null; + constructor(init: { + type: SSEEventType; + data: BrowserCreatedData | LogData | ResultData | ErrorData | Record; + timestamp?: string | null; + }); + static from_json(event_json: string): SSEEvent; + is_browser_created(): boolean; + is_log(): boolean; + is_result(): boolean; + is_error(): boolean; +} diff --git a/dist/sandbox/views.js b/dist/sandbox/views.js new file mode 100644 index 00000000..0aaa1c4a --- /dev/null +++ b/dist/sandbox/views.js @@ -0,0 +1,121 @@ +export class SandboxError extends Error { + constructor(message) { + super(message); + this.name = 'SandboxError'; + } +} +export var SSEEventType; +(function (SSEEventType) { + SSEEventType["BROWSER_CREATED"] = "browser_created"; + SSEEventType["INSTANCE_CREATED"] = "instance_created"; + SSEEventType["INSTANCE_READY"] = "instance_ready"; + SSEEventType["LOG"] = "log"; + SSEEventType["RESULT"] = "result"; + SSEEventType["ERROR"] = "error"; + SSEEventType["STREAM_COMPLETE"] = "stream_complete"; +})(SSEEventType || (SSEEventType = {})); +export class BrowserCreatedData { + session_id; + live_url; + status; + constructor(init) { + this.session_id = init.session_id; + this.live_url = init.live_url; + this.status = init.status; + } +} +export class LogData { + message; + level; + constructor(init) { + this.message = init.message; + this.level = init.level ?? 'info'; + } +} +export class ResultData { + execution_response; + constructor(init) { + this.execution_response = init.execution_response; + } +} +export class ErrorData { + error; + traceback; + status_code; + constructor(init) { + this.error = init.error; + this.traceback = init.traceback ?? null; + this.status_code = init.status_code ?? 500; + } +} +export class SSEEvent { + type; + data; + timestamp; + constructor(init) { + this.type = init.type; + this.data = init.data; + this.timestamp = init.timestamp ?? null; + } + static from_json(event_json) { + const raw = JSON.parse(event_json); + const type = raw.type; + const payload = raw.data ?? {}; + let data; + if (type === SSEEventType.BROWSER_CREATED) { + data = new BrowserCreatedData({ + session_id: String(payload.session_id ?? ''), + live_url: String(payload.live_url ?? ''), + status: String(payload.status ?? ''), + }); + } + else if (type === SSEEventType.LOG) { + data = new LogData({ + message: String(payload.message ?? ''), + level: payload.level == null ? undefined : String(payload.level), + }); + } + else if (type === SSEEventType.RESULT) { + data = new ResultData({ + execution_response: { + success: Boolean(payload.execution_response?.success), + result: payload.execution_response?.result, + error: payload.execution_response?.error == null + ? null + : String(payload.execution_response?.error), + traceback: payload.execution_response?.traceback == null + ? null + : String(payload.execution_response?.traceback), + }, + }); + } + else if (type === SSEEventType.ERROR) { + data = new ErrorData({ + error: String(payload.error ?? ''), + traceback: payload.traceback == null ? null : String(payload.traceback), + status_code: payload.status_code == null ? undefined : Number(payload.status_code), + }); + } + else { + data = payload; + } + return new SSEEvent({ + type, + data, + timestamp: raw.timestamp ?? null, + }); + } + is_browser_created() { + return (this.type === SSEEventType.BROWSER_CREATED && + this.data instanceof BrowserCreatedData); + } + is_log() { + return this.type === SSEEventType.LOG && this.data instanceof LogData; + } + is_result() { + return this.type === SSEEventType.RESULT && this.data instanceof ResultData; + } + is_error() { + return this.type === SSEEventType.ERROR && this.data instanceof ErrorData; + } +} diff --git a/dist/screenshots/index.d.ts b/dist/screenshots/index.d.ts new file mode 100644 index 00000000..bfa0f513 --- /dev/null +++ b/dist/screenshots/index.d.ts @@ -0,0 +1 @@ +export * from './service.js'; diff --git a/dist/screenshots/index.js b/dist/screenshots/index.js new file mode 100644 index 00000000..bfa0f513 --- /dev/null +++ b/dist/screenshots/index.js @@ -0,0 +1 @@ +export * from './service.js'; diff --git a/dist/screenshots/service.d.ts b/dist/screenshots/service.d.ts new file mode 100644 index 00000000..626f6ee7 --- /dev/null +++ b/dist/screenshots/service.d.ts @@ -0,0 +1,6 @@ +export declare class ScreenshotService { + private screenshotsDir; + constructor(agentDirectory: string); + store_screenshot(screenshot_b64: string, step_number: number): Promise; + get_screenshot(screenshot_path: string): Promise; +} diff --git a/dist/screenshots/service.js b/dist/screenshots/service.js new file mode 100644 index 00000000..1395cdd5 --- /dev/null +++ b/dist/screenshots/service.js @@ -0,0 +1,28 @@ +import fs from 'node:fs'; +import path from 'node:path'; +const decodeBase64 = (data) => Buffer.from(data, 'base64'); +export class ScreenshotService { + screenshotsDir; + constructor(agentDirectory) { + this.screenshotsDir = path.join(agentDirectory, 'screenshots'); + fs.mkdirSync(this.screenshotsDir, { recursive: true }); + } + async store_screenshot(screenshot_b64, step_number) { + const filename = `step_${step_number}.png`; + const filepath = path.join(this.screenshotsDir, filename); + await fs.promises.writeFile(filepath, decodeBase64(screenshot_b64)); + return filepath; + } + async get_screenshot(screenshot_path) { + if (!screenshot_path) { + return null; + } + try { + const data = await fs.promises.readFile(screenshot_path); + return data.toString('base64'); + } + catch { + return null; + } + } +} diff --git a/dist/skill-cli/direct.d.ts b/dist/skill-cli/direct.d.ts new file mode 100644 index 00000000..a0d7af70 --- /dev/null +++ b/dist/skill-cli/direct.d.ts @@ -0,0 +1,100 @@ +#!/usr/bin/env node +import { CloudBrowserClient } from '../browser/cloud/cloud.js'; +export interface DirectModeState { + mode?: 'local' | 'remote'; + cdp_url?: string | null; + session_id?: string | null; + browser_pid?: number | null; + user_data_dir?: string | null; + owns_user_data_dir?: boolean | null; + active_url?: string | null; +} +export declare const DIRECT_STATE_FILE: string; +interface StreamLike { + write(chunk: string): void; +} +interface DirectSessionLike { + tabs?: Array<{ + target_id?: string | null; + url?: string | null; + }>; + active_tab?: { + target_id?: string | null; + url?: string | null; + } | null; + event_bus?: { + stop?: () => Promise | void; + } | null; + browser_context?: { + cookies?: (urls?: string[]) => Promise; + addCookies?: (cookies: any[]) => Promise; + clearCookies?: () => Promise; + } | null; + detach_all_watchdogs?: () => void; + start: () => Promise; + navigate_to?: (url: string) => Promise; + get_current_page?: () => Promise; + get_browser_state_with_recovery?: (options?: { + include_screenshot?: boolean; + }) => Promise<{ + llm_representation: () => string; + url?: string; + title?: string; + tabs?: unknown[]; + }>; + get_page_info?: () => Promise; + get_dom_element_by_index?: (index: number) => Promise; + get_locate_element?: (node: any) => Promise; + _click_element_node?: (node: any) => Promise; + click_coordinates?: (x: number, y: number, options?: { + button?: 'left' | 'middle' | 'right'; + }) => Promise; + send_keys?: (text: string) => Promise; + _input_text_element_node?: (node: any, text: string, options?: { + clear?: boolean; + }) => Promise; + take_screenshot?: (full_page?: boolean) => Promise; + scroll?: (direction: 'up' | 'down' | 'left' | 'right', amount: number) => Promise; + go_back?: () => Promise; + go_forward?: () => Promise; + get_page_html?: () => Promise; + execute_javascript?: (script: string) => Promise; + switch_to_tab?: (identifier: number | string) => Promise; + close_tab?: (identifier: number | string) => Promise; + select_dropdown_option?: (node: any, value: string) => Promise; + wait_for_element?: (selector: string, timeout: number) => Promise; + get_cookies?: () => Promise; +} +export interface DirectCliEnvironment { + state_file?: string; + stdout?: StreamLike; + stderr?: StreamLike; + session_factory?: (init: { + cdp_url?: string | null; + }) => DirectSessionLike; + cloud_client_factory?: () => Pick; + local_launcher?: (options: { + state: DirectModeState; + }) => Promise<{ + cdp_url: string; + browser_pid?: number | null; + user_data_dir?: string | null; + owns_user_data_dir?: boolean | null; + }>; + kill_process?: (pid: number) => void | Promise; +} +export declare const load_direct_state: (state_file?: string) => DirectModeState; +export declare const save_direct_state: (state: DirectModeState, state_file?: string) => void; +export declare const clear_direct_state: (state_file?: string) => void; +export declare const defaultLocalLauncher: (options: { + state: DirectModeState; + timeout_ms?: number; +}) => Promise<{ + cdp_url: string; + browser_pid: number | null; + user_data_dir: string | null | undefined; + owns_user_data_dir: boolean; +}>; +export declare const run_direct_command: (argv: string[], options?: DirectCliEnvironment) => Promise<0 | 1>; +export declare const main: (argv?: string[]) => Promise<0 | 1>; +export {}; diff --git a/dist/skill-cli/direct.js b/dist/skill-cli/direct.js new file mode 100644 index 00000000..dede5ef0 --- /dev/null +++ b/dist/skill-cli/direct.js @@ -0,0 +1,984 @@ +#!/usr/bin/env node +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import net from 'node:net'; +import { spawn } from 'node:child_process'; +import { BrowserSession, systemChrome } from '../browser/session.js'; +import { CloudBrowserClient } from '../browser/cloud/cloud.js'; +export const DIRECT_STATE_FILE = path.join(os.tmpdir(), 'browser-use-direct.json'); +const normalizeCookieDomain = (value) => String(value ?? '') + .trim() + .replace(/^\./, '') + .toLowerCase(); +const parseCookieHostname = (url) => { + const value = String(url ?? '').trim(); + if (!value) { + return ''; + } + try { + return new URL(value).hostname.toLowerCase(); + } + catch { + return ''; + } +}; +const parseCookieUrl = (url) => { + const value = String(url ?? '').trim(); + if (!value) { + return null; + } + try { + return new URL(value); + } + catch { + return null; + } +}; +const cookiePathMatches = (cookiePath, urlPath) => { + const normalizedCookiePath = typeof cookiePath === 'string' && cookiePath.length > 0 ? cookiePath : '/'; + if (normalizedCookiePath === '/') { + return true; + } + if (urlPath === normalizedCookiePath) { + return true; + } + return urlPath.startsWith(normalizedCookiePath.endsWith('/') + ? normalizedCookiePath + : `${normalizedCookiePath}/`); +}; +const cookieMatchesUrl = (cookie, url) => { + const parsedUrl = parseCookieUrl(url); + const hostname = parsedUrl?.hostname.toLowerCase() ?? ''; + const domain = normalizeCookieDomain(cookie.domain); + if (!hostname || !domain) { + return false; + } + if (!(hostname === domain || + hostname.endsWith(`.${domain}`) || + domain.endsWith(`.${hostname}`))) { + return false; + } + if (!cookiePathMatches(cookie.path, parsedUrl?.pathname || '/')) { + return false; + } + if (cookie.secure && parsedUrl?.protocol !== 'https:') { + return false; + } + return true; +}; +const normalizeSameSite = (value) => { + const normalized = String(value ?? '') + .trim() + .toLowerCase(); + if (normalized === 'strict') { + return 'Strict'; + } + if (normalized === 'lax') { + return 'Lax'; + } + if (normalized === 'none') { + return 'None'; + } + return undefined; +}; +const DEFAULT_STDOUT = process.stdout; +const DEFAULT_STDERR = process.stderr; +const writeLine = (stream, message) => { + stream.write(`${message}\n`); +}; +export const load_direct_state = (state_file = DIRECT_STATE_FILE) => { + if (!fs.existsSync(state_file)) { + return {}; + } + try { + return JSON.parse(fs.readFileSync(state_file, 'utf8')); + } + catch { + return {}; + } +}; +export const save_direct_state = (state, state_file = DIRECT_STATE_FILE) => { + fs.writeFileSync(state_file, JSON.stringify(state, null, 2)); +}; +export const clear_direct_state = (state_file = DIRECT_STATE_FILE) => { + fs.rmSync(state_file, { force: true }); +}; +const cleanupOwnedDirectUserDataDir = (state) => { + if (!state.owns_user_data_dir || !state.user_data_dir) { + return; + } + try { + fs.rmSync(state.user_data_dir, { recursive: true, force: true }); + } + catch { + // Ignore cleanup failures for ephemeral direct-mode profiles. + } +}; +const normalizeDirectUrl = (input) => { + const trimmed = input.trim(); + if (!trimmed) { + return trimmed; + } + if (/^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(trimmed)) { + return trimmed; + } + return `https://${trimmed}`; +}; +const formatDirectUsage = () => `Usage: browser-use-direct [args] + +Commands: + open Navigate to URL + state Get current browser state + click Click element by DOM index + click Click viewport coordinates + type Type into focused element + input Click element and type text + screenshot [path] Take screenshot + scroll [up|down] Scroll page + back Go back in history + forward Go forward in history + switch Switch to tab index or target id + close-tab [tab] Close a tab + keys Send keyboard keys + select Select dropdown option + wait selector Wait for a selector + wait text Wait for text + hover Hover element by DOM index + dblclick Double-click element by DOM index + rightclick Right-click element by DOM index + cookies Manage cookies (get/set/clear/export/import) + get title Get page title + get html [selector] Get page HTML or a CSS selector + get text Get element text + get value Get element value + get attributes Get element attributes + get bbox Get element bounding box + extract Explain that extraction requires agent mode + html [selector] Get page HTML or a CSS selector + eval Execute JavaScript + close Close the active direct-mode browser + +Flags: + --remote Launch browser-use cloud browser`; +const extractDirectModeArgs = (argv) => { + let useRemote = false; + let index = 0; + while (index < argv.length) { + const arg = argv[index] ?? ''; + if (arg === '--remote') { + useRemote = true; + index += 1; + continue; + } + break; + } + return { + useRemote, + args: argv.slice(index), + }; +}; +const getFreePort = async () => await new Promise((resolve, reject) => { + const server = net.createServer(); + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + if (!address || typeof address === 'string') { + server.close(() => reject(new Error('Failed to allocate local port'))); + return; + } + const port = address.port; + server.close((error) => { + if (error) { + reject(error); + return; + } + resolve(port); + }); + }); +}); +const waitForLocalCdpEndpoint = async (port, timeoutMs = 15000) => { + const deadline = Date.now() + timeoutMs; + const endpoint = `http://127.0.0.1:${port}`; + while (Date.now() < deadline) { + try { + const response = await fetch(`${endpoint}/json/version`); + if (response.ok) { + const payload = (await response.json()); + if (typeof payload.webSocketDebuggerUrl === 'string') { + return endpoint; + } + } + } + catch { + // Keep polling until timeout. + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`Timed out waiting for local Chrome debugging endpoint on port ${port}`); +}; +export const defaultLocalLauncher = async (options) => { + const executablePath = systemChrome.findExecutable(); + if (!executablePath) { + throw new Error('Chrome not found. Install Chrome or provide an already-running browser via cdp_url.'); + } + const port = await getFreePort(); + const reusingUserDataDir = options.state.user_data_dir && + options.state.user_data_dir.trim().length > 0; + const userDataDir = reusingUserDataDir + ? options.state.user_data_dir + : fs.mkdtempSync(path.join(os.tmpdir(), 'browser-use-direct-')); + const child = spawn(executablePath, [ + `--remote-debugging-port=${port}`, + `--user-data-dir=${userDataDir}`, + '--no-first-run', + '--no-default-browser-check', + 'about:blank', + ], { + detached: true, + stdio: 'ignore', + }); + child.unref(); + try { + const cdp_url = await waitForLocalCdpEndpoint(port, options.timeout_ms); + return { + cdp_url, + browser_pid: child.pid ?? null, + user_data_dir: userDataDir, + owns_user_data_dir: !reusingUserDataDir, + }; + } + catch (error) { + if (typeof child.pid === 'number' && child.pid > 0) { + try { + process.kill(child.pid, 'SIGTERM'); + } + catch { + // Ignore cleanup failures for a process that may not have started. + } + } + if (!reusingUserDataDir && typeof userDataDir === 'string') { + try { + fs.rmSync(userDataDir, { recursive: true, force: true }); + } + catch { + // Ignore cleanup failures for ephemeral launch profiles. + } + } + throw error; + } +}; +const cleanupDirectSession = async (session) => { + try { + session.detach_all_watchdogs?.(); + } + catch { + // Ignore cleanup failures. + } + try { + await session.event_bus?.stop?.(); + } + catch { + // Ignore event bus cleanup failures. + } +}; +const requireDirectNodeByIndex = async (session, indexValue) => { + const index = Number(indexValue ?? Number.NaN); + if (!Number.isFinite(index)) { + throw new Error('Missing index'); + } + const node = await session.get_dom_element_by_index?.(index); + if (!node) { + throw new Error(`Element index ${index} not found - run "state" first`); + } + return { index, node }; +}; +const readDirectNodeData = async (session, node, kind) => { + if (!node?.xpath) { + throw new Error('DOM element does not include an XPath selector'); + } + const page = await session.get_current_page?.(); + if (!page?.evaluate) { + throw new Error('No active page available'); + } + return await page.evaluate(({ xpath, dataKind }) => { + const element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + if (!element) { + return null; + } + if (dataKind === 'text') { + return element.textContent?.trim() ?? ''; + } + if (dataKind === 'value') { + return 'value' in element + ? String(element.value ?? '') + : null; + } + if (dataKind === 'attributes') { + return Object.fromEntries(Array.from(element.attributes).map((attribute) => [ + attribute.name, + attribute.value, + ])); + } + if (dataKind === 'bbox') { + const rect = element.getBoundingClientRect(); + return { + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + top: rect.top, + right: rect.right, + bottom: rect.bottom, + left: rect.left, + }; + } + return null; + }, { xpath: node.xpath, dataKind: kind }); +}; +const takeDirectOptionValue = (args, index, option) => { + const next = args[index + 1]?.trim(); + if (!next || next === '--' || next.startsWith('-')) { + throw new Error(`Missing value for ${option}`); + } + return next; +}; +const parseDirectCookieOptions = (args) => { + const positional = []; + let url = null; + let domain = null; + let cookiePath = '/'; + let secure = false; + let httpOnly = false; + let sameSite; + let expires; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] ?? ''; + if (arg === '--') { + positional.push(...args.slice(index + 1)); + break; + } + if (arg === '--url' || + arg === '--domain' || + arg === '--path' || + arg === '--same-site' || + arg === '--expires') { + const next = takeDirectOptionValue(args, index, arg); + if (arg === '--url') { + url = next; + } + else if (arg === '--domain') { + domain = next; + } + else if (arg === '--path') { + cookiePath = next; + } + else if (arg === '--same-site') { + sameSite = normalizeSameSite(next); + if (!sameSite) { + throw new Error('Invalid --same-site value. Expected Strict, Lax, or None'); + } + } + else { + const parsed = Number(next); + if (!Number.isFinite(parsed)) { + throw new Error(`Invalid --expires value: ${next}`); + } + expires = parsed; + } + index += 1; + continue; + } + if (arg === '--secure') { + secure = true; + continue; + } + if (arg === '--http-only') { + httpOnly = true; + continue; + } + if (arg.startsWith('-')) { + throw new Error(`Unknown option: ${arg}`); + } + positional.push(arg); + } + return { + positional, + url, + domain, + path: cookiePath, + secure, + httpOnly, + sameSite, + expires, + }; +}; +const restoreActiveTab = async (session, state) => { + if (typeof state.active_url !== 'string' || + !state.active_url || + !Array.isArray(session.tabs) || + typeof session.switch_to_tab !== 'function') { + return; + } + const matchingTab = session.tabs.find((tab) => tab?.url === state.active_url); + if (!matchingTab?.target_id) { + return; + } + try { + await session.switch_to_tab(matchingTab.target_id); + } + catch { + // Fall back to the default page if the tab cannot be restored. + } +}; +const createDefaultSessionFactory = () => (init) => new BrowserSession({ + cdp_url: init.cdp_url ?? null, + profile: { + keep_alive: true, + }, +}); +const connectDirectSession = async (useRemote, environment) => { + let state = load_direct_state(environment.state_file); + const session_factory = environment.session_factory ?? createDefaultSessionFactory(); + const connectWithState = async (currentState) => { + const session = session_factory({ cdp_url: currentState.cdp_url ?? null }); + await session.start(); + await restoreActiveTab(session, currentState); + return session; + }; + const cleanupDisconnectedState = async (currentState) => { + if (currentState.mode === 'remote' && currentState.session_id) { + try { + await environment + .cloud_client_factory() + .stop_browser(currentState.session_id); + } + catch { + // Best-effort cleanup for stale remote sessions. + } + return; + } + if (currentState.mode === 'local' && + typeof currentState.browser_pid === 'number' && + currentState.browser_pid > 0) { + try { + await environment.kill_process(currentState.browser_pid); + } + catch { + // Ignore cleanup errors for stale local browser processes. + } + } + cleanupOwnedDirectUserDataDir(currentState); + }; + if (state.cdp_url) { + try { + const session = await connectWithState(state); + return { session, state }; + } + catch { + await cleanupDisconnectedState(state); + clear_direct_state(environment.state_file); + state = {}; + } + } + if (useRemote) { + const cloudClient = environment.cloud_client_factory?.() ?? new CloudBrowserClient(); + const browser = await cloudClient.create_browser({}); + state = { + mode: 'remote', + cdp_url: browser.cdpUrl, + session_id: browser.id, + active_url: null, + }; + save_direct_state(state, environment.state_file); + return { + session: await connectWithState(state), + state, + }; + } + const localLaunch = await (environment.local_launcher ?? defaultLocalLauncher)({ + state, + }); + state = { + mode: 'local', + cdp_url: localLaunch.cdp_url, + browser_pid: localLaunch.browser_pid ?? null, + user_data_dir: localLaunch.user_data_dir ?? null, + owns_user_data_dir: localLaunch.owns_user_data_dir ?? null, + active_url: null, + }; + save_direct_state(state, environment.state_file); + return { + session: await connectWithState(state), + state, + }; +}; +const updateDirectStateFromSession = async (session, state, environment) => { + const currentPage = await session.get_current_page?.(); + const active_url = typeof currentPage?.url === 'function' + ? String(currentPage.url() ?? '') + : (session.active_tab?.url ?? null); + save_direct_state({ + ...state, + active_url: typeof active_url === 'string' && active_url.trim().length > 0 + ? active_url + : null, + }, environment.state_file); +}; +export const run_direct_command = async (argv, options = {}) => { + const environment = { + state_file: options.state_file ?? DIRECT_STATE_FILE, + stdout: options.stdout ?? DEFAULT_STDOUT, + stderr: options.stderr ?? DEFAULT_STDERR, + session_factory: options.session_factory ?? createDefaultSessionFactory(), + cloud_client_factory: options.cloud_client_factory ?? (() => new CloudBrowserClient()), + local_launcher: options.local_launcher ?? defaultLocalLauncher, + kill_process: options.kill_process ?? + ((pid) => { + process.kill(pid, 'SIGTERM'); + }), + }; + const { useRemote, args } = extractDirectModeArgs(argv); + const command = args[0] ?? ''; + if (!command || + command === 'help' || + command === '--help' || + command === '-h') { + writeLine(environment.stdout, formatDirectUsage()); + return command ? 0 : 1; + } + if (command === 'close') { + const state = load_direct_state(environment.state_file); + if (!state.cdp_url) { + writeLine(environment.stdout, 'No active browser session'); + clear_direct_state(environment.state_file); + return 0; + } + if (state.mode === 'remote' && state.session_id) { + try { + await environment.cloud_client_factory().stop_browser(state.session_id); + } + catch { + // Best-effort remote cleanup. + } + } + else if (typeof state.browser_pid === 'number' && state.browser_pid > 0) { + try { + await environment.kill_process(state.browser_pid); + } + catch { + // Ignore close errors for an already-exited browser. + } + } + cleanupOwnedDirectUserDataDir(state); + clear_direct_state(environment.state_file); + writeLine(environment.stdout, 'Browser closed'); + return 0; + } + let connected = null; + try { + connected = await connectDirectSession(useRemote, environment); + const { session, state } = connected; + if (command === 'open') { + const url = normalizeDirectUrl(args[1] ?? ''); + if (!url) { + throw new Error('Missing url'); + } + await session.navigate_to?.(url); + writeLine(environment.stdout, `Navigated to: ${url}`); + } + else if (command === 'state') { + const summary = await session.get_browser_state_with_recovery?.({ + include_screenshot: false, + }); + if (!summary) { + throw new Error('No browser state available'); + } + const pageInfo = await session.get_page_info?.(); + let output = summary.llm_representation(); + if (pageInfo) { + output = + `viewport: ${pageInfo.viewport_width}x${pageInfo.viewport_height}\n` + + `page: ${pageInfo.page_width}x${pageInfo.page_height}\n` + + `scroll: (${pageInfo.scroll_x}, ${pageInfo.scroll_y})\n` + + output; + } + writeLine(environment.stdout, output); + } + else if (command === 'click') { + const numericArgs = args.slice(1).map((arg) => Number(arg)); + if (numericArgs.length === 2 && numericArgs.every(Number.isFinite)) { + const [x, y] = numericArgs; + await session.click_coordinates?.(x, y); + writeLine(environment.stdout, `Clicked at (${x}, ${y})`); + } + else if (numericArgs.length === 1 && + Number.isFinite(numericArgs[0] ?? Number.NaN)) { + const { node } = await requireDirectNodeByIndex(session, String(numericArgs[0])); + await session._click_element_node?.(node); + writeLine(environment.stdout, `Clicked element [${numericArgs[0]}]`); + } + else { + throw new Error('Usage: click or click '); + } + } + else if (command === 'type') { + const text = args.slice(1).join(' ').trim(); + if (!text) { + throw new Error('Missing text'); + } + await session.send_keys?.(text); + writeLine(environment.stdout, `Typed: ${text}`); + } + else if (command === 'input') { + const index = Number(args[1] ?? Number.NaN); + const text = args.slice(2).join(' ').trim(); + if (!Number.isFinite(index) || !text) { + throw new Error('Usage: input '); + } + const { node } = await requireDirectNodeByIndex(session, String(index)); + await session._input_text_element_node?.(node, text, { clear: true }); + writeLine(environment.stdout, `Typed "${text}" into element [${index}]`); + } + else if (command === 'screenshot') { + const outputPath = args[1] ? path.resolve(args[1]) : null; + const screenshot = await session.take_screenshot?.(false); + if (!screenshot) { + throw new Error('Failed to capture screenshot'); + } + const bytes = Buffer.from(screenshot, 'base64'); + if (outputPath) { + fs.writeFileSync(outputPath, bytes); + writeLine(environment.stdout, `Screenshot saved to ${outputPath} (${bytes.length} bytes)`); + } + else { + writeLine(environment.stdout, JSON.stringify({ + screenshot, + size_bytes: bytes.length, + })); + } + } + else if (command === 'scroll') { + const direction = args[1] === 'up' || args[1] === 'left' || args[1] === 'right' + ? args[1] + : 'down'; + await session.scroll?.(direction, 500); + writeLine(environment.stdout, `Scrolled ${direction}`); + } + else if (command === 'back') { + await session.go_back?.(); + writeLine(environment.stdout, 'Navigated back'); + } + else if (command === 'forward') { + await session.go_forward?.(); + writeLine(environment.stdout, 'Navigated forward'); + } + else if (command === 'switch') { + const rawIdentifier = args[1]?.trim(); + if (!rawIdentifier) { + throw new Error('Usage: switch '); + } + const numericIdentifier = Number(rawIdentifier); + const identifier = Number.isFinite(numericIdentifier) + ? numericIdentifier + : rawIdentifier; + await session.switch_to_tab?.(identifier); + writeLine(environment.stdout, `Switched to tab: ${rawIdentifier}`); + } + else if (command === 'close-tab') { + const rawIdentifier = args[1]?.trim(); + const numericIdentifier = rawIdentifier && rawIdentifier.length > 0 ? Number(rawIdentifier) : NaN; + const identifier = rawIdentifier && rawIdentifier.length > 0 + ? Number.isFinite(numericIdentifier) + ? numericIdentifier + : rawIdentifier + : (session.active_tab?.target_id ?? null); + if (identifier === null) { + throw new Error('Usage: close-tab [tab]'); + } + await session.close_tab?.(identifier); + writeLine(environment.stdout, `Closed tab: ${identifier}`); + } + else if (command === 'keys') { + const keys = args.slice(1).join(' ').trim(); + if (!keys) { + throw new Error('Missing keys'); + } + await session.send_keys?.(keys); + writeLine(environment.stdout, `Sent keys: ${keys}`); + } + else if (command === 'select') { + const index = args[1]; + const value = args.slice(2).join(' ').trim(); + if (!index || !value) { + throw new Error('Usage: select '); + } + const { node, index: numericIndex } = await requireDirectNodeByIndex(session, index); + await session.select_dropdown_option?.(node, value); + writeLine(environment.stdout, `Selected "${value}" for element [${numericIndex}]`); + } + else if (command === 'wait') { + const waitCommand = args[1] ?? ''; + if (waitCommand === 'selector') { + const selector = args[2]?.trim(); + const timeout = Number(args[3] ?? 5000); + if (!selector) { + throw new Error('Usage: wait selector [timeout]'); + } + await session.wait_for_element?.(selector, timeout); + writeLine(environment.stdout, `Waited for selector "${selector}" (${timeout}ms)`); + } + else if (waitCommand === 'text') { + const text = args.slice(2).join(' ').trim(); + if (!text) { + throw new Error('Usage: wait text '); + } + const page = await session.get_current_page?.(); + if (!page?.waitForFunction) { + throw new Error('No active page available for wait text'); + } + await page.waitForFunction((needle) => document.body?.innerText?.includes(needle) ?? false, text, { timeout: 5000 }); + writeLine(environment.stdout, `Waited for text "${text}"`); + } + else { + throw new Error('Usage: wait selector | wait text '); + } + } + else if (command === 'hover') { + const { node, index } = await requireDirectNodeByIndex(session, args[1]); + const locator = await session.get_locate_element?.(node); + if (!locator?.hover) { + throw new Error('Hover is not available for this element'); + } + await locator.hover({ timeout: 5000 }); + writeLine(environment.stdout, `Hovered element [${index}]`); + } + else if (command === 'dblclick') { + const { node, index } = await requireDirectNodeByIndex(session, args[1]); + const locator = await session.get_locate_element?.(node); + if (!locator?.dblclick) { + throw new Error('Double-click is not available for this element'); + } + await locator.dblclick({ timeout: 5000 }); + writeLine(environment.stdout, `Double-clicked element [${index}]`); + } + else if (command === 'rightclick') { + const { node, index } = await requireDirectNodeByIndex(session, args[1]); + const locator = await session.get_locate_element?.(node); + if (!locator?.click) { + throw new Error('Right-click is not available for this element'); + } + await locator.click({ button: 'right', timeout: 5000 }); + writeLine(environment.stdout, `Right-clicked element [${index}]`); + } + else if (command === 'cookies') { + const cookieCommand = args[1] ?? ''; + if (cookieCommand === 'get') { + const parsed = parseDirectCookieOptions(args.slice(2)); + const url = parsed.url ?? parsed.positional[0] ?? null; + const allCookies = (await session.get_cookies?.()) ?? []; + const cookies = url + ? allCookies.filter((cookie) => cookieMatchesUrl(cookie, url)) + : allCookies; + writeLine(environment.stdout, JSON.stringify({ cookies, count: cookies.length }, null, 2)); + } + else if (cookieCommand === 'set') { + if (!session.browser_context?.addCookies) { + throw new Error('Browser context does not support setting cookies'); + } + const parsed = parseDirectCookieOptions(args.slice(2)); + const name = parsed.positional[0]?.trim(); + const value = parsed.positional[1] ?? ''; + if (!name || parsed.positional.length < 2) { + throw new Error('Usage: cookies set [--url ] [--domain ] [--path ] [--secure] [--http-only] [--same-site ] [--expires ]'); + } + const currentPage = await session.get_current_page?.(); + const currentUrl = typeof currentPage?.url === 'function' ? currentPage.url() : ''; + const cookie = { + name, + value, + path: parsed.path, + secure: parsed.secure, + httpOnly: parsed.httpOnly, + sameSite: parsed.sameSite, + expires: parsed.expires, + }; + if (parsed.url) { + cookie.url = parsed.url; + } + else if (parsed.domain) { + cookie.domain = parsed.domain; + } + else if (currentUrl) { + cookie.url = currentUrl; + } + else { + throw new Error('Provide cookie url/domain or open a page first'); + } + await session.browser_context.addCookies([cookie]); + writeLine(environment.stdout, `Set cookie ${name}`); + } + else if (cookieCommand === 'clear') { + if (!session.browser_context?.clearCookies) { + throw new Error('Browser context does not support clearing cookies'); + } + const parsed = parseDirectCookieOptions(args.slice(2)); + const url = parsed.url ?? parsed.positional[0] ?? null; + if (!url) { + await session.browser_context.clearCookies(); + writeLine(environment.stdout, 'Cleared cookies'); + } + else { + const allCookies = (await session.get_cookies?.()) ?? []; + const remaining = allCookies.filter((cookie) => !cookieMatchesUrl(cookie, url)); + const removedCount = allCookies.length - remaining.length; + await session.browser_context.clearCookies(); + if (remaining.length > 0 && session.browser_context.addCookies) { + await session.browser_context.addCookies(remaining); + } + writeLine(environment.stdout, `Cleared ${removedCount} cookies matching ${url}`); + } + } + else if (cookieCommand === 'export') { + const file = args[2]?.trim(); + if (!file) { + throw new Error('Usage: cookies export [--url ]'); + } + const parsed = parseDirectCookieOptions(args.slice(3)); + const url = parsed.url ?? parsed.positional[0] ?? null; + const allCookies = (await session.get_cookies?.()) ?? []; + const cookies = url + ? allCookies.filter((cookie) => cookieMatchesUrl(cookie, url)) + : allCookies; + const outputPath = path.resolve(file); + fs.writeFileSync(outputPath, JSON.stringify(cookies, null, 2), 'utf8'); + writeLine(environment.stdout, `Exported ${cookies.length} cookies to ${outputPath}`); + } + else if (cookieCommand === 'import') { + if (!session.browser_context?.addCookies) { + throw new Error('Browser context does not support importing cookies'); + } + const file = args[2]?.trim(); + if (!file) { + throw new Error('Usage: cookies import '); + } + const inputPath = path.resolve(file); + const raw = fs.readFileSync(inputPath, 'utf8'); + const cookies = JSON.parse(raw); + if (!Array.isArray(cookies)) { + throw new Error('Cookie import file must contain a JSON array'); + } + await session.browser_context.addCookies(cookies); + writeLine(environment.stdout, `Imported ${cookies.length} cookies from ${inputPath}`); + } + else { + throw new Error('Usage: cookies get [url|--url ] | cookies set | cookies clear [--url ] | cookies export [--url ] | cookies import '); + } + } + else if (command === 'get') { + const subcommand = args[1] ?? ''; + if (subcommand === 'title') { + const page = await session.get_current_page?.(); + if (!page?.title) { + throw new Error('No active page available for get title'); + } + writeLine(environment.stdout, await page.title()); + } + else if (subcommand === 'html') { + const selector = args.slice(2).join(' ').trim(); + if (!selector) { + writeLine(environment.stdout, (await session.get_page_html?.()) ?? ''); + } + else { + const page = await session.get_current_page?.(); + if (!page?.evaluate) { + throw new Error('No active page available for get html'); + } + const html = await page.evaluate((targetSelector) => { + const element = document.querySelector(targetSelector); + return element ? element.outerHTML : null; + }, selector); + if (typeof html !== 'string' || html.length === 0) { + throw new Error(`No element found for selector: ${selector}`); + } + writeLine(environment.stdout, html); + } + } + else if (subcommand === 'text' || + subcommand === 'value' || + subcommand === 'attributes' || + subcommand === 'bbox') { + const { node } = await requireDirectNodeByIndex(session, args[2]); + const value = await readDirectNodeData(session, node, subcommand); + if (value == null) { + throw new Error(`Unable to retrieve ${subcommand} for element`); + } + writeLine(environment.stdout, typeof value === 'string' ? value : JSON.stringify(value)); + } + else { + throw new Error('Usage: get title | get html [selector] | get text | get value | get attributes | get bbox '); + } + } + else if (command === 'extract') { + const query = args.slice(1).join(' ').trim(); + if (!query) { + throw new Error('Missing query'); + } + writeLine(environment.stdout, JSON.stringify({ + query, + error: 'extract requires agent mode - use: browser-use run "extract ..."', + })); + } + else if (command === 'html') { + const selector = args.slice(1).join(' ').trim(); + if (!selector) { + writeLine(environment.stdout, (await session.get_page_html?.()) ?? ''); + } + else { + const page = await session.get_current_page?.(); + if (!page?.evaluate) { + throw new Error('No active page available for html'); + } + const html = await page.evaluate((targetSelector) => { + const element = document.querySelector(targetSelector); + return element ? element.outerHTML : null; + }, selector); + if (typeof html !== 'string' || html.length === 0) { + throw new Error(`No element found for selector: ${selector}`); + } + writeLine(environment.stdout, html); + } + } + else if (command === 'eval') { + const script = args.slice(1).join(' ').trim(); + if (!script) { + throw new Error('Missing js'); + } + const result = await session.execute_javascript?.(script); + writeLine(environment.stdout, result === undefined ? 'undefined' : JSON.stringify(result)); + } + else { + throw new Error(`Unknown command: ${command}`); + } + await updateDirectStateFromSession(session, state, environment); + await cleanupDirectSession(session); + return 0; + } + catch (error) { + if (connected?.session) { + await cleanupDirectSession(connected.session); + } + writeLine(environment.stderr, `Error: ${error?.message ?? String(error)}`); + return 1; + } +}; +export const main = async (argv = process.argv.slice(2)) => { + const exitCode = await run_direct_command(argv); + if (import.meta.url === `file://${process.argv[1]}`) { + process.exit(exitCode); + } + return exitCode; +}; +if (import.meta.url === `file://${process.argv[1]}`) { + void main(); +} diff --git a/dist/skill-cli/index.d.ts b/dist/skill-cli/index.d.ts new file mode 100644 index 00000000..46506cd1 --- /dev/null +++ b/dist/skill-cli/index.d.ts @@ -0,0 +1,5 @@ +export * from './protocol.js'; +export * from './sessions.js'; +export * from './server.js'; +export * from './direct.js'; +export * from './tunnel.js'; diff --git a/dist/skill-cli/index.js b/dist/skill-cli/index.js new file mode 100644 index 00000000..46506cd1 --- /dev/null +++ b/dist/skill-cli/index.js @@ -0,0 +1,5 @@ +export * from './protocol.js'; +export * from './sessions.js'; +export * from './server.js'; +export * from './direct.js'; +export * from './tunnel.js'; diff --git a/dist/skill-cli/protocol.d.ts b/dist/skill-cli/protocol.d.ts new file mode 100644 index 00000000..12875bc8 --- /dev/null +++ b/dist/skill-cli/protocol.d.ts @@ -0,0 +1,30 @@ +export interface RequestInit { + id: string; + action: string; + session: string; + params?: Record; +} +export declare class Request { + id: string; + action: string; + session: string; + params: Record; + constructor(init: RequestInit); + to_json(): string; + static from_json(data: string): Request; +} +export interface ResponseInit { + id: string; + success: boolean; + data?: unknown; + error?: string | null; +} +export declare class Response { + id: string; + success: boolean; + data: unknown; + error: string | null; + constructor(init: ResponseInit); + to_json(): string; + static from_json(data: string): Response; +} diff --git a/dist/skill-cli/protocol.js b/dist/skill-cli/protocol.js new file mode 100644 index 00000000..190cf4fa --- /dev/null +++ b/dist/skill-cli/protocol.js @@ -0,0 +1,48 @@ +export class Request { + id; + action; + session; + params; + constructor(init) { + this.id = init.id; + this.action = init.action; + this.session = init.session; + this.params = init.params ?? {}; + } + to_json() { + return JSON.stringify({ + id: this.id, + action: this.action, + session: this.session, + params: this.params, + }); + } + static from_json(data) { + const parsed = JSON.parse(data); + return new Request(parsed); + } +} +export class Response { + id; + success; + data; + error; + constructor(init) { + this.id = init.id; + this.success = init.success; + this.data = init.data ?? null; + this.error = init.error ?? null; + } + to_json() { + return JSON.stringify({ + id: this.id, + success: this.success, + data: this.data, + error: this.error, + }); + } + static from_json(data) { + const parsed = JSON.parse(data); + return new Response(parsed); + } +} diff --git a/dist/skill-cli/server.d.ts b/dist/skill-cli/server.d.ts new file mode 100644 index 00000000..b7b56812 --- /dev/null +++ b/dist/skill-cli/server.d.ts @@ -0,0 +1,13 @@ +import { Request, Response } from './protocol.js'; +import { SessionRegistry } from './sessions.js'; +export interface SkillCliServerOptions { + registry?: SessionRegistry; +} +export declare class SkillCliServer { + readonly registry: SessionRegistry; + constructor(options?: SkillCliServerOptions); + private _require_node_by_index; + private _read_node_data; + private _handle_browser_action; + handle_request(request: Request | string): Promise; +} diff --git a/dist/skill-cli/server.js b/dist/skill-cli/server.js new file mode 100644 index 00000000..3dbb7a12 --- /dev/null +++ b/dist/skill-cli/server.js @@ -0,0 +1,546 @@ +import { promises as fsp } from 'node:fs'; +import path from 'node:path'; +import { Request, Response } from './protocol.js'; +import { SessionRegistry } from './sessions.js'; +const normalizeCookieDomain = (value) => String(value ?? '') + .trim() + .replace(/^\./, '') + .toLowerCase(); +const parseCookieHostname = (url) => { + const value = String(url ?? '').trim(); + if (!value) { + return ''; + } + try { + return new URL(value).hostname.toLowerCase(); + } + catch { + return ''; + } +}; +const parseCookieUrl = (url) => { + const value = String(url ?? '').trim(); + if (!value) { + return null; + } + try { + return new URL(value); + } + catch { + return null; + } +}; +const cookiePathMatches = (cookiePath, urlPath) => { + const normalizedCookiePath = typeof cookiePath === 'string' && cookiePath.length > 0 ? cookiePath : '/'; + if (normalizedCookiePath === '/') { + return true; + } + if (urlPath === normalizedCookiePath) { + return true; + } + return urlPath.startsWith(normalizedCookiePath.endsWith('/') + ? normalizedCookiePath + : `${normalizedCookiePath}/`); +}; +const cookieMatchesUrl = (cookie, url) => { + const parsedUrl = parseCookieUrl(url); + const hostname = parsedUrl?.hostname.toLowerCase() ?? ''; + const domain = normalizeCookieDomain(cookie.domain); + if (!hostname || !domain) { + return false; + } + if (!(hostname === domain || + hostname.endsWith(`.${domain}`) || + domain.endsWith(`.${hostname}`))) { + return false; + } + if (!cookiePathMatches(cookie.path, parsedUrl?.pathname || '/')) { + return false; + } + if (cookie.secure && parsedUrl?.protocol !== 'https:') { + return false; + } + return true; +}; +const normalizeSameSite = (value) => { + const normalized = String(value ?? '') + .trim() + .toLowerCase(); + if (normalized === 'strict') { + return 'Strict'; + } + if (normalized === 'lax') { + return 'Lax'; + } + if (normalized === 'none') { + return 'None'; + } + return undefined; +}; +const parseCookieExpires = (value) => { + if (value == null || value === '') { + return undefined; + } + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : undefined; +}; +export class SkillCliServer { + registry; + constructor(options = {}) { + this.registry = options.registry ?? new SessionRegistry(); + } + async _require_node_by_index(browser_session, index) { + const parsedIndex = Number(index); + if (!Number.isFinite(parsedIndex)) { + throw new Error('Missing index'); + } + const node = await browser_session.get_dom_element_by_index(parsedIndex); + if (!node) { + return { + error: `Element index ${parsedIndex} not found - page may have changed`, + }; + } + return node; + } + async _read_node_data(browser_session, node, kind) { + if (!node?.xpath) { + throw new Error('DOM element does not include an XPath selector'); + } + const page = await browser_session.get_current_page(); + if (!page?.evaluate) { + throw new Error('No active page available'); + } + return await page.evaluate(({ xpath, dataKind }) => { + const element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + if (!element) { + return null; + } + if (dataKind === 'text') { + return element.textContent?.trim() ?? ''; + } + if (dataKind === 'value') { + return 'value' in element + ? String(element.value ?? '') + : null; + } + if (dataKind === 'attributes') { + return Object.fromEntries(Array.from(element.attributes).map((attribute) => [ + attribute.name, + attribute.value, + ])); + } + if (dataKind === 'bbox') { + const rect = element.getBoundingClientRect(); + return { + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + top: rect.top, + right: rect.right, + bottom: rect.bottom, + left: rect.left, + }; + } + return null; + }, { xpath: node.xpath, dataKind: kind }); + } + async _handle_browser_action(action, sessionName, params) { + const session = await this.registry.get_or_create_session(sessionName); + const browser_session = session.browser_session; + if (action === 'open') { + let url = String(params.url ?? '').trim(); + if (!url) { + throw new Error('Missing url'); + } + if (!/^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(url)) { + url = `https://${url}`; + } + await browser_session.navigate_to(url); + return { url }; + } + if (action === 'click') { + const node = await this._require_node_by_index(browser_session, params.index); + if ('error' in node) { + return node; + } + await browser_session._click_element_node(node); + return { clicked: Number(params.index) }; + } + if (action === 'hover') { + const node = await this._require_node_by_index(browser_session, params.index); + if ('error' in node) { + return node; + } + const locator = await browser_session.get_locate_element(node); + if (!locator?.hover) { + throw new Error('Hover is not available for this element'); + } + await locator.hover({ timeout: 5000 }); + return { hovered: Number(params.index) }; + } + if (action === 'dblclick') { + const node = await this._require_node_by_index(browser_session, params.index); + if ('error' in node) { + return node; + } + const locator = await browser_session.get_locate_element(node); + if (!locator?.dblclick) { + throw new Error('Double-click is not available for this element'); + } + await locator.dblclick({ timeout: 5000 }); + return { double_clicked: Number(params.index) }; + } + if (action === 'rightclick') { + const node = await this._require_node_by_index(browser_session, params.index); + if ('error' in node) { + return node; + } + const locator = await browser_session.get_locate_element(node); + if (!locator?.click) { + throw new Error('Right-click is not available for this element'); + } + await locator.click({ button: 'right', timeout: 5000 }); + return { right_clicked: Number(params.index) }; + } + if (action === 'type') { + const text = String(params.text ?? ''); + await browser_session.send_keys(text); + return { typed: text }; + } + if (action === 'input') { + const node = await this._require_node_by_index(browser_session, params.index); + if ('error' in node) { + return node; + } + const text = String(params.text ?? ''); + const clear = typeof params.clear === 'boolean' ? params.clear : true; + await browser_session._input_text_element_node(node, text, { clear }); + return { input: Number(params.index), text, clear }; + } + if (action === 'state') { + const state = await browser_session.get_browser_state_with_recovery({ + include_screenshot: false, + }); + const page_info = typeof browser_session.get_page_info === 'function' + ? await browser_session.get_page_info() + : null; + return { + url: state.url, + title: state.title, + tabs: state.tabs, + page_info, + llm_representation: state.llm_representation(), + }; + } + if (action === 'screenshot') { + const screenshot = await browser_session.take_screenshot(Boolean(params.full)); + if (!screenshot) { + throw new Error('Failed to capture screenshot'); + } + const file = typeof params.file === 'string' ? params.file.trim() : ''; + if (!file) { + return { screenshot }; + } + const filePath = path.resolve(file); + await fsp.writeFile(filePath, Buffer.from(screenshot, 'base64')); + return { file: filePath }; + } + if (action === 'wait_selector') { + const selector = String(params.selector ?? ''); + if (!selector) { + throw new Error('Missing selector'); + } + const timeout = Number(params.timeout ?? 5000); + await browser_session.wait_for_element(selector, timeout); + return { waited_for: 'selector', selector, timeout }; + } + if (action === 'wait_text') { + const text = String(params.text ?? ''); + if (!text) { + throw new Error('Missing text'); + } + const timeout = Number(params.timeout ?? 5000); + const page = await browser_session.get_current_page(); + if (!page?.waitForFunction) { + throw new Error('No active page available for wait_text'); + } + await page.waitForFunction((needle) => document.body?.innerText?.includes(needle) ?? false, text, { timeout }); + return { waited_for: 'text', text, timeout }; + } + if (action === 'scroll') { + let direction = 'down'; + if (typeof params.direction === 'string' && + ['up', 'down', 'left', 'right'].includes(params.direction)) { + direction = params.direction; + } + const amount = Number(params.amount ?? 500); + await browser_session.scroll(direction, amount); + return { direction, amount }; + } + if (action === 'back') { + await browser_session.go_back(); + return { navigated: 'back' }; + } + if (action === 'forward') { + await browser_session.go_forward(); + return { navigated: 'forward' }; + } + if (action === 'switch') { + const identifier = params.tab ?? params.target_id; + if (typeof identifier !== 'string' && typeof identifier !== 'number') { + throw new Error('Missing tab'); + } + await browser_session.switch_to_tab(identifier); + return { + active_tab: browser_session.active_tab?.target_id ?? + browser_session.active_tab?.tab_id ?? + browser_session.active_tab?.page_id ?? + null, + }; + } + if (action === 'close_tab' || action === 'close-tab') { + const identifier = params.tab ?? + params.target_id ?? + browser_session.active_tab?.target_id ?? + browser_session.active_tab?.page_id ?? + browser_session.active_tab_index; + if (typeof identifier !== 'string' && typeof identifier !== 'number') { + throw new Error('Missing tab'); + } + await browser_session.close_tab(identifier); + return { closed_tab: identifier }; + } + if (action === 'keys') { + const keys = String(params.keys ?? ''); + if (!keys) { + throw new Error('Missing keys'); + } + await browser_session.send_keys(keys); + return { keys }; + } + if (action === 'select') { + const node = await this._require_node_by_index(browser_session, params.index); + if ('error' in node) { + return node; + } + const value = String(params.value ?? ''); + if (!value) { + throw new Error('Missing value'); + } + const selected = await browser_session.select_dropdown_option(node, value); + return { + index: Number(params.index), + value, + selected, + }; + } + if (action === 'html') { + const selector = typeof params.selector === 'string' ? params.selector.trim() : ''; + if (!selector) { + return { html: await browser_session.get_page_html() }; + } + const page = await browser_session.get_current_page(); + if (!page?.evaluate) { + throw new Error('No active page available for html'); + } + const html = await page.evaluate((targetSelector) => { + const element = document.querySelector(targetSelector); + return element ? element.outerHTML : null; + }, selector); + if (typeof html !== 'string' || html.length === 0) { + throw new Error(`No element found for selector: ${selector}`); + } + return { selector, html }; + } + if (action === 'eval') { + const script = String(params.js ?? params.script ?? '').trim(); + if (!script) { + throw new Error('Missing js'); + } + return { + result: await browser_session.execute_javascript(script), + }; + } + if (action === 'extract') { + const query = String(params.query ?? '').trim(); + if (!query) { + throw new Error('Missing query'); + } + return { + query, + error: 'extract requires agent mode - use: browser-use run "extract ..."', + }; + } + if (action === 'get_title') { + const page = await browser_session.get_current_page?.(); + if (!page?.title) { + throw new Error('No active page available for get_title'); + } + return { + title: await page.title(), + }; + } + if (action === 'get_html') { + const selector = typeof params.selector === 'string' ? params.selector.trim() : ''; + return selector + ? await this._handle_browser_action('html', sessionName, { selector }) + : await this._handle_browser_action('html', sessionName, {}); + } + if (action === 'get_text' || + action === 'get_value' || + action === 'get_attributes' || + action === 'get_bbox') { + const node = await this._require_node_by_index(browser_session, params.index); + if ('error' in node) { + return node; + } + const kind = action.replace('get_', ''); + const value = await this._read_node_data(browser_session, node, kind); + if (value == null) { + throw new Error(`Unable to retrieve ${kind} for element`); + } + return { + index: Number(params.index), + [kind]: value, + }; + } + if (action === 'cookies_get') { + const url = typeof params.url === 'string' ? params.url.trim() : ''; + const allCookies = (await browser_session.get_cookies()); + const cookies = url + ? allCookies.filter((cookie) => cookieMatchesUrl(cookie, url)) + : allCookies; + return { cookies, count: cookies.length }; + } + if (action === 'cookies_set') { + const name = String(params.name ?? '').trim(); + const value = String(params.value ?? ''); + if (!name) { + throw new Error('Missing cookie name'); + } + if (!browser_session.browser_context?.addCookies) { + throw new Error('Browser context does not support setting cookies'); + } + const currentPage = await browser_session.get_current_page?.(); + const currentUrl = typeof currentPage?.url === 'function' ? currentPage.url() : ''; + const cookie = { + name, + value, + url: typeof params.url === 'string' && params.url.trim().length > 0 + ? params.url.trim() + : undefined, + domain: typeof params.domain === 'string' ? params.domain.trim() : undefined, + path: typeof params.path === 'string' ? params.path : '/', + secure: Boolean(params.secure), + httpOnly: Boolean(params.http_only), + sameSite: normalizeSameSite(params.same_site ?? params.sameSite), + expires: parseCookieExpires(params.expires), + }; + if (!cookie.url && !cookie.domain && currentUrl) { + cookie.url = currentUrl; + } + if (!cookie.url && !cookie.domain) { + throw new Error('Provide cookie url/domain or open a page first'); + } + await browser_session.browser_context.addCookies([cookie]); + return { set: name }; + } + if (action === 'cookies_clear') { + if (!browser_session.browser_context?.clearCookies) { + throw new Error('Browser context does not support clearing cookies'); + } + const url = typeof params.url === 'string' ? params.url.trim() : ''; + if (!url) { + await browser_session.browser_context.clearCookies(); + return { cleared: true }; + } + const allCookies = (await browser_session.get_cookies()); + const remainingCookies = allCookies.filter((cookie) => !cookieMatchesUrl(cookie, url)); + const removedCount = allCookies.length - remainingCookies.length; + await browser_session.browser_context.clearCookies(); + if (remainingCookies.length > 0 && + browser_session.browser_context.addCookies) { + await browser_session.browser_context.addCookies(remainingCookies); + } + return { cleared: true, url, count: removedCount }; + } + if (action === 'cookies_export') { + const file = String(params.file ?? '').trim(); + if (!file) { + throw new Error('Missing file'); + } + const url = typeof params.url === 'string' ? params.url.trim() : ''; + const allCookies = (await browser_session.get_cookies()); + const cookies = url + ? allCookies.filter((cookie) => cookieMatchesUrl(cookie, url)) + : allCookies; + const filePath = path.resolve(file); + await fsp.writeFile(filePath, JSON.stringify(cookies, null, 2)); + return { file: filePath, count: cookies.length }; + } + if (action === 'cookies_import') { + const file = String(params.file ?? '').trim(); + if (!file) { + throw new Error('Missing file'); + } + if (!browser_session.browser_context?.addCookies) { + throw new Error('Browser context does not support importing cookies'); + } + const filePath = path.resolve(file); + const raw = await fsp.readFile(filePath, 'utf8'); + const cookies = JSON.parse(raw); + if (!Array.isArray(cookies)) { + throw new Error('Cookie import file must contain a JSON array'); + } + const importedCookies = cookies.map((cookie) => { + if (!cookie || typeof cookie !== 'object') { + throw new Error('Each imported cookie must be a JSON object'); + } + const typedCookie = cookie; + if (typeof typedCookie.name !== 'string' || + typeof typedCookie.value !== 'string') { + throw new Error('Each imported cookie must include string name/value'); + } + return typedCookie; + }); + await browser_session.browser_context.addCookies(importedCookies); + return { file: filePath, imported: importedCookies.length }; + } + if (action === 'close') { + await this.registry.close_session(sessionName); + return { closed: sessionName }; + } + if (action === 'sessions') { + const sessions = this.registry.list_sessions(); + return { sessions, count: sessions.length }; + } + throw new Error(`Unknown action: ${action}`); + } + async handle_request(request) { + const req = typeof request === 'string' ? Request.from_json(request) : request; + try { + const data = await this._handle_browser_action(req.action, req.session, req.params); + if (data && typeof data === 'object' && 'error' in data) { + return new Response({ + id: req.id, + success: false, + data: null, + error: String(data.error), + }); + } + return new Response({ + id: req.id, + success: true, + data, + }); + } + catch (error) { + return new Response({ + id: req.id, + success: false, + error: String(error?.message ?? error), + }); + } + } +} diff --git a/dist/skill-cli/sessions.d.ts b/dist/skill-cli/sessions.d.ts new file mode 100644 index 00000000..dd8fc5fc --- /dev/null +++ b/dist/skill-cli/sessions.d.ts @@ -0,0 +1,24 @@ +import { BrowserSession } from '../browser/session.js'; +export interface SessionInfo { + name: string; + browser_session: BrowserSession; + created_at: Date; + updated_at: Date; +} +export interface SessionRegistryOptions { + session_factory?: (name: string) => BrowserSession; +} +export declare class SessionRegistry { + private readonly sessions; + private readonly session_factory; + constructor(options?: SessionRegistryOptions); + get_or_create_session(name: string): Promise; + list_sessions(): { + name: string; + created_at: string; + updated_at: string; + tab_count: number; + }[]; + close_session(name: string): Promise; + close_all(): Promise; +} diff --git a/dist/skill-cli/sessions.js b/dist/skill-cli/sessions.js new file mode 100644 index 00000000..ace97099 --- /dev/null +++ b/dist/skill-cli/sessions.js @@ -0,0 +1,47 @@ +import { BrowserSession } from '../browser/session.js'; +export class SessionRegistry { + sessions = new Map(); + session_factory; + constructor(options = {}) { + this.session_factory = + options.session_factory ?? (() => new BrowserSession()); + } + async get_or_create_session(name) { + const existing = this.sessions.get(name); + if (existing) { + existing.updated_at = new Date(); + return existing; + } + const browser_session = this.session_factory(name); + const session = { + name, + browser_session, + created_at: new Date(), + updated_at: new Date(), + }; + this.sessions.set(name, session); + return session; + } + list_sessions() { + return [...this.sessions.values()].map((session) => ({ + name: session.name, + created_at: session.created_at.toISOString(), + updated_at: session.updated_at.toISOString(), + tab_count: session.browser_session.tabs.length, + })); + } + async close_session(name) { + const existing = this.sessions.get(name); + if (!existing) { + return false; + } + await existing.browser_session.stop(); + this.sessions.delete(name); + return true; + } + async close_all() { + for (const name of [...this.sessions.keys()]) { + await this.close_session(name); + } + } +} diff --git a/dist/skill-cli/tunnel.d.ts b/dist/skill-cli/tunnel.d.ts new file mode 100644 index 00000000..bcb2babc --- /dev/null +++ b/dist/skill-cli/tunnel.d.ts @@ -0,0 +1,61 @@ +import { spawn } from 'node:child_process'; +export type TunnelStatus = { + available: boolean; + source: 'system' | null; + path: string | null; + note: string; +}; +export type StartTunnelResult = { + url: string; + port: number; + existing?: boolean; +} | { + error: string; +}; +export type ListTunnelsResult = { + tunnels: Array<{ + port: number; + url: string; + }>; + count: number; +}; +export type StopTunnelResult = { + stopped: number; + url: string; +} | { + error: string; +}; +export type StopAllTunnelsResult = { + stopped: number[]; + count: number; +}; +export interface TunnelManagerOptions { + tunnel_dir?: string; + binary_resolver?: (binary: string) => string | null; + spawn_impl?: typeof spawn; + sleep_impl?: (ms: number) => Promise; + is_process_alive?: (pid: number) => boolean; + kill_process?: (pid: number) => Promise; +} +export declare class TunnelManager { + private readonly tunnel_dir; + private readonly binary_resolver; + private readonly spawn_impl; + private readonly sleep_impl; + private readonly is_process_alive_impl; + private readonly kill_process_impl; + private binary_path; + constructor(options?: TunnelManagerOptions); + private get_tunnel_file; + private get_tunnel_log_file; + private save_tunnel_info; + private load_tunnel_info; + get_binary_path(): string; + is_available(): boolean; + get_status(): TunnelStatus; + start_tunnel(port: number): Promise; + list_tunnels(): ListTunnelsResult; + stop_tunnel(port: number): Promise; + stop_all_tunnels(): Promise; +} +export declare const get_tunnel_manager: () => TunnelManager; diff --git a/dist/skill-cli/tunnel.js b/dist/skill-cli/tunnel.js new file mode 100644 index 00000000..3d39b611 --- /dev/null +++ b/dist/skill-cli/tunnel.js @@ -0,0 +1,257 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn, spawnSync } from 'node:child_process'; +const TUNNEL_URL_PATTERN = /(https:\/\/\S+\.trycloudflare\.com)/; +const DEFAULT_TUNNELS_DIR = path.join(os.homedir(), '.browser-use', 'tunnels'); +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const findSystemBinary = (binary) => { + const command = process.platform === 'win32' ? 'where' : 'which'; + const result = spawnSync(command, [binary], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + if (result.status !== 0) { + return null; + } + return (result.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .find(Boolean) ?? null); +}; +export class TunnelManager { + tunnel_dir; + binary_resolver; + spawn_impl; + sleep_impl; + is_process_alive_impl; + kill_process_impl; + binary_path = null; + constructor(options = {}) { + this.tunnel_dir = options.tunnel_dir ?? DEFAULT_TUNNELS_DIR; + this.binary_resolver = options.binary_resolver ?? findSystemBinary; + this.spawn_impl = options.spawn_impl ?? spawn; + this.sleep_impl = options.sleep_impl ?? sleep; + this.is_process_alive_impl = + options.is_process_alive ?? default_is_process_alive; + this.kill_process_impl = options.kill_process ?? default_kill_process; + } + get_tunnel_file(port) { + return path.join(this.tunnel_dir, `${port}.json`); + } + get_tunnel_log_file(port) { + return path.join(this.tunnel_dir, `${port}.log`); + } + save_tunnel_info(port, pid, url) { + fs.mkdirSync(this.tunnel_dir, { recursive: true }); + fs.writeFileSync(this.get_tunnel_file(port), JSON.stringify({ port, pid, url }), 'utf-8'); + } + load_tunnel_info(port) { + const filePath = this.get_tunnel_file(port); + if (!fs.existsSync(filePath)) { + return null; + } + try { + const parsed = JSON.parse(fs.readFileSync(filePath, 'utf-8')); + if (!parsed || + typeof parsed.port !== 'number' || + typeof parsed.pid !== 'number' || + typeof parsed.url !== 'string') { + fs.rmSync(filePath, { force: true }); + return null; + } + if (!this.is_process_alive_impl(parsed.pid)) { + fs.rmSync(filePath, { force: true }); + fs.rmSync(this.get_tunnel_log_file(port), { force: true }); + return null; + } + return { + port: parsed.port, + pid: parsed.pid, + url: parsed.url, + }; + } + catch { + fs.rmSync(filePath, { force: true }); + return null; + } + } + get_binary_path() { + if (this.binary_path) { + return this.binary_path; + } + const systemBinary = this.binary_resolver('cloudflared'); + if (systemBinary) { + this.binary_path = systemBinary; + return systemBinary; + } + throw new Error('cloudflared not installed.\n\nInstall cloudflared:\n macOS: brew install cloudflared\n Linux: curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o ~/.local/bin/cloudflared && chmod +x ~/.local/bin/cloudflared\n Windows: winget install Cloudflare.cloudflared'); + } + is_available() { + if (this.binary_path) { + return true; + } + return Boolean(this.binary_resolver('cloudflared')); + } + get_status() { + const systemBinary = this.binary_resolver('cloudflared'); + if (systemBinary) { + return { + available: true, + source: 'system', + path: systemBinary, + note: 'cloudflared installed', + }; + } + return { + available: false, + source: null, + path: null, + note: 'cloudflared not installed - install it manually before using tunnel', + }; + } + async start_tunnel(port) { + const existing = this.load_tunnel_info(port); + if (existing) { + return { url: existing.url, port, existing: true }; + } + let binaryPath; + try { + binaryPath = this.get_binary_path(); + } + catch (error) { + return { error: error.message }; + } + fs.mkdirSync(this.tunnel_dir, { recursive: true }); + const logPath = this.get_tunnel_log_file(port); + const logFd = fs.openSync(logPath, 'w'); + try { + const child = this.spawn_impl(binaryPath, ['tunnel', '--url', `http://localhost:${port}`], { + detached: true, + stdio: ['ignore', 'ignore', logFd], + }); + child.unref?.(); + const deadline = Date.now() + 15_000; + while (Date.now() < deadline) { + const pid = child.pid; + if (typeof pid === 'number' && !this.is_process_alive_impl(pid)) { + const content = fs.existsSync(logPath) + ? fs.readFileSync(logPath, 'utf-8') + : ''; + return { + error: `cloudflared exited unexpectedly: ${content.slice(0, 500)}`, + }; + } + const content = fs.existsSync(logPath) + ? fs.readFileSync(logPath, 'utf-8') + : ''; + const match = content.match(TUNNEL_URL_PATTERN); + if (match?.[1] && typeof child.pid === 'number') { + this.save_tunnel_info(port, child.pid, match[1]); + return { url: match[1], port }; + } + await this.sleep_impl(200); + } + if (typeof child.pid === 'number') { + await this.kill_process_impl(child.pid); + } + return { error: 'Timed out waiting for cloudflare tunnel URL (15s)' }; + } + finally { + fs.closeSync(logFd); + } + } + list_tunnels() { + const tunnels = []; + if (!fs.existsSync(this.tunnel_dir)) { + return { tunnels, count: 0 }; + } + for (const entry of fs.readdirSync(this.tunnel_dir)) { + if (!entry.endsWith('.json')) { + continue; + } + const port = Number.parseInt(path.basename(entry, '.json'), 10); + if (!Number.isFinite(port)) { + continue; + } + const info = this.load_tunnel_info(port); + if (info) { + tunnels.push({ port: info.port, url: info.url }); + } + } + return { tunnels, count: tunnels.length }; + } + async stop_tunnel(port) { + const info = this.load_tunnel_info(port); + if (!info) { + return { error: `No tunnel running on port ${port}` }; + } + await this.kill_process_impl(info.pid); + fs.rmSync(this.get_tunnel_file(port), { force: true }); + fs.rmSync(this.get_tunnel_log_file(port), { force: true }); + return { + stopped: port, + url: info.url, + }; + } + async stop_all_tunnels() { + const stopped = []; + if (!fs.existsSync(this.tunnel_dir)) { + return { stopped, count: 0 }; + } + for (const entry of fs.readdirSync(this.tunnel_dir)) { + if (!entry.endsWith('.json')) { + continue; + } + const port = Number.parseInt(path.basename(entry, '.json'), 10); + if (!Number.isFinite(port)) { + continue; + } + const result = await this.stop_tunnel(port); + if ('stopped' in result) { + stopped.push(result.stopped); + } + } + return { + stopped, + count: stopped.length, + }; + } +} +const default_is_process_alive = (pid) => { + try { + process.kill(pid, 0); + return true; + } + catch { + return false; + } +}; +const default_kill_process = async (pid) => { + try { + process.kill(pid, 'SIGTERM'); + } + catch { + return false; + } + for (let attempt = 0; attempt < 10; attempt += 1) { + if (!default_is_process_alive(pid)) { + return true; + } + await sleep(100); + } + try { + process.kill(pid, 'SIGKILL'); + return true; + } + catch { + return false; + } +}; +let tunnel_manager = null; +export const get_tunnel_manager = () => { + if (!tunnel_manager) { + tunnel_manager = new TunnelManager(); + } + return tunnel_manager; +}; diff --git a/dist/skills/index.d.ts b/dist/skills/index.d.ts new file mode 100644 index 00000000..192a7838 --- /dev/null +++ b/dist/skills/index.d.ts @@ -0,0 +1,3 @@ +export * from './views.js'; +export * from './utils.js'; +export * from './service.js'; diff --git a/dist/skills/index.js b/dist/skills/index.js new file mode 100644 index 00000000..192a7838 --- /dev/null +++ b/dist/skills/index.js @@ -0,0 +1,3 @@ +export * from './views.js'; +export * from './utils.js'; +export * from './service.js'; diff --git a/dist/skills/service.d.ts b/dist/skills/service.d.ts new file mode 100644 index 00000000..73883f16 --- /dev/null +++ b/dist/skills/service.d.ts @@ -0,0 +1,27 @@ +import { build_skill_parameters_schema } from './utils.js'; +import { type BrowserCookie, type ExecuteSkillInput, type SkillDefinition, type SkillExecutionResult, type SkillService } from './views.js'; +interface CloudSkillServiceOptions { + skill_ids: Array; + api_key?: string | null; + base_url?: string | null; + fetch_impl?: typeof fetch; +} +export declare class CloudSkillService implements SkillService { + private readonly skill_ids; + private readonly api_key; + private readonly base_url; + private readonly fetch_impl; + private initialized; + private readonly skills; + constructor(options: CloudSkillServiceOptions); + private requestJson; + private listSkillsPage; + private ensureInitialized; + get_skill(skill_id: string): Promise; + get_all_skills(): Promise; + execute_skill(input: ExecuteSkillInput): Promise; + close(): Promise; +} +export declare const register_skills_as_actions: (skills: SkillDefinition[], registerAction: (slug: string, description: string, params: ReturnType, skill: SkillDefinition) => void) => Promise; +export declare const cookies_to_map: (cookies: BrowserCookie[]) => Map; +export {}; diff --git a/dist/skills/service.js b/dist/skills/service.js new file mode 100644 index 00000000..243d0abb --- /dev/null +++ b/dist/skills/service.js @@ -0,0 +1,266 @@ +import { CONFIG } from '../config.js'; +import { createLogger } from '../logging-config.js'; +import { build_skill_parameters_schema, get_skill_slug } from './utils.js'; +import { MissingCookieException, } from './views.js'; +const logger = createLogger('browser_use.skills'); +const toSkillParameter = (raw) => { + if (!raw || typeof raw !== 'object') { + return null; + } + const record = raw; + const name = typeof record.name === 'string' ? record.name.trim() : ''; + const type = typeof record.type === 'string' ? record.type.trim() : ''; + if (!name || !type) { + return null; + } + if (type !== 'string' && + type !== 'number' && + type !== 'boolean' && + type !== 'object' && + type !== 'array' && + type !== 'cookie') { + return null; + } + return { + name, + type, + required: typeof record.required === 'boolean' ? record.required : undefined, + description: typeof record.description === 'string' ? record.description : undefined, + }; +}; +const toSkillDefinition = (raw) => { + if (!raw || typeof raw !== 'object') { + return null; + } + const record = raw; + const id = typeof record.id === 'string' ? record.id.trim() : ''; + const title = typeof record.title === 'string' ? record.title.trim() : ''; + const description = typeof record.description === 'string' ? record.description.trim() : ''; + if (!id || !title) { + return null; + } + const parametersRaw = Array.isArray(record.parameters) + ? record.parameters + : []; + const parameters = parametersRaw + .map((entry) => toSkillParameter(entry)) + .filter((entry) => entry != null); + const output_schema = record.output_schema && typeof record.output_schema === 'object' + ? record.output_schema + : null; + return { + id, + title, + description, + parameters, + output_schema, + }; +}; +export class CloudSkillService { + skill_ids; + api_key; + base_url; + fetch_impl; + initialized = false; + skills = new Map(); + constructor(options) { + this.skill_ids = options.skill_ids; + this.api_key = options.api_key ?? process.env.BROWSER_USE_API_KEY ?? ''; + this.base_url = options.base_url ?? CONFIG.BROWSER_USE_CLOUD_API_URL; + this.fetch_impl = options.fetch_impl ?? fetch; + if (!this.api_key) { + throw new Error('BROWSER_USE_API_KEY environment variable is not set'); + } + } + async requestJson(path, init = {}) { + const response = await this.fetch_impl(`${this.base_url}${path}`, { + ...init, + headers: { + Authorization: `Bearer ${this.api_key}`, + ...(init.body ? { 'Content-Type': 'application/json' } : {}), + ...(init.headers ?? {}), + }, + }); + let payload; + try { + payload = await response.json(); + } + catch { + payload = null; + } + if (!response.ok) { + const details = payload && typeof payload === 'object' + ? JSON.stringify(payload) + : String(payload ?? ''); + throw new Error(`Skill API request failed (${response.status}): ${details}`); + } + return payload; + } + async listSkillsPage(page_number, page_size) { + const query = new URLSearchParams({ + page_size: String(page_size), + page_number: String(page_number), + is_enabled: 'true', + }); + const payload = (await this.requestJson(`/api/v1/skills?${query.toString()}`)); + const items = Array.isArray(payload?.items) ? payload.items : []; + const skills = []; + for (const item of items) { + const status = item && typeof item === 'object' + ? item.status + : undefined; + if (typeof status === 'string' && status !== 'finished') { + continue; + } + const skill = toSkillDefinition(item); + if (skill) { + skills.push(skill); + } + } + return skills; + } + async ensureInitialized() { + if (this.initialized) { + return; + } + try { + const useWildcard = this.skill_ids.includes('*'); + const requestedIds = new Set(this.skill_ids.filter((entry) => entry !== '*')); + const page_size = 100; + const max_pages = useWildcard ? 1 : 5; + const loaded = []; + let reachedPaginationLimit = true; + for (let page = 1; page <= max_pages; page += 1) { + const pageSkills = await this.listSkillsPage(page, page_size); + loaded.push(...pageSkills); + if (pageSkills.length < page_size) { + reachedPaginationLimit = false; + break; + } + if (!useWildcard && requestedIds.size > 0) { + const found = new Set(loaded.map((entry) => entry.id).filter((id) => requestedIds.has(id))); + if (found.size === requestedIds.size) { + reachedPaginationLimit = false; + break; + } + } + } + if (useWildcard && loaded.length >= page_size) { + logger.warning('Wildcard "*" limited to first 100 skills. Specify explicit skill IDs if you need specific skills beyond this limit.'); + } + if (!useWildcard && reachedPaginationLimit) { + logger.warning('Reached pagination limit (5 pages) before finding all requested skills'); + } + const selected = useWildcard + ? loaded + : loaded.filter((entry) => requestedIds.has(entry.id)); + if (!useWildcard && requestedIds.size > 0) { + const foundIds = new Set(selected.map((entry) => entry.id)); + const missingIds = Array.from(requestedIds).filter((id) => !foundIds.has(id)); + if (missingIds.length > 0) { + logger.warning(`Requested skills not found or not available: ${missingIds.join(', ')}`); + } + } + for (const skill of selected) { + this.skills.set(skill.id, skill); + } + this.initialized = true; + logger.info(`Loaded ${this.skills.size} skills${useWildcard ? ' (wildcard mode)' : ''}`); + } + catch (error) { + // Match Python semantics: avoid retry loops after an initialization failure. + this.initialized = true; + throw error; + } + } + async get_skill(skill_id) { + await this.ensureInitialized(); + return this.skills.get(skill_id) ?? null; + } + async get_all_skills() { + await this.ensureInitialized(); + return Array.from(this.skills.values()); + } + async execute_skill(input) { + await this.ensureInitialized(); + const skill = this.skills.get(input.skill_id); + if (!skill) { + throw new Error(`Skill ${input.skill_id} not found in cache. Available skills: ${Array.from(this.skills.keys()).join(', ')}`); + } + const cookieMap = new Map(); + for (const cookie of input.cookies ?? []) { + if (!cookie?.name) { + continue; + } + cookieMap.set(cookie.name, cookie.value ?? ''); + } + const payload = { + ...(input.parameters ?? {}), + }; + for (const param of skill.parameters) { + if (param.type !== 'cookie') { + continue; + } + const required = param.required !== false; + if (required && !cookieMap.has(param.name)) { + throw new MissingCookieException(param.name, param.description || 'No description provided'); + } + if (cookieMap.has(param.name)) { + payload[param.name] = cookieMap.get(param.name) ?? ''; + } + } + const validator = build_skill_parameters_schema(skill.parameters, { + exclude_cookies: false, + }); + const validated = validator.safeParse(payload); + if (!validated.success) { + throw new Error(`Parameter validation failed for skill ${skill.title}: ${validated.error.message}`); + } + try { + const response = (await this.requestJson(`/api/v1/skills/${encodeURIComponent(input.skill_id)}/execute`, { + method: 'POST', + body: JSON.stringify({ parameters: validated.data }), + })); + const success = response.success === true; + const result = response.result ?? response.output ?? response.data ?? null; + const error = typeof response.error === 'string' ? response.error : null; + const latency_ms = typeof response.latency_ms === 'number' ? response.latency_ms : null; + return { + success, + result, + error, + latency_ms, + }; + } + catch (error) { + const errorText = error instanceof Error + ? `${error.name}: ${error.message}` + : String(error); + return { + success: false, + error: `Failed to execute skill: ${errorText}`, + }; + } + } + async close() { + this.skills.clear(); + this.initialized = false; + } +} +export const register_skills_as_actions = async (skills, registerAction) => { + for (const skill of skills) { + const slug = get_skill_slug(skill, skills); + const paramSchema = build_skill_parameters_schema(skill.parameters, { + exclude_cookies: true, + }); + const description = `${skill.description} (Skill: "${skill.title}")`; + registerAction(slug, description, paramSchema, skill); + } +}; +export const cookies_to_map = (cookies) => { + const map = new Map(); + for (const cookie of cookies) { + map.set(cookie.name, cookie.value); + } + return map; +}; diff --git a/dist/skills/utils.d.ts b/dist/skills/utils.d.ts new file mode 100644 index 00000000..828c14aa --- /dev/null +++ b/dist/skills/utils.d.ts @@ -0,0 +1,6 @@ +import { z } from 'zod'; +import type { SkillDefinition, SkillParameterSchema } from './views.js'; +export declare const get_skill_slug: (skill: SkillDefinition, all_skills: SkillDefinition[]) => string; +export declare const build_skill_parameters_schema: (parameters: SkillParameterSchema[], options?: { + exclude_cookies?: boolean; +}) => z.ZodObject>; diff --git a/dist/skills/utils.js b/dist/skills/utils.js new file mode 100644 index 00000000..522c9783 --- /dev/null +++ b/dist/skills/utils.js @@ -0,0 +1,53 @@ +import { z } from 'zod'; +const normalizeSlug = (title) => { + const cleaned = title + .toLowerCase() + .replace(/[^\w\s-]/g, '') + .replace(/[\s-]+/g, '_') + .replace(/^_+|_+$/g, ''); + return cleaned || 'skill'; +}; +const normalizeTypeSchema = (param) => { + switch (param.type) { + case 'string': + case 'cookie': + return z.string(); + case 'number': + return z.number(); + case 'boolean': + return z.boolean(); + case 'array': + return z.array(z.unknown()); + case 'object': + return z.record(z.string(), z.unknown()); + default: + return z.unknown(); + } +}; +export const get_skill_slug = (skill, all_skills) => { + const slug = normalizeSlug(skill.title); + const duplicateCount = all_skills.filter((entry) => { + return normalizeSlug(entry.title) === slug; + }).length; + if (duplicateCount > 1) { + return `${slug}_${skill.id.slice(0, 4)}`; + } + return slug; +}; +export const build_skill_parameters_schema = (parameters, options = {}) => { + const { exclude_cookies = false } = options; + const shape = {}; + for (const param of parameters) { + if (exclude_cookies && param.type === 'cookie') { + continue; + } + const description = typeof param.description === 'string' ? param.description.trim() : ''; + const required = param.required !== false; + let schema = normalizeTypeSchema(param); + if (description) { + schema = schema.describe(description); + } + shape[param.name] = required ? schema : schema.optional(); + } + return z.object(shape); +}; diff --git a/dist/skills/views.d.ts b/dist/skills/views.d.ts new file mode 100644 index 00000000..842c4f46 --- /dev/null +++ b/dist/skills/views.d.ts @@ -0,0 +1,40 @@ +export type SkillParameterType = 'string' | 'number' | 'boolean' | 'object' | 'array' | 'cookie'; +export interface SkillParameterSchema { + name: string; + type: SkillParameterType; + required?: boolean | null; + description?: string | null; +} +export interface SkillDefinition { + id: string; + title: string; + description: string; + parameters: SkillParameterSchema[]; + output_schema?: Record | null; +} +export interface SkillExecutionResult { + success: boolean; + result?: unknown; + error?: string | null; + latency_ms?: number | null; +} +export interface BrowserCookie { + name: string; + value: string; +} +export interface ExecuteSkillInput { + skill_id: string; + parameters: Record; + cookies: BrowserCookie[]; +} +export interface SkillService { + get_skill?(skill_id: string): Promise; + get_all_skills(): Promise; + execute_skill(input: ExecuteSkillInput): Promise; + close?(): Promise | void; +} +export declare class MissingCookieException extends Error { + cookie_name: string; + cookie_description: string; + constructor(cookie_name: string, cookie_description: string); +} diff --git a/dist/skills/views.js b/dist/skills/views.js new file mode 100644 index 00000000..a39df432 --- /dev/null +++ b/dist/skills/views.js @@ -0,0 +1,10 @@ +export class MissingCookieException extends Error { + cookie_name; + cookie_description; + constructor(cookie_name, cookie_description) { + super(`Missing required cookie '${cookie_name}': ${cookie_description}`); + this.name = 'MissingCookieException'; + this.cookie_name = cookie_name; + this.cookie_description = cookie_description; + } +} diff --git a/dist/sync/auth.d.ts b/dist/sync/auth.d.ts new file mode 100644 index 00000000..dd00ea01 --- /dev/null +++ b/dist/sync/auth.d.ts @@ -0,0 +1,35 @@ +import { type AxiosInstance } from 'axios'; +export declare const TEMP_USER_ID = "99999999-9999-9999-9999-999999999999"; +interface CloudAuthConfigData { + api_token: string | null; + user_id: string | null; + authorized_at: string | null; +} +export declare const load_cloud_auth_config: () => CloudAuthConfigData; +export declare const save_cloud_api_token: (api_token: string, user_id?: string | null) => void; +export declare class DeviceAuthClient { + private readonly baseUrl; + private readonly clientId; + private readonly scope; + private readonly httpClient?; + private authConfig; + private _deviceId; + constructor(baseUrl?: string, httpClient?: AxiosInstance); + get device_id(): string; + get is_authenticated(): boolean; + get api_token(): string | null; + get user_id(): string; + private get client(); + private buildUrl; + private postForm; + start_device_authorization(agent_session_id?: string | null): Promise>; + poll_for_token(device_code: string, interval?: number, timeout?: number): Promise | null>; + authenticate(agent_session_id?: string | null, show_instructions?: boolean): Promise; + get_headers(): { + Authorization: string; + } | { + Authorization?: undefined; + }; + clear_auth(): void; +} +export {}; diff --git a/dist/sync/auth.js b/dist/sync/auth.js new file mode 100644 index 00000000..b190e0cc --- /dev/null +++ b/dist/sync/auth.js @@ -0,0 +1,222 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import axios from 'axios'; +import { CONFIG } from '../config.js'; +import { createLogger } from '../logging-config.js'; +import { uuid7str } from '../utils.js'; +export const TEMP_USER_ID = '99999999-9999-9999-9999-999999999999'; +const logger = createLogger('browser_use.sync.auth'); +const CONFIG_DIR = () => CONFIG.BROWSER_USE_CONFIG_DIR ?? path.join(process.cwd(), '.browseruse'); +const DEVICE_ID_PATH = () => path.join(CONFIG_DIR(), 'device_id'); +const CLOUD_AUTH_PATH = () => path.join(CONFIG_DIR(), 'cloud_auth.json'); +const ensureDir = () => { + fs.mkdirSync(CONFIG_DIR(), { recursive: true }); +}; +const loadAuthConfig = () => { + try { + const contents = fs.readFileSync(CLOUD_AUTH_PATH(), 'utf-8'); + const parsed = JSON.parse(contents); + return { + api_token: parsed.api_token ?? null, + user_id: parsed.user_id ?? null, + authorized_at: parsed.authorized_at ?? null, + }; + } + catch { + return { api_token: null, user_id: null, authorized_at: null }; + } +}; +const saveAuthConfig = (config) => { + ensureDir(); + fs.writeFileSync(CLOUD_AUTH_PATH(), JSON.stringify(config, null, 2), 'utf-8'); + try { + fs.chmodSync(CLOUD_AUTH_PATH(), 0o600); + } + catch { + /* noop */ + } +}; +export const load_cloud_auth_config = () => loadAuthConfig(); +export const save_cloud_api_token = (api_token, user_id = TEMP_USER_ID) => { + const normalized = api_token.trim(); + if (!normalized) { + throw new Error('API token cannot be empty'); + } + saveAuthConfig({ + api_token: normalized, + user_id, + authorized_at: new Date().toISOString(), + }); +}; +const getOrCreateDeviceId = () => { + ensureDir(); + try { + const existing = fs.readFileSync(DEVICE_ID_PATH(), 'utf-8').trim(); + if (existing) { + return existing; + } + } + catch { + /* continue */ + } + const deviceId = uuid7str(); + fs.writeFileSync(DEVICE_ID_PATH(), deviceId, 'utf-8'); + return deviceId; +}; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const stripTrailingSlash = (input) => input.replace(/\/+$/, ''); +const terminalWidth = () => Math.max((process.stdout?.columns ?? 80) - 40, 20); +export class DeviceAuthClient { + baseUrl; + clientId = 'library'; + scope = 'read write'; + httpClient; + authConfig; + _deviceId; + constructor(baseUrl, httpClient) { + this.baseUrl = stripTrailingSlash(baseUrl ?? CONFIG.BROWSER_USE_CLOUD_API_URL); + this.httpClient = httpClient; + this.authConfig = loadAuthConfig(); + this._deviceId = getOrCreateDeviceId(); + } + get device_id() { + return this._deviceId; + } + get is_authenticated() { + return Boolean(this.authConfig.api_token && this.authConfig.user_id); + } + get api_token() { + return this.authConfig.api_token; + } + get user_id() { + return this.authConfig.user_id ?? TEMP_USER_ID; + } + get client() { + return this.httpClient ?? axios; + } + buildUrl(pathname) { + return `${this.baseUrl}${pathname}`; + } + async postForm(pathname, data) { + const form = new URLSearchParams(); + for (const [key, value] of Object.entries(data)) { + if (value !== undefined && value !== null) { + form.append(key, String(value)); + } + } + return this.client.post(this.buildUrl(pathname), form, { + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + }); + } + async start_device_authorization(agent_session_id) { + const response = await this.postForm('/api/v1/oauth/device/authorize', { + client_id: this.clientId, + scope: this.scope, + agent_session_id: agent_session_id ?? '', + device_id: this.device_id, + }); + return response.data; + } + async poll_for_token(device_code, interval = 3, timeout = 1800) { + const started = Date.now(); + let delay = interval; + while (Date.now() - started < timeout * 1000) { + try { + const response = await this.postForm('/api/v1/oauth/device/token', { + grant_type: 'urn:ietf:params:oauth:grant-type:device_code', + device_code, + client_id: this.clientId, + }); + const data = response.data; + if (data.error === 'authorization_pending') { + await sleep(delay * 1000); + continue; + } + if (data.error === 'slow_down') { + delay = data.interval ?? delay * 2; + await sleep(delay * 1000); + continue; + } + if (data.error) { + logger.warning(`Device token error: ${data.error}`); + return null; + } + if (data.access_token) { + return data; + } + } + catch (error) { + const status = error?.response?.status; + const payload = error?.response?.data; + if (status === 400 && + payload?.error && + ['authorization_pending', 'slow_down'].includes(payload.error)) { + if (payload.error === 'slow_down') { + delay = payload.interval ?? delay * 2; + } + await sleep(delay * 1000); + continue; + } + logger.debug(`Error polling for token: ${error?.message ?? error}`); + return null; + } + await sleep(delay * 1000); + } + return null; + } + async authenticate(agent_session_id, show_instructions = true) { + try { + const deviceAuth = await this.start_device_authorization(agent_session_id); + const frontendBase = CONFIG.BROWSER_USE_CLOUD_UI_URL || + this.baseUrl.replace('//api.', '//cloud.'); + const replaceHost = (value) => value?.replace(this.baseUrl, frontendBase); + const verificationUri = replaceHost(deviceAuth.verification_uri); + const verificationUriComplete = replaceHost(deviceAuth.verification_uri_complete); + if (show_instructions && CONFIG.BROWSER_USE_CLOUD_SYNC) { + const divider = '─'.repeat(terminalWidth()); + logger.info(divider); + logger.info('🌐 View the details of this run in Browser Use Cloud:'); + logger.info(` 👉 ${verificationUriComplete}`); + logger.info(divider + '\n'); + } + const tokenData = await this.poll_for_token(deviceAuth.device_code, deviceAuth.interval ?? 5); + if (tokenData?.access_token) { + this.authConfig = { + api_token: tokenData.access_token, + user_id: tokenData.user_id ?? this.user_id, + authorized_at: new Date().toISOString(), + }; + saveAuthConfig(this.authConfig); + if (show_instructions) { + logger.debug('✅ Authentication successful, cloud sync enabled.'); + } + return true; + } + } + catch (error) { + const status = error?.response?.status; + if (status === 404) { + logger.warning('Cloud sync authentication endpoint not found (404).'); + } + else { + logger.warning(`Cloud sync auth error: ${error?.message ?? error}`); + } + } + if (show_instructions) { + logger.debug(`❌ Sync authentication failed for ${this.baseUrl}`); + } + return false; + } + get_headers() { + return this.api_token ? { Authorization: `Bearer ${this.api_token}` } : {}; + } + clear_auth() { + this.authConfig = { api_token: null, user_id: null, authorized_at: null }; + try { + fs.unlinkSync(CLOUD_AUTH_PATH()); + } + catch { + /* noop */ + } + } +} diff --git a/dist/sync/index.d.ts b/dist/sync/index.d.ts new file mode 100644 index 00000000..74ebab7a --- /dev/null +++ b/dist/sync/index.d.ts @@ -0,0 +1,2 @@ +export * from './auth.js'; +export * from './service.js'; diff --git a/dist/sync/index.js b/dist/sync/index.js new file mode 100644 index 00000000..74ebab7a --- /dev/null +++ b/dist/sync/index.js @@ -0,0 +1,2 @@ +export * from './auth.js'; +export * from './service.js'; diff --git a/dist/sync/service.d.ts b/dist/sync/service.d.ts new file mode 100644 index 00000000..7fde2b89 --- /dev/null +++ b/dist/sync/service.d.ts @@ -0,0 +1,21 @@ +import type { BaseEvent } from '../agent/cloud-events.js'; +import { DeviceAuthClient } from './auth.js'; +export interface CloudSyncOptions { + baseUrl?: string; + enableAuth?: boolean; + allowSessionEventsForAuth?: boolean; +} +export declare class CloudSync { + private readonly baseUrl; + private readonly enabled; + readonly auth_client: DeviceAuthClient; + private sessionId; + private allowSessionEventsForAuth; + private authFlowActive; + constructor(options?: CloudSyncOptions); + handle_event(event: BaseEvent): Promise; + private sendEvent; + set_auth_flow_active(): void; + wait_for_auth(): Promise; + authenticate(showInstructions?: boolean): Promise; +} diff --git a/dist/sync/service.js b/dist/sync/service.js new file mode 100644 index 00000000..20a30483 --- /dev/null +++ b/dist/sync/service.js @@ -0,0 +1,111 @@ +import axios from 'axios'; +import { createLogger } from '../logging-config.js'; +import { CONFIG } from '../config.js'; +import { DeviceAuthClient, TEMP_USER_ID } from './auth.js'; +const logger = createLogger('browser_use.sync'); +const stripTrailingSlash = (input) => input.replace(/\/+$/, ''); +const ensureArray = (value) => Array.isArray(value) ? value : [value]; +export class CloudSync { + baseUrl; + enabled; + auth_client; + sessionId = null; + allowSessionEventsForAuth; + authFlowActive = false; + constructor(options = {}) { + const enableAuth = options.enableAuth ?? true; + this.baseUrl = stripTrailingSlash(options.baseUrl ?? CONFIG.BROWSER_USE_CLOUD_API_URL); + this.enabled = CONFIG.BROWSER_USE_CLOUD_SYNC && enableAuth; + this.allowSessionEventsForAuth = options.allowSessionEventsForAuth ?? false; + this.auth_client = new DeviceAuthClient(this.baseUrl); + } + async handle_event(event) { + try { + if (!this.enabled) { + return; + } + if (event.event_type === 'CreateAgentSessionEvent' && + event?.id != null) { + this.sessionId = String(event.id); + } + if (this.auth_client.is_authenticated) { + await this.sendEvent(event); + return; + } + if (this.allowSessionEventsForAuth || this.authFlowActive) { + await this.sendEvent(event); + if (event.event_type === 'CreateAgentSessionEvent') { + this.authFlowActive = true; + } + return; + } + logger.debug(`Skipping event ${event.event_type} - user not authenticated`); + } + catch (error) { + logger.error(`Failed to handle ${event.event_type}: ${error.message}`); + } + } + async sendEvent(event) { + try { + const headers = {}; + const currentUserId = event.user_id ?? null; + if (this.auth_client.is_authenticated) { + if (currentUserId !== TEMP_USER_ID) { + event.user_id = this.auth_client.user_id; + } + } + else if (!currentUserId) { + event.user_id = TEMP_USER_ID; + } + Object.assign(headers, this.auth_client.get_headers()); + event.device_id = + event.device_id ?? this.auth_client.device_id; + const payload = typeof event?.toJSON === 'function' + ? event.toJSON() + : { ...event }; + const events = ensureArray(payload); + await axios.post(`${this.baseUrl}/api/v1/events`, { + events: events.map((entry) => ({ + ...entry, + device_id: entry.device_id ?? this.auth_client.device_id, + })), + }, { headers, timeout: 10_000 }); + } + catch (error) { + const status = error?.response?.status; + if (status) { + logger.debug(`Failed to send sync event: POST ${this.baseUrl}/api/v1/events ${status} - ${String(error?.response?.data ?? '')}`); + } + else if (error?.code === 'ECONNABORTED') { + logger.debug(`Event send timed out after 10 seconds: ${event}`); + } + else { + logger.debug(`Unexpected error sending event ${event}: ${typeOfError(error)}`); + } + } + } + set_auth_flow_active() { + this.authFlowActive = true; + this.allowSessionEventsForAuth = true; + } + // Backward-compatible no-op; auth is no longer background-scheduled here. + async wait_for_auth() { } + async authenticate(showInstructions = true) { + if (!this.enabled) { + return false; + } + if (this.auth_client.is_authenticated) { + if (showInstructions) { + logger.info('✅ Already authenticated! Skipping OAuth flow.'); + } + return true; + } + return this.auth_client.authenticate(this.sessionId, showInstructions); + } +} +const typeOfError = (error) => { + if (error instanceof Error) { + return `${error.name}: ${error.message}`; + } + return String(error); +}; diff --git a/dist/telemetry/index.d.ts b/dist/telemetry/index.d.ts new file mode 100644 index 00000000..c6dd1f8b --- /dev/null +++ b/dist/telemetry/index.d.ts @@ -0,0 +1,2 @@ +export * from './service.js'; +export * from './views.js'; diff --git a/dist/telemetry/index.js b/dist/telemetry/index.js new file mode 100644 index 00000000..c6dd1f8b --- /dev/null +++ b/dist/telemetry/index.js @@ -0,0 +1,2 @@ +export * from './service.js'; +export * from './views.js'; diff --git a/dist/telemetry/service.d.ts b/dist/telemetry/service.d.ts new file mode 100644 index 00000000..ae4fd7fa --- /dev/null +++ b/dist/telemetry/service.d.ts @@ -0,0 +1,12 @@ +import type { BaseTelemetryEvent } from './views.js'; +export declare class ProductTelemetry { + private client; + private debugLogging; + private userIdFile; + private cachedUserId; + constructor(); + capture(event: BaseTelemetryEvent): void; + flush(): void; + get userId(): string; +} +export declare const productTelemetry: ProductTelemetry; diff --git a/dist/telemetry/service.js b/dist/telemetry/service.js new file mode 100644 index 00000000..43e4773d --- /dev/null +++ b/dist/telemetry/service.js @@ -0,0 +1,85 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { PostHog } from 'posthog-node'; +import { createLogger } from '../logging-config.js'; +import { CONFIG } from '../config.js'; +import { uuid7str } from '../utils.js'; +const logger = createLogger('browser_use.telemetry'); +const POSTHOG_EVENT_SETTINGS = { + process_person_profile: true, +}; +export class ProductTelemetry { + client = null; + debugLogging; + userIdFile; + cachedUserId = null; + constructor() { + this.debugLogging = CONFIG.BROWSER_USE_LOGGING_LEVEL === 'debug'; + this.userIdFile = path.join(CONFIG.BROWSER_USE_CONFIG_DIR, 'device_id'); + if (!CONFIG.ANONYMIZED_TELEMETRY) { + logger.debug('Telemetry disabled'); + return; + } + try { + this.client = new PostHog('phc_F8JMNjW1i2KbGUTaW1unnDdLSPCoyc52SGRU0JecaUh', { + host: 'https://eu.i.posthog.com', + disableGeoip: false, + enableExceptionAutocapture: true, + }); + } + catch (error) { + logger.error(`Failed to initialize PostHog client: ${error.message}`); + this.client = null; + } + } + capture(event) { + if (!this.client) { + return; + } + try { + this.client.capture({ + distinctId: this.userId, + event: event.name, + properties: { + ...event.properties(), + ...POSTHOG_EVENT_SETTINGS, + }, + }); + } + catch (error) { + logger.error(`Failed to send telemetry event ${event.name}: ${error.message}`); + } + } + flush() { + if (!this.client) { + return; + } + try { + this.client.flush(); + logger.debug('PostHog client telemetry queue flushed.'); + } + catch (error) { + logger.error(`Failed to flush PostHog client: ${error.message}`); + } + } + get userId() { + if (this.cachedUserId) { + return this.cachedUserId; + } + try { + if (!fs.existsSync(this.userIdFile)) { + fs.mkdirSync(path.dirname(this.userIdFile), { recursive: true }); + this.cachedUserId = uuid7str(); + fs.writeFileSync(this.userIdFile, this.cachedUserId, 'utf-8'); + } + else { + this.cachedUserId = fs.readFileSync(this.userIdFile, 'utf-8'); + } + } + catch { + this.cachedUserId = 'UNKNOWN_USER_ID'; + } + return this.cachedUserId; + } +} +export const productTelemetry = new ProductTelemetry(); diff --git a/dist/telemetry/views.d.ts b/dist/telemetry/views.d.ts new file mode 100644 index 00000000..a9db6cbf --- /dev/null +++ b/dist/telemetry/views.d.ts @@ -0,0 +1,126 @@ +export declare abstract class BaseTelemetryEvent { + abstract name: string; + properties(): Record; +} +type BaseSequence = Array | undefined; +export interface AgentTelemetryPayload { + task: string; + model: string; + model_provider: string; + max_steps: number; + max_actions_per_step: number; + use_vision: boolean | 'auto'; + version: string; + source: string; + cdp_url: string | null; + agent_type: string | null; + action_errors: BaseSequence; + action_history: Array> | null> | undefined; + urls_visited: BaseSequence; + steps: number; + total_input_tokens: number; + total_output_tokens: number; + prompt_cached_tokens: number; + total_tokens: number; + total_duration_seconds: number; + success: boolean | null; + final_result_response: string | null; + error_message: string | null; + judge_verdict?: boolean | null; + judge_reasoning?: string | null; + judge_failure_reason?: string | null; + judge_reached_captcha?: boolean | null; + judge_impossible_task?: boolean | null; +} +export declare class AgentTelemetryEvent extends BaseTelemetryEvent implements AgentTelemetryPayload { + name: string; + task: string; + model: string; + model_provider: string; + max_steps: number; + max_actions_per_step: number; + use_vision: boolean | 'auto'; + version: string; + source: string; + cdp_url: string | null; + agent_type: string | null; + action_errors: BaseSequence; + action_history: Array> | null> | undefined; + urls_visited: BaseSequence; + steps: number; + total_input_tokens: number; + total_output_tokens: number; + prompt_cached_tokens: number; + total_tokens: number; + total_duration_seconds: number; + success: boolean | null; + final_result_response: string | null; + error_message: string | null; + judge_verdict: boolean | null; + judge_reasoning: string | null; + judge_failure_reason: string | null; + judge_reached_captcha: boolean | null; + judge_impossible_task: boolean | null; + constructor(payload: AgentTelemetryPayload); +} +export interface MCPClientTelemetryPayload { + server_name: string; + command: string; + tools_discovered: number; + version: string; + action: string; + tool_name?: string | null; + duration_seconds?: number | null; + error_message?: string | null; +} +export declare class MCPClientTelemetryEvent extends BaseTelemetryEvent implements MCPClientTelemetryPayload { + name: string; + server_name: string; + command: string; + tools_discovered: number; + version: string; + action: string; + tool_name: string | null; + duration_seconds: number | null; + error_message: string | null; + constructor(payload: MCPClientTelemetryPayload); +} +export interface MCPServerTelemetryPayload { + version: string; + action: string; + tool_name?: string | null; + duration_seconds?: number | null; + error_message?: string | null; + parent_process_cmdline?: string | null; +} +export declare class MCPServerTelemetryEvent extends BaseTelemetryEvent implements MCPServerTelemetryPayload { + name: string; + version: string; + action: string; + tool_name: string | null; + duration_seconds: number | null; + error_message: string | null; + parent_process_cmdline: string | null; + constructor(payload: MCPServerTelemetryPayload); +} +export interface CLITelemetryPayload { + version: string; + action: string; + mode: string; + model?: string | null; + model_provider?: string | null; + duration_seconds?: number | null; + error_message?: string | null; +} +export declare class CLITelemetryEvent extends BaseTelemetryEvent implements CLITelemetryPayload { + name: string; + version: string; + action: string; + mode: string; + model: string | null; + model_provider: string | null; + duration_seconds: number | null; + error_message: string | null; + constructor(payload: CLITelemetryPayload); +} +export {}; diff --git a/dist/telemetry/views.js b/dist/telemetry/views.js new file mode 100644 index 00000000..01dde805 --- /dev/null +++ b/dist/telemetry/views.js @@ -0,0 +1,130 @@ +import { is_running_in_docker } from '../config.js'; +export class BaseTelemetryEvent { + properties() { + const entries = Object.entries(this).filter(([key]) => key !== 'name'); + return { + ...Object.fromEntries(entries), + is_docker: is_running_in_docker(), + }; + } +} +export class AgentTelemetryEvent extends BaseTelemetryEvent { + name = 'agent_event'; + task; + model; + model_provider; + max_steps; + max_actions_per_step; + use_vision; + version; + source; + cdp_url; + agent_type; + action_errors; + action_history; + urls_visited; + steps; + total_input_tokens; + total_output_tokens; + prompt_cached_tokens; + total_tokens; + total_duration_seconds; + success; + final_result_response; + error_message; + judge_verdict; + judge_reasoning; + judge_failure_reason; + judge_reached_captcha; + judge_impossible_task; + constructor(payload) { + super(); + this.task = payload.task; + this.model = payload.model; + this.model_provider = payload.model_provider; + this.max_steps = payload.max_steps; + this.max_actions_per_step = payload.max_actions_per_step; + this.use_vision = payload.use_vision; + this.version = payload.version; + this.source = payload.source; + this.cdp_url = payload.cdp_url; + this.agent_type = payload.agent_type; + this.action_errors = payload.action_errors; + this.action_history = payload.action_history; + this.urls_visited = payload.urls_visited; + this.steps = payload.steps; + this.total_input_tokens = payload.total_input_tokens; + this.total_output_tokens = payload.total_output_tokens; + this.prompt_cached_tokens = payload.prompt_cached_tokens; + this.total_tokens = payload.total_tokens; + this.total_duration_seconds = payload.total_duration_seconds; + this.success = payload.success; + this.final_result_response = payload.final_result_response; + this.error_message = payload.error_message; + this.judge_verdict = payload.judge_verdict ?? null; + this.judge_reasoning = payload.judge_reasoning ?? null; + this.judge_failure_reason = payload.judge_failure_reason ?? null; + this.judge_reached_captcha = payload.judge_reached_captcha ?? null; + this.judge_impossible_task = payload.judge_impossible_task ?? null; + } +} +export class MCPClientTelemetryEvent extends BaseTelemetryEvent { + name = 'mcp_client_event'; + server_name; + command; + tools_discovered; + version; + action; + tool_name; + duration_seconds; + error_message; + constructor(payload) { + super(); + this.server_name = payload.server_name; + this.command = payload.command; + this.tools_discovered = payload.tools_discovered; + this.version = payload.version; + this.action = payload.action; + this.tool_name = payload.tool_name ?? null; + this.duration_seconds = payload.duration_seconds ?? null; + this.error_message = payload.error_message ?? null; + } +} +export class MCPServerTelemetryEvent extends BaseTelemetryEvent { + name = 'mcp_server_event'; + version; + action; + tool_name; + duration_seconds; + error_message; + parent_process_cmdline; + constructor(payload) { + super(); + this.version = payload.version; + this.action = payload.action; + this.tool_name = payload.tool_name ?? null; + this.duration_seconds = payload.duration_seconds ?? null; + this.error_message = payload.error_message ?? null; + this.parent_process_cmdline = payload.parent_process_cmdline ?? null; + } +} +export class CLITelemetryEvent extends BaseTelemetryEvent { + name = 'cli_event'; + version; + action; + mode; + model; + model_provider; + duration_seconds; + error_message; + constructor(payload) { + super(); + this.version = payload.version; + this.action = payload.action; + this.mode = payload.mode; + this.model = payload.model ?? null; + this.model_provider = payload.model_provider ?? null; + this.duration_seconds = payload.duration_seconds ?? null; + this.error_message = payload.error_message ?? null; + } +} diff --git a/dist/tokens/custom-pricing.d.ts b/dist/tokens/custom-pricing.d.ts new file mode 100644 index 00000000..9984948a --- /dev/null +++ b/dist/tokens/custom-pricing.d.ts @@ -0,0 +1,2 @@ +import type { ModelPricing } from './views.js'; +export declare const CUSTOM_MODEL_PRICING: Record & Record>; diff --git a/dist/tokens/custom-pricing.js b/dist/tokens/custom-pricing.js new file mode 100644 index 00000000..c501873f --- /dev/null +++ b/dist/tokens/custom-pricing.js @@ -0,0 +1,22 @@ +export const CUSTOM_MODEL_PRICING = { + 'bu-1-0': { + input_cost_per_token: 0.2 / 1_000_000, + output_cost_per_token: 2.0 / 1_000_000, + cache_read_input_token_cost: 0.02 / 1_000_000, + cache_creation_input_token_cost: null, + max_tokens: null, + max_input_tokens: null, + max_output_tokens: null, + }, + 'bu-2-0': { + input_cost_per_token: 0.6 / 1_000_000, + output_cost_per_token: 3.5 / 1_000_000, + cache_read_input_token_cost: 0.06 / 1_000_000, + cache_creation_input_token_cost: null, + max_tokens: null, + max_input_tokens: null, + max_output_tokens: null, + }, +}; +CUSTOM_MODEL_PRICING['bu-latest'] = CUSTOM_MODEL_PRICING['bu-1-0']; +CUSTOM_MODEL_PRICING.smart = CUSTOM_MODEL_PRICING['bu-1-0']; diff --git a/dist/tokens/index.d.ts b/dist/tokens/index.d.ts new file mode 100644 index 00000000..9cf25d66 --- /dev/null +++ b/dist/tokens/index.d.ts @@ -0,0 +1,4 @@ +export * from './service.js'; +export * from './views.js'; +export * from './custom-pricing.js'; +export * from './mappings.js'; diff --git a/dist/tokens/index.js b/dist/tokens/index.js new file mode 100644 index 00000000..9cf25d66 --- /dev/null +++ b/dist/tokens/index.js @@ -0,0 +1,4 @@ +export * from './service.js'; +export * from './views.js'; +export * from './custom-pricing.js'; +export * from './mappings.js'; diff --git a/dist/tokens/mappings.d.ts b/dist/tokens/mappings.d.ts new file mode 100644 index 00000000..bb1b3d42 --- /dev/null +++ b/dist/tokens/mappings.d.ts @@ -0,0 +1 @@ +export declare const MODEL_TO_LITELLM: Record; diff --git a/dist/tokens/mappings.js b/dist/tokens/mappings.js new file mode 100644 index 00000000..e5121141 --- /dev/null +++ b/dist/tokens/mappings.js @@ -0,0 +1,3 @@ +export const MODEL_TO_LITELLM = { + 'gemini-flash-latest': 'gemini/gemini-flash-latest', +}; diff --git a/dist/tokens/service.d.ts b/dist/tokens/service.d.ts new file mode 100644 index 00000000..7b33a9d6 --- /dev/null +++ b/dist/tokens/service.d.ts @@ -0,0 +1,35 @@ +import type { BaseChatModel } from '../llm/base.js'; +import type { ChatInvokeUsage } from '../llm/views.js'; +import { ModelPricing, ModelUsageStats, ModelUsageTokens, TokenCostCalculated, TokenUsageEntry, UsageSummary } from './views.js'; +export declare class TokenCost { + private includeCost; + private usageHistory; + private registeredLlms; + private originalAinvoke; + private pricingData; + private initialized; + private cacheDir; + constructor(includeCost?: boolean); + initialize(): Promise; + private loadPricingData; + private findValidCache; + private isCacheValid; + private loadFromCache; + private fetchAndCachePricing; + addUsage(model: string, usage: ChatInvokeUsage): TokenUsageEntry; + register_llm(llm: BaseChatModel): BaseChatModel; + private logUsage; + private buildInputDisplay; + private formatTokens; + calculateCost(model: string, usage: ChatInvokeUsage): Promise; + getModelPricing(modelName: string): Promise; + get_usage_tokens_for_model(model: string): ModelUsageTokens; + get_usage_summary(model?: string, since?: Date): Promise; + log_usage_summary(): Promise; + get_cost_by_model: () => Promise>; + clear_history(): void; + refresh_pricing_data(): Promise; + clean_old_caches(keepCount?: number): Promise; + ensurePricingLoaded(): Promise; + estimateTokens(text: string): number; +} diff --git a/dist/tokens/service.js b/dist/tokens/service.js new file mode 100644 index 00000000..a11a53d2 --- /dev/null +++ b/dist/tokens/service.js @@ -0,0 +1,441 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { encode } from 'gpt-tokenizer'; +import axios from 'axios'; +import { CONFIG } from '../config.js'; +import { createLogger } from '../logging-config.js'; +import { create_task_with_error_handling } from '../utils.js'; +import { CUSTOM_MODEL_PRICING } from './custom-pricing.js'; +import { MODEL_TO_LITELLM } from './mappings.js'; +const logger = createLogger('browser_use.tokens'); +const costLogger = createLogger('browser_use.tokens.cost'); +const PRICING_URL = 'https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json'; +const CACHE_DURATION_MS = 24 * 60 * 60 * 1000; // 1 day +const CACHE_DIR_NAME = 'browser_use/token_cost'; +const ansi = { + cyan: '\u001b[96m', + yellow: '\u001b[93m', + green: '\u001b[92m', + blue: '\u001b[94m', + magenta: '\u001b[95m', + reset: '\u001b[0m', + bold: '\u001b[1m', +}; +const xdgCacheHome = () => { + const configured = CONFIG.XDG_CACHE_HOME; + if (configured && path.isAbsolute(configured)) { + return configured; + } + return path.join(process.env.HOME ?? process.cwd(), '.cache'); +}; +const ensureDir = async (dir) => { + await fs.promises.mkdir(dir, { recursive: true }); +}; +const parsePricingTimestamp = (data) => new Date(data.timestamp); +const usagePromptCost = (cost) => { + if (!cost) + return 0; + return (cost.new_prompt_cost + + (cost.prompt_read_cached_cost ?? 0) + + (cost.prompt_cache_creation_cost ?? 0)); +}; +export class TokenCost { + includeCost; + usageHistory = []; + registeredLlms = new WeakSet(); + originalAinvoke = new WeakMap(); + pricingData = null; + initialized = false; + cacheDir; + constructor(includeCost) { + const envFlag = process.env.BROWSER_USE_CALCULATE_COST?.toLowerCase() === 'true'; + this.includeCost = includeCost ?? envFlag ?? false; + this.cacheDir = path.join(xdgCacheHome(), CACHE_DIR_NAME); + } + async initialize() { + if (this.initialized) + return; + if (this.includeCost) { + await this.loadPricingData(); + } + this.initialized = true; + } + async loadPricingData() { + try { + const cacheFile = await this.findValidCache(); + if (cacheFile) { + await this.loadFromCache(cacheFile); + return; + } + } + catch (error) { + logger.debug(`Failed to load token pricing cache: ${error.message}`); + } + await this.fetchAndCachePricing(); + } + async findValidCache() { + try { + await ensureDir(this.cacheDir); + const files = await fs.promises.readdir(this.cacheDir); + const jsonFiles = files.filter((file) => file.endsWith('.json')); + if (!jsonFiles.length) { + return null; + } + const sorted = jsonFiles + .map((file) => path.join(this.cacheDir, file)) + .sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs); + for (const file of sorted) { + const isValid = await this.isCacheValid(file); + if (isValid) { + return file; + } + await fs.promises.unlink(file).catch(() => undefined); + } + return null; + } + catch { + return null; + } + } + async isCacheValid(file) { + try { + const content = await fs.promises.readFile(file, 'utf-8'); + const parsed = JSON.parse(content); + const timestamp = parsePricingTimestamp(parsed); + return Date.now() - timestamp.getTime() < CACHE_DURATION_MS; + } + catch { + return false; + } + } + async loadFromCache(file) { + const content = await fs.promises.readFile(file, 'utf-8'); + const cached = JSON.parse(content); + this.pricingData = cached.data; + } + async fetchAndCachePricing() { + try { + const response = await axios.get(PRICING_URL, { + timeout: 30_000, + }); + this.pricingData = response.data; + const cached = { + timestamp: new Date(), + data: response.data, + }; + await ensureDir(this.cacheDir); + const fileName = `pricing_${new Date().toISOString().replace(/[:.]/g, '-')}.json`; + await fs.promises.writeFile(path.join(this.cacheDir, fileName), JSON.stringify(cached, null, 2)); + } + catch (error) { + logger.debug(`Failed to fetch LiteLLM pricing: ${error.message}`); + this.pricingData = this.pricingData ?? {}; + } + } + addUsage(model, usage) { + const entry = { + model, + timestamp: new Date(), + usage, + }; + this.usageHistory.push(entry); + return entry; + } + register_llm(llm) { + if (this.registeredLlms.has(llm)) { + return llm; + } + this.registeredLlms.add(llm); + const original = llm.ainvoke.bind(llm); + this.originalAinvoke.set(llm, original); + llm.ainvoke = (async (...args) => { + // @ts-ignore - Spread operator type compatibility + const result = await original(...args); + if (result?.usage) { + const usageEntry = this.addUsage(llm.model, result.usage); + create_task_with_error_handling(this.logUsage(llm.model, usageEntry), { + name: 'log_token_usage', + suppress_exceptions: true, + }); + } + return result; + }); + return llm; + } + async logUsage(model, entry) { + if (!this.includeCost) { + return; + } + await this.ensurePricingLoaded(); + const cost = await this.calculateCost(model, entry.usage); + const inputPart = this.buildInputDisplay(entry.usage, cost); + const completionTokensFmt = this.formatTokens(entry.usage.completion_tokens); + const completionSection = this.includeCost && cost && cost.completion_cost > 0 + ? `📤 ${ansi.green}${completionTokensFmt} ($${cost.completion_cost.toFixed(4)})${ansi.reset}` + : `📤 ${ansi.green}${completionTokensFmt}${ansi.reset}`; + costLogger.debug(`🧠 ${ansi.cyan}${model}${ansi.reset} | ${inputPart} | ${completionSection}`); + } + buildInputDisplay(usage, cost) { + const parts = []; + const cached = usage.prompt_cached_tokens ?? 0; + const cacheCreation = usage.prompt_cache_creation_tokens ?? 0; + const uncached = usage.prompt_tokens - cached; + if (cached || cacheCreation) { + if (uncached > 0) { + const formatted = this.formatTokens(uncached); + const costPart = this.includeCost && cost && cost.new_prompt_cost > 0 + ? ` ($${cost.new_prompt_cost.toFixed(4)})` + : ''; + parts.push(`🆕 ${ansi.yellow}${formatted}${costPart}${ansi.reset}`); + } + if (cached) { + const formatted = this.formatTokens(cached); + const cacheCost = this.includeCost && cost?.prompt_read_cached_cost + ? ` ($${cost.prompt_read_cached_cost.toFixed(4)})` + : ''; + parts.push(`💾 ${ansi.blue}${formatted}${cacheCost}${ansi.reset}`); + } + if (cacheCreation) { + const formatted = this.formatTokens(cacheCreation); + const creationCost = this.includeCost && cost?.prompt_cache_creation_cost + ? ` ($${cost.prompt_cache_creation_cost.toFixed(4)})` + : ''; + parts.push(`🔧 ${ansi.blue}${formatted}${creationCost}${ansi.reset}`); + } + } + else { + const formatted = this.formatTokens(usage.prompt_tokens); + const costPart = this.includeCost && cost && cost.new_prompt_cost > 0 + ? ` ($${cost.new_prompt_cost.toFixed(4)})` + : ''; + parts.push(`📥 ${ansi.yellow}${formatted}${costPart}${ansi.reset}`); + } + return parts.join(' + '); + } + formatTokens(tokens) { + if (tokens >= 1_000_000_000) + return `${(tokens / 1_000_000_000).toFixed(1)}B`; + if (tokens >= 1_000_000) + return `${(tokens / 1_000_000).toFixed(1)}M`; + if (tokens >= 1_000) + return `${(tokens / 1_000).toFixed(1)}k`; + return `${tokens}`; + } + async calculateCost(model, usage) { + if (!this.includeCost) { + return null; + } + await this.ensurePricingLoaded(); + const pricing = await this.getModelPricing(model); + if (!pricing) { + return null; + } + const cached = usage.prompt_cached_tokens ?? 0; + const uncachedPrompt = usage.prompt_tokens - cached; + return { + new_prompt_tokens: usage.prompt_tokens, + new_prompt_cost: uncachedPrompt * (pricing.input_cost_per_token ?? 0), + prompt_read_cached_tokens: cached || null, + prompt_read_cached_cost: cached + ? cached * (pricing.cache_read_input_token_cost ?? 0) + : null, + prompt_cached_creation_tokens: usage.prompt_cache_creation_tokens ?? null, + prompt_cache_creation_cost: usage.prompt_cache_creation_tokens && + pricing.cache_creation_input_token_cost + ? usage.prompt_cache_creation_tokens * + pricing.cache_creation_input_token_cost + : null, + completion_tokens: usage.completion_tokens, + completion_cost: usage.completion_tokens * (pricing.output_cost_per_token ?? 0), + }; + } + async getModelPricing(modelName) { + const customPricing = CUSTOM_MODEL_PRICING[modelName]; + if (customPricing) { + return { + model: modelName, + input_cost_per_token: customPricing.input_cost_per_token ?? null, + output_cost_per_token: customPricing.output_cost_per_token ?? null, + cache_read_input_token_cost: customPricing.cache_read_input_token_cost ?? null, + cache_creation_input_token_cost: customPricing.cache_creation_input_token_cost ?? null, + max_tokens: customPricing.max_tokens ?? null, + max_input_tokens: customPricing.max_input_tokens ?? null, + max_output_tokens: customPricing.max_output_tokens ?? null, + }; + } + await this.ensurePricingLoaded(); + if (!this.pricingData) { + return null; + } + const litellmModelName = MODEL_TO_LITELLM[modelName] ?? modelName; + const pricing = this.pricingData[litellmModelName]; + if (!pricing) { + return null; + } + return { + model: modelName, + input_cost_per_token: pricing.input_cost_per_token ?? null, + output_cost_per_token: pricing.output_cost_per_token ?? null, + cache_read_input_token_cost: pricing.cache_read_input_token_cost ?? null, + cache_creation_input_token_cost: pricing.cache_creation_input_token_cost ?? null, + max_tokens: pricing.max_tokens ?? null, + max_input_tokens: pricing.max_input_tokens ?? null, + max_output_tokens: pricing.max_output_tokens ?? null, + }; + } + get_usage_tokens_for_model(model) { + const filtered = this.usageHistory.filter((entry) => entry.model === model); + return { + model, + prompt_tokens: filtered.reduce((sum, entry) => sum + entry.usage.prompt_tokens, 0), + prompt_cached_tokens: filtered.reduce((sum, entry) => sum + (entry.usage.prompt_cached_tokens ?? 0), 0), + completion_tokens: filtered.reduce((sum, entry) => sum + entry.usage.completion_tokens, 0), + total_tokens: filtered.reduce((sum, entry) => sum + entry.usage.prompt_tokens + entry.usage.completion_tokens, 0), + }; + } + async get_usage_summary(model, since) { + let entries = this.usageHistory; + if (model) { + entries = entries.filter((entry) => entry.model === model); + } + if (since) { + entries = entries.filter((entry) => entry.timestamp >= since); + } + if (!entries.length) { + return { + total_prompt_tokens: 0, + total_prompt_cost: 0, + total_prompt_cached_tokens: 0, + total_prompt_cached_cost: 0, + total_completion_tokens: 0, + total_completion_cost: 0, + total_tokens: 0, + total_cost: 0, + entry_count: 0, + by_model: {}, + }; + } + const byModel = {}; + let totalPrompt = 0; + let totalCompletion = 0; + let totalPromptCached = 0; + let totalPromptCachedCost = 0; + let totalPromptCost = 0; + let totalCompletionCost = 0; + for (const entry of entries) { + const stats = (byModel[entry.model] ||= { + model: entry.model, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + cost: 0, + invocations: 0, + average_tokens_per_invocation: 0, + }); + stats.prompt_tokens += entry.usage.prompt_tokens; + stats.completion_tokens += entry.usage.completion_tokens; + const totalEntryTokens = entry.usage.prompt_tokens + entry.usage.completion_tokens; + stats.total_tokens += totalEntryTokens; + stats.invocations += 1; + totalPrompt += entry.usage.prompt_tokens; + totalCompletion += entry.usage.completion_tokens; + totalPromptCached += entry.usage.prompt_cached_tokens ?? 0; + if (this.includeCost) { + const cost = await this.calculateCost(entry.model, entry.usage); + const promptCost = usagePromptCost(cost); + const completionCost = cost?.completion_cost ?? 0; + const cachedCost = cost?.prompt_read_cached_cost ?? 0; + stats.cost += promptCost + completionCost; + totalPromptCost += promptCost; + totalCompletionCost += completionCost; + totalPromptCachedCost += cachedCost; + } + } + Object.values(byModel).forEach((stats) => { + stats.average_tokens_per_invocation = stats.invocations + ? stats.total_tokens / stats.invocations + : 0; + }); + return { + total_prompt_tokens: totalPrompt, + total_prompt_cost: totalPromptCost, + total_prompt_cached_tokens: totalPromptCached, + total_prompt_cached_cost: totalPromptCachedCost, + total_completion_tokens: totalCompletion, + total_completion_cost: totalCompletionCost, + total_tokens: totalPrompt + totalCompletion, + total_cost: totalPromptCost + totalCompletionCost, + entry_count: entries.length, + by_model: byModel, + }; + } + async log_usage_summary() { + if (!this.usageHistory.length) + return; + const summary = await this.get_usage_summary(); + if (!summary.entry_count) + return; + const totalTokens = this.formatTokens(summary.total_tokens); + const totalCostPart = this.includeCost && summary.total_cost > 0 + ? ` ($${summary.total_cost.toFixed(4)})` + : ''; + const promptTokens = this.formatTokens(summary.total_prompt_tokens); + const promptCostPart = this.includeCost && summary.total_prompt_cost > 0 + ? ` ($${summary.total_prompt_cost.toFixed(4)})` + : ''; + const completionTokens = this.formatTokens(summary.total_completion_tokens); + const completionCostPart = this.includeCost && summary.total_completion_cost > 0 + ? ` ($${summary.total_completion_cost.toFixed(4)})` + : ''; + costLogger.debug(`💲 ${ansi.bold}Total Usage Summary${ansi.reset}: ${ansi.blue}${totalTokens} tokens${ansi.reset}${totalCostPart} | ` + + `⬅️ ${ansi.yellow}${promptTokens}${promptCostPart}${ansi.reset} | ➡️ ${ansi.green}${completionTokens}${completionCostPart}${ansi.reset}`); + for (const [model, stats] of Object.entries(summary.by_model)) { + const totalFmt = this.formatTokens(stats.total_tokens); + const promptFmt = this.formatTokens(stats.prompt_tokens); + const completionFmt = this.formatTokens(stats.completion_tokens); + const avgFmt = this.formatTokens(Math.round(stats.average_tokens_per_invocation)); + const costPart = this.includeCost && stats.cost > 0 + ? ` ($${stats.cost.toFixed(4)})` + : ''; + costLogger.debug(` 🤖 ${ansi.cyan}${model}${ansi.reset}: ${ansi.blue}${totalFmt} tokens${ansi.reset}${costPart} | ` + + `⬅️ ${ansi.yellow}${promptFmt}${ansi.reset} | ➡️ ${ansi.green}${completionFmt}${ansi.reset} | ` + + `📞 ${stats.invocations} calls | 📈 ${avgFmt}/call`); + } + } + get_cost_by_model = async () => { + const summary = await this.get_usage_summary(); + return summary.by_model; + }; + clear_history() { + this.usageHistory = []; + } + async refresh_pricing_data() { + if (!this.includeCost) + return; + await this.fetchAndCachePricing(); + } + async clean_old_caches(keepCount = 3) { + try { + const files = await fs.promises.readdir(this.cacheDir); + const jsonFiles = files.filter((file) => file.endsWith('.json')); + if (jsonFiles.length <= keepCount) + return; + const sorted = jsonFiles + .map((file) => path.join(this.cacheDir, file)) + .sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs); + const toDelete = sorted.slice(0, sorted.length - keepCount); + await Promise.all(toDelete.map((file) => fs.promises.unlink(file).catch(() => undefined))); + } + catch (error) { + logger.debug(`Failed to clean token cache: ${error.message}`); + } + } + async ensurePricingLoaded() { + if (!this.includeCost || this.pricingData) + return; + await this.loadPricingData(); + } + estimateTokens(text) { + return encode(text).length; + } +} diff --git a/dist/tokens/views.d.ts b/dist/tokens/views.d.ts new file mode 100644 index 00000000..29ea5e24 --- /dev/null +++ b/dist/tokens/views.d.ts @@ -0,0 +1,58 @@ +import type { ChatInvokeUsage } from '../llm/views.js'; +export interface TokenUsageEntry { + model: string; + timestamp: Date; + usage: ChatInvokeUsage; +} +export interface TokenCostCalculated { + new_prompt_tokens: number; + new_prompt_cost: number; + prompt_read_cached_tokens: number | null; + prompt_read_cached_cost: number | null; + prompt_cached_creation_tokens: number | null; + prompt_cache_creation_cost: number | null; + completion_tokens: number; + completion_cost: number; +} +export interface ModelPricing { + model: string; + input_cost_per_token: number | null; + output_cost_per_token: number | null; + cache_read_input_token_cost: number | null; + cache_creation_input_token_cost: number | null; + max_tokens: number | null; + max_input_tokens: number | null; + max_output_tokens: number | null; +} +export interface CachedPricingData { + timestamp: Date; + data: Record; +} +export interface ModelUsageStats { + model: string; + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + cost: number; + invocations: number; + average_tokens_per_invocation: number; +} +export interface ModelUsageTokens { + model: string; + prompt_tokens: number; + prompt_cached_tokens: number; + completion_tokens: number; + total_tokens: number; +} +export interface UsageSummary { + total_prompt_tokens: number; + total_prompt_cost: number; + total_prompt_cached_tokens: number; + total_prompt_cached_cost: number; + total_completion_tokens: number; + total_completion_cost: number; + total_tokens: number; + total_cost: number; + entry_count: number; + by_model: Record; +} diff --git a/dist/tokens/views.js b/dist/tokens/views.js new file mode 100644 index 00000000..cb0ff5c3 --- /dev/null +++ b/dist/tokens/views.js @@ -0,0 +1 @@ +export {}; diff --git a/dist/tools/extraction/index.d.ts b/dist/tools/extraction/index.d.ts new file mode 100644 index 00000000..7d9070ac --- /dev/null +++ b/dist/tools/extraction/index.d.ts @@ -0,0 +1,2 @@ +export * from './schema-utils.js'; +export * from './views.js'; diff --git a/dist/tools/extraction/index.js b/dist/tools/extraction/index.js new file mode 100644 index 00000000..7d9070ac --- /dev/null +++ b/dist/tools/extraction/index.js @@ -0,0 +1,2 @@ +export * from './schema-utils.js'; +export * from './views.js'; diff --git a/dist/tools/extraction/schema-utils.d.ts b/dist/tools/extraction/schema-utils.d.ts new file mode 100644 index 00000000..36f105bd --- /dev/null +++ b/dist/tools/extraction/schema-utils.d.ts @@ -0,0 +1,6 @@ +import { z } from 'zod'; +export declare const findUnsupportedJsonSchemaKeyword: (schema: unknown) => string | null; +export declare const schemaDictToZodSchema: (schema: unknown) => z.ZodTypeAny; +export declare const resolveDefaultForSchema: (schema: unknown) => unknown; +export declare const normalizeStructuredDataBySchema: (value: unknown, schema: unknown) => unknown; +export declare const schema_dict_to_zod_schema: (schema: unknown) => z.ZodTypeAny; diff --git a/dist/tools/extraction/schema-utils.js b/dist/tools/extraction/schema-utils.js new file mode 100644 index 00000000..28905c1c --- /dev/null +++ b/dist/tools/extraction/schema-utils.js @@ -0,0 +1,237 @@ +import { z } from 'zod'; +const UNSUPPORTED_KEYWORDS = new Set([ + '$ref', + 'allOf', + 'anyOf', + 'oneOf', + 'not', + '$defs', + 'definitions', + 'if', + 'then', + 'else', + 'dependentSchemas', + 'dependentRequired', +]); +const PRIMITIVE_DEFAULTS = { + string: '', + number: 0.0, + integer: 0, + boolean: false, +}; +const isRecord = (value) => !!value && typeof value === 'object' && !Array.isArray(value); +const getTypeList = (schema) => { + const schemaType = schema.type; + if (Array.isArray(schemaType)) { + return schemaType.map((item) => String(item).toLowerCase()); + } + if (typeof schemaType === 'string' && schemaType.trim().length > 0) { + return [schemaType.toLowerCase()]; + } + return ['string']; +}; +export const findUnsupportedJsonSchemaKeyword = (schema) => { + if (Array.isArray(schema)) { + for (const item of schema) { + const found = findUnsupportedJsonSchemaKeyword(item); + if (found) { + return found; + } + } + return null; + } + if (!isRecord(schema)) { + return null; + } + for (const [key, value] of Object.entries(schema)) { + if (UNSUPPORTED_KEYWORDS.has(key)) { + return key; + } + const found = findUnsupportedJsonSchemaKeyword(value); + if (found) { + return found; + } + } + return null; +}; +const checkUnsupported = (schema) => { + const unsupported = findUnsupportedJsonSchemaKeyword(schema); + if (unsupported) { + throw new Error(`Unsupported JSON Schema keyword: ${unsupported}`); + } +}; +const resolveType = (schema, name) => { + checkUnsupported(schema); + const typeList = getTypeList(schema); + if (Array.isArray(schema.enum)) { + return z.string(); + } + if (typeList.includes('object')) { + const properties = isRecord(schema.properties) + ? schema.properties + : null; + const base = properties + ? buildObjectSchema(schema, name) + : z.record(z.string(), z.any()); + return typeList.includes('null') || schema.nullable === true + ? base.nullable() + : base; + } + if (typeList.includes('array')) { + const items = isRecord(schema.items) + ? resolveType(schema.items, `${name}_item`) + : z.any(); + const arraySchema = z.array(items); + return typeList.includes('null') || schema.nullable === true + ? arraySchema.nullable() + : arraySchema; + } + let primitive = z.string(); + if (typeList.includes('integer')) { + primitive = z.number().int(); + } + else if (typeList.includes('number')) { + primitive = z.number(); + } + else if (typeList.includes('boolean')) { + primitive = z.boolean(); + } + else if (typeList.includes('null')) { + primitive = z.null(); + } + else if (typeList.includes('string')) { + primitive = z.string(); + } + if (schema.nullable === true && !typeList.includes('null')) { + return primitive.nullable(); + } + return primitive; +}; +const applyOptionalDefaults = (propertySchema, propertyType) => { + if (Object.prototype.hasOwnProperty.call(propertySchema, 'default')) { + return propertyType.default(propertySchema.default); + } + const typeList = getTypeList(propertySchema); + const allowsNull = propertySchema.nullable === true || typeList.includes('null'); + if (allowsNull) { + return propertyType.nullable().default(null); + } + if (Array.isArray(propertySchema.enum)) { + return propertyType.nullable().default(null); + } + if (typeList.includes('array')) { + return propertyType.default([]); + } + const primitiveType = typeList.find((item) => item in PRIMITIVE_DEFAULTS); + if (primitiveType) { + return propertyType.default(PRIMITIVE_DEFAULTS[primitiveType]); + } + return propertyType.nullable().default(null); +}; +const buildObjectSchema = (schema, name) => { + checkUnsupported(schema); + const properties = isRecord(schema.properties) + ? schema.properties + : {}; + const required = new Set(Array.isArray(schema.required) + ? schema.required + .map((item) => String(item)) + .filter((item) => item.length > 0) + : []); + const shape = {}; + for (const [propertyName, propertySchema] of Object.entries(properties)) { + if (!isRecord(propertySchema)) { + shape[propertyName] = z.any(); + continue; + } + const propertyType = resolveType(propertySchema, `${name}_${propertyName}`); + if (required.has(propertyName)) { + shape[propertyName] = propertyType; + continue; + } + shape[propertyName] = applyOptionalDefaults(propertySchema, propertyType); + } + return z.object(shape).strict(); +}; +export const schemaDictToZodSchema = (schema) => { + if (!isRecord(schema)) { + throw new Error('Top-level schema must be an object'); + } + checkUnsupported(schema); + const typeList = getTypeList(schema); + if (!typeList.includes('object')) { + throw new Error('Top-level schema must have type "object"'); + } + if (!isRecord(schema.properties) || !Object.keys(schema.properties).length) { + throw new Error('Top-level schema must have at least one property'); + } + const modelName = typeof schema.title === 'string' && schema.title.trim().length > 0 + ? schema.title + : 'DynamicExtractionModel'; + return buildObjectSchema(schema, modelName); +}; +export const resolveDefaultForSchema = (schema) => { + if (!isRecord(schema)) { + return null; + } + if (Object.prototype.hasOwnProperty.call(schema, 'default')) { + return schema.default; + } + const typeList = getTypeList(schema); + const allowsNull = schema.nullable === true || typeList.includes('null'); + if (allowsNull) { + return null; + } + if (Array.isArray(schema.enum)) { + return null; + } + if (typeList.includes('array')) { + return []; + } + const primitiveType = typeList.find((item) => item in PRIMITIVE_DEFAULTS); + if (primitiveType) { + return PRIMITIVE_DEFAULTS[primitiveType]; + } + return null; +}; +export const normalizeStructuredDataBySchema = (value, schema) => { + if (!isRecord(schema)) { + return value; + } + const typeList = getTypeList(schema); + if (typeList.includes('object')) { + const properties = isRecord(schema.properties) + ? schema.properties + : {}; + const required = new Set(Array.isArray(schema.required) + ? schema.required + .map((item) => String(item)) + .filter((item) => item.length > 0) + : []); + const source = isRecord(value) ? value : {}; + const normalized = {}; + for (const [propertyName, propertySchema] of Object.entries(properties)) { + if (Object.prototype.hasOwnProperty.call(source, propertyName)) { + normalized[propertyName] = normalizeStructuredDataBySchema(source[propertyName], propertySchema); + continue; + } + if (required.has(propertyName)) { + continue; + } + normalized[propertyName] = resolveDefaultForSchema(propertySchema); + } + for (const [propertyName, propertyValue] of Object.entries(source)) { + if (!Object.prototype.hasOwnProperty.call(normalized, propertyName)) { + normalized[propertyName] = propertyValue; + } + } + return normalized; + } + if (typeList.includes('array') && + Array.isArray(value) && + isRecord(schema.items)) { + return value.map((item) => normalizeStructuredDataBySchema(item, schema.items)); + } + return value; +}; +export const schema_dict_to_zod_schema = schemaDictToZodSchema; diff --git a/dist/tools/extraction/views.d.ts b/dist/tools/extraction/views.d.ts new file mode 100644 index 00000000..50afde0f --- /dev/null +++ b/dist/tools/extraction/views.d.ts @@ -0,0 +1,7 @@ +export interface ExtractionResult { + data: Record; + schema_used: Record; + is_partial?: boolean; + source_url?: string | null; + content_stats?: Record; +} diff --git a/dist/tools/extraction/views.js b/dist/tools/extraction/views.js new file mode 100644 index 00000000..cb0ff5c3 --- /dev/null +++ b/dist/tools/extraction/views.js @@ -0,0 +1 @@ +export {}; diff --git a/dist/tools/index.d.ts b/dist/tools/index.d.ts new file mode 100644 index 00000000..ae9468ec --- /dev/null +++ b/dist/tools/index.d.ts @@ -0,0 +1,5 @@ +export * from './service.js'; +export * from './views.js'; +export * from './registry/index.js'; +export * from './extraction/index.js'; +export * from './utils.js'; diff --git a/dist/tools/index.js b/dist/tools/index.js new file mode 100644 index 00000000..ae9468ec --- /dev/null +++ b/dist/tools/index.js @@ -0,0 +1,5 @@ +export * from './service.js'; +export * from './views.js'; +export * from './registry/index.js'; +export * from './extraction/index.js'; +export * from './utils.js'; diff --git a/dist/tools/registry/index.d.ts b/dist/tools/registry/index.d.ts new file mode 100644 index 00000000..c6dd1f8b --- /dev/null +++ b/dist/tools/registry/index.d.ts @@ -0,0 +1,2 @@ +export * from './service.js'; +export * from './views.js'; diff --git a/dist/tools/registry/index.js b/dist/tools/registry/index.js new file mode 100644 index 00000000..c6dd1f8b --- /dev/null +++ b/dist/tools/registry/index.js @@ -0,0 +1,2 @@ +export * from './service.js'; +export * from './views.js'; diff --git a/dist/tools/registry/service.d.ts b/dist/tools/registry/service.d.ts new file mode 100644 index 00000000..ff4f95dc --- /dev/null +++ b/dist/tools/registry/service.d.ts @@ -0,0 +1 @@ +export * from '../../controller/registry/service.js'; diff --git a/dist/tools/registry/service.js b/dist/tools/registry/service.js new file mode 100644 index 00000000..ff4f95dc --- /dev/null +++ b/dist/tools/registry/service.js @@ -0,0 +1 @@ +export * from '../../controller/registry/service.js'; diff --git a/dist/tools/registry/views.d.ts b/dist/tools/registry/views.d.ts new file mode 100644 index 00000000..0b9d20f1 --- /dev/null +++ b/dist/tools/registry/views.d.ts @@ -0,0 +1 @@ +export * from '../../controller/registry/views.js'; diff --git a/dist/tools/registry/views.js b/dist/tools/registry/views.js new file mode 100644 index 00000000..0b9d20f1 --- /dev/null +++ b/dist/tools/registry/views.js @@ -0,0 +1 @@ +export * from '../../controller/registry/views.js'; diff --git a/dist/tools/service.d.ts b/dist/tools/service.d.ts new file mode 100644 index 00000000..d310ea73 --- /dev/null +++ b/dist/tools/service.d.ts @@ -0,0 +1,2 @@ +export { Controller as Tools } from '../controller/service.js'; +export type { ControllerOptions as ToolsOptions, ActParams as ToolsActParams, } from '../controller/service.js'; diff --git a/dist/tools/service.js b/dist/tools/service.js new file mode 100644 index 00000000..29017bb5 --- /dev/null +++ b/dist/tools/service.js @@ -0,0 +1 @@ +export { Controller as Tools } from '../controller/service.js'; diff --git a/dist/tools/utils.d.ts b/dist/tools/utils.d.ts new file mode 100644 index 00000000..b331f1fa --- /dev/null +++ b/dist/tools/utils.d.ts @@ -0,0 +1,2 @@ +import { DOMElementNode } from '../dom/views.js'; +export declare const getClickDescription: (node: DOMElementNode) => string; diff --git a/dist/tools/utils.js b/dist/tools/utils.js new file mode 100644 index 00000000..f18221af --- /dev/null +++ b/dist/tools/utils.js @@ -0,0 +1,57 @@ +import { DOMElementNode } from '../dom/views.js'; +const normalizeBoolLike = (value) => { + const normalized = String(value ?? '') + .trim() + .toLowerCase(); + return normalized === 'true' || normalized === 'checked' || normalized === ''; +}; +const summarizeText = (text, maxLength = 30) => { + const compact = text.replace(/\s+/g, ' ').trim(); + if (!compact) { + return ''; + } + return compact.length > maxLength + ? `${compact.slice(0, maxLength)}...` + : compact; +}; +export const getClickDescription = (node) => { + const parts = [node.tag_name]; + const inputType = node.attributes.type; + if (node.tag_name === 'input' && inputType) { + parts.push(`type=${inputType}`); + if (inputType === 'checkbox') { + const isChecked = normalizeBoolLike(node.attributes.checked); + parts.push(`checkbox-state=${isChecked ? 'checked' : 'unchecked'}`); + } + } + const role = node.attributes.role; + if (role) { + parts.push(`role=${role}`); + if (role === 'checkbox') { + const isChecked = normalizeBoolLike(node.attributes['aria-checked']); + parts.push(`checkbox-state=${isChecked ? 'checked' : 'unchecked'}`); + } + } + if (['label', 'span', 'div'].includes(node.tag_name) && + !parts.some((part) => part.startsWith('type='))) { + const hiddenCheckboxChild = node.children.find((child) => child instanceof DOMElementNode && + child.tag_name === 'input' && + child.attributes.type === 'checkbox' && + !child.is_visible); + if (hiddenCheckboxChild) { + const isChecked = normalizeBoolLike(hiddenCheckboxChild.attributes.checked); + parts.push(`checkbox-state=${isChecked ? 'checked' : 'unchecked'}`); + } + } + const text = summarizeText(node.get_all_text_till_next_clickable_element()); + if (text) { + parts.push(`"${text}"`); + } + for (const attribute of ['id', 'name', 'aria-label']) { + const value = node.attributes[attribute]; + if (value && value.trim().length > 0) { + parts.push(`${attribute}=${value.slice(0, 20)}`); + } + } + return parts.join(' '); +}; diff --git a/dist/tools/views.d.ts b/dist/tools/views.d.ts new file mode 100644 index 00000000..d539625b --- /dev/null +++ b/dist/tools/views.d.ts @@ -0,0 +1 @@ +export * from '../controller/views.js'; diff --git a/dist/tools/views.js b/dist/tools/views.js new file mode 100644 index 00000000..d539625b --- /dev/null +++ b/dist/tools/views.js @@ -0,0 +1 @@ +export * from '../controller/views.js'; diff --git a/dist/utils.d.ts b/dist/utils.d.ts new file mode 100644 index 00000000..ec8b824c --- /dev/null +++ b/dist/utils.d.ts @@ -0,0 +1,137 @@ +import { createLogger } from './logging-config.js'; +type Callback = (() => void) | undefined; +export interface SignalHandlerOptions { + pause_callback?: Callback; + resume_callback?: Callback; + custom_exit_callback?: Callback; + exit_on_second_int?: boolean; + interruptible_task_patterns?: string[]; +} +export declare class SignalHandler { + loop: NodeJS.EventEmitter | null; + pause_callback?: Callback; + resume_callback?: Callback; + custom_exit_callback?: Callback; + exit_on_second_int: boolean; + interruptible_task_patterns: string[]; + is_windows: boolean; + private ctrl_c_pressed; + private waiting_for_input; + private bound_sigint; + private bound_sigterm; + constructor(options?: SignalHandlerOptions); + register(): void; + unregister(): void; + private _handle_second_ctrl_c; + private _cancel_interruptible_tasks; + wait_for_resume(): Promise; + reset(): void; + private sigint_handler; + private sigterm_handler; +} +export declare const time_execution_sync: (additional_text?: string) => any>(func: T) => T; +export declare const time_execution_async: (additional_text?: string) => Promise>(func: T) => T; +export declare const singleton: any>(cls: T) => (...args: Parameters) => ReturnType; +export declare const check_env_variables: (keys: string[], predicate?: (values: string[]) => boolean) => boolean; +export declare const is_unsafe_pattern: (pattern: string) => boolean; +export declare const merge_dicts: (a: Record, b: Record, path?: (string | number)[]) => Record; +export declare const get_browser_use_version: () => string; +export declare const check_latest_browser_use_version: () => Promise; +export interface CreateTaskWithErrorHandlingOptions { + name?: string; + logger_instance?: ReturnType; + suppress_exceptions?: boolean; +} +export declare const create_task_with_error_handling: (promise: Promise, options?: CreateTaskWithErrorHandlingOptions) => Promise; +export declare const sanitize_surrogates: (text: string) => string; +export declare const get_git_info: () => Record | null; +export declare const _log_pretty_path: (input: unknown) => string; +export declare const _log_pretty_url: (value: string, max_len?: number | null) => string; +export declare const log_pretty_path: (input: unknown) => string; +export declare const log_pretty_url: (value: string, max_len?: number | null) => string; +export declare const uuid7str: () => string; +/** + * Retry configuration options + */ +export interface RetryOptions { + /** Maximum number of retry attempts (default: 3) */ + maxAttempts?: number; + /** Delay between retries in milliseconds (default: 1000) */ + delayMs?: number; + /** Exponential backoff multiplier (default: 1 = no backoff) */ + backoffMultiplier?: number; + /** Maximum delay in milliseconds for exponential backoff (default: 30000) */ + maxDelayMs?: number; + /** Function to determine if error is retryable (default: all errors retryable) */ + shouldRetry?: (error: Error, attempt: number) => boolean; + /** Callback called on each retry attempt */ + onRetry?: (error: Error, attempt: number, nextDelayMs: number) => void; +} +/** + * Retry an async function with configurable attempts and delays + * Implements exponential backoff with jitter + * + * @param fn - The async function to retry + * @param options - Retry configuration + * @returns The result of the function + * @throws The last error if all retries fail + * + * @example + * const result = await retryAsync( + * async () => await fetchData(), + * { maxAttempts: 3, delayMs: 1000, backoffMultiplier: 2 } + * ); + */ +export declare function retryAsync(fn: () => Promise, options?: RetryOptions): Promise; +/** + * Create a semaphore for limiting concurrent operations + * + * @example + * const semaphore = createSemaphore(3); // Allow max 3 concurrent operations + * await semaphore.acquire(); + * try { + * await doWork(); + * } finally { + * semaphore.release(); + * } + */ +export declare function createSemaphore(maxConcurrent: number): { + /** + * Acquire a semaphore slot + * Waits if max concurrent operations are already running + */ + acquire(): Promise; + /** + * Release a semaphore slot + * Allows next queued operation to proceed + */ + release(): void; + /** + * Get current active count + */ + getActiveCount(): number; + /** + * Get queue length + */ + getQueueLength(): number; +}; +/** + * Check if a URL is a new tab page (about:blank/about:newtab/chrome://new-tab-page/chrome://newtab). + */ +export declare function is_new_tab_page(url: string): boolean; +/** + * Check if a URL matches a domain pattern. SECURITY CRITICAL. + * + * Supports optional glob patterns and schemes: + * - *.example.com will match sub.example.com and example.com + * - *google.com will match google.com, agoogle.com, and www.google.com + * - http*://example.com will match http://example.com, https://example.com + * - chrome-extension://* will match chrome-extension://aaaaaaaaaaaa and chrome-extension://bbbbbbbbbbbbb + * + * When no scheme is specified, https is used by default for security. + * For example, 'example.com' will match 'https://example.com' but not 'http://example.com'. + * + * Note: New tab pages (about:blank, chrome://new-tab-page) must be handled at the callsite, not inside this function. + */ +export declare function match_url_with_domain_pattern(url: string, domain_pattern: string, log_warnings?: boolean): boolean; +export {}; diff --git a/dist/utils.js b/dist/utils.js new file mode 100644 index 00000000..f2ecf443 --- /dev/null +++ b/dist/utils.js @@ -0,0 +1,597 @@ +import { execSync } from 'node:child_process'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import readline from 'node:readline'; +import { stderr } from 'node:process'; +import { performance } from 'node:perf_hooks'; +import { fileURLToPath } from 'node:url'; +import { config as loadEnv } from 'dotenv'; +import * as minimatchModule from 'minimatch'; +import { createLogger } from './logging-config.js'; +loadEnv(); +const logger = createLogger('browser_use.utils'); +let _exiting = false; +const minimatch = (minimatchModule.minimatch ?? + minimatchModule.default ?? + minimatchModule); +export class SignalHandler { + loop = null; + pause_callback; + resume_callback; + custom_exit_callback; + exit_on_second_int; + interruptible_task_patterns; + is_windows; + ctrl_c_pressed = false; + waiting_for_input = false; + bound_sigint = this.sigint_handler.bind(this); + bound_sigterm = this.sigterm_handler.bind(this); + constructor(options = {}) { + this.pause_callback = options.pause_callback; + this.resume_callback = options.resume_callback; + this.custom_exit_callback = options.custom_exit_callback; + this.exit_on_second_int = options.exit_on_second_int ?? true; + this.interruptible_task_patterns = options.interruptible_task_patterns ?? [ + 'step', + 'multi_act', + 'get_next_action', + ]; + this.is_windows = os.platform() === 'win32'; + } + register() { + process.on('SIGINT', this.bound_sigint); + process.on('SIGTERM', this.bound_sigterm); + } + unregister() { + process.off('SIGINT', this.bound_sigint); + process.off('SIGTERM', this.bound_sigterm); + } + _handle_second_ctrl_c() { + if (!_exiting) { + _exiting = true; + if (this.custom_exit_callback) { + try { + this.custom_exit_callback(); + } + catch (error) { + logger.error(`Error in exit callback: ${error.message}`); + } + } + } + stderr.write('\n\n🛑 Got second Ctrl+C. Exiting immediately...\n'); + stderr.write('\x1b[?25h\x1b[0m\x1b[?1l\x1b[?2004l\r'); + process.exit(0); + } + _cancel_interruptible_tasks() { + // Node.js does not provide asyncio-style task cancellation. + // Users should manage their own interruptible work via pause/resume callbacks. + } + async wait_for_resume() { + this.waiting_for_input = true; + const green = '\x1b[32;1m'; + const red = '\x1b[31m'; + const blink = '\x1b[33;5m'; + const unblink = '\x1b[0m'; + const reset = '\x1b[0m'; + stderr.write(`➡️ Press ${green}[Enter]${reset} to resume or ${red}[Ctrl+C]${reset} again to exit${blink}...${unblink} `); + await new Promise((resolve) => { + const rl = readline.createInterface({ + input: process.stdin, + output: stderr, + }); + const cleanup = () => { + this.waiting_for_input = false; + rl.close(); + resolve(); + }; + rl.once('line', () => { + if (this.resume_callback) { + try { + this.resume_callback(); + } + catch (error) { + logger.error(`Error in resume callback: ${error.message}`); + } + } + cleanup(); + }); + rl.once('SIGINT', () => { + this._handle_second_ctrl_c(); + cleanup(); + }); + }); + } + reset() { + this.ctrl_c_pressed = false; + this.waiting_for_input = false; + } + sigint_handler() { + if (_exiting) { + process.exit(0); + } + if (this.ctrl_c_pressed) { + if (this.waiting_for_input) { + return; + } + if (this.exit_on_second_int) { + this._handle_second_ctrl_c(); + } + } + this.ctrl_c_pressed = true; + this._cancel_interruptible_tasks(); + if (this.pause_callback) { + try { + this.pause_callback(); + } + catch (error) { + logger.error(`Error in pause callback: ${error.message}`); + } + } + stderr.write('----------------------------------------------------------------------\n'); + } + sigterm_handler() { + if (!_exiting) { + _exiting = true; + stderr.write('\n\n🛑 SIGTERM received. Exiting immediately...\n\n'); + if (this.custom_exit_callback) { + this.custom_exit_callback(); + } + } + process.exit(0); + } +} +const strip_hyphen = (value) => value.replace(/^-+|-+$/g, '').trim(); +const pick_logger = (args) => { + if (args.length > 0) { + const candidate = args[0]; + if (candidate && candidate.logger) { + return candidate.logger; + } + } + return logger; +}; +export const time_execution_sync = (additional_text = '') => (func) => { + const label = strip_hyphen(additional_text); + const wrapper = function (...args) { + const start = performance.now(); + const result = func.apply(this, args); + const execution_time = (performance.now() - start) / 1000; + if (execution_time > 0.25) { + pick_logger(args).debug(`⏳ ${label}() took ${execution_time.toFixed(2)}s`); + } + return result; + }; + return wrapper; +}; +export const time_execution_async = (additional_text = '') => (func) => { + const label = strip_hyphen(additional_text); + const wrapper = async function (...args) { + const start = performance.now(); + const result = await func.apply(this, args); + const execution_time = (performance.now() - start) / 1000; + if (execution_time > 0.25) { + pick_logger(args).debug(`⏳ ${label}() took ${execution_time.toFixed(2)}s`); + } + return result; + }; + return wrapper; +}; +export const singleton = (cls) => { + let instance; + return (...args) => { + if (instance === undefined) { + instance = cls(...args); + } + return instance; + }; +}; +export const check_env_variables = (keys, predicate = (values) => values.every((value) => value.trim().length > 0)) => { + const values = keys.map((key) => process.env[key] ?? ''); + return predicate(values); +}; +export const is_unsafe_pattern = (pattern) => { + if (pattern.includes('://')) { + const [, ...rest] = pattern.split('://'); + pattern = rest.join('://'); + } + const bare_domain = pattern.replace('.*', '').replace('*.', ''); + return bare_domain.includes('*'); +}; +export const merge_dicts = (a, b, path = []) => { + for (const key of Object.keys(b)) { + if (key in a) { + if (typeof a[key] === 'object' && + !Array.isArray(a[key]) && + typeof b[key] === 'object' && + !Array.isArray(b[key])) { + merge_dicts(a[key], b[key], [...path, key]); + } + else if (Array.isArray(a[key]) && Array.isArray(b[key])) { + a[key] = [...a[key], ...b[key]]; + } + else if (a[key] !== b[key]) { + throw new Error(`Conflict at ${[...path, key].join('.')}`); + } + } + else { + a[key] = b[key]; + } + } + return a; +}; +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const package_root = path.resolve(__dirname, '..'); +let cached_version = null; +export const get_browser_use_version = () => { + if (cached_version) { + return cached_version; + } + try { + const package_json = JSON.parse(fs.readFileSync(path.join(package_root, 'package.json'), 'utf-8')); + if (package_json?.version) { + const version = String(package_json.version); + cached_version = version; + process.env.LIBRARY_VERSION = version; + return version; + } + } + catch (error) { + logger.debug(`Error detecting browser-use version: ${error.message}`); + } + return 'unknown'; +}; +export const check_latest_browser_use_version = async () => { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 3000); + timeout.unref?.(); + try { + const response = await fetch('https://registry.npmjs.org/browser-use/latest', { + method: 'GET', + headers: { + Accept: 'application/json', + }, + signal: controller.signal, + }); + if (!response.ok) { + return null; + } + const payload = (await response.json()); + if (typeof payload.version === 'string' && payload.version.trim()) { + return payload.version.trim(); + } + return null; + } + catch { + return null; + } + finally { + clearTimeout(timeout); + } +}; +export const create_task_with_error_handling = (promise, options = {}) => { + const { name = 'unnamed', logger_instance, suppress_exceptions = false, } = options; + const log = logger_instance ?? logger; + return promise.catch((error) => { + const message = error instanceof Error ? error.message : String(error); + if (suppress_exceptions) { + log.error(`Exception in background task [${name}]: ${message}`); + return undefined; + } + log.warning(`Exception in background task [${name}]: ${message}`); + throw error; + }); +}; +export const sanitize_surrogates = (text) => { + let result = ''; + for (let index = 0; index < text.length; index += 1) { + const code = text.charCodeAt(index); + // High surrogate + if (code >= 0xd800 && code <= 0xdbff) { + const nextCode = index + 1 < text.length ? text.charCodeAt(index + 1) : null; + if (nextCode != null && nextCode >= 0xdc00 && nextCode <= 0xdfff) { + result += text[index] + text[index + 1]; + index += 1; + } + continue; + } + // Low surrogate without preceding high surrogate + if (code >= 0xdc00 && code <= 0xdfff) { + continue; + } + result += text[index]; + } + return result; +}; +let cached_git_info; +export const get_git_info = () => { + if (cached_git_info !== undefined) { + return cached_git_info; + } + try { + const git_dir = path.join(package_root, '.git'); + if (!fs.existsSync(git_dir)) { + cached_git_info = null; + return null; + } + const commit_hash = execSync('git rev-parse HEAD', { + cwd: package_root, + stdio: ['ignore', 'pipe', 'ignore'], + }) + .toString() + .trim(); + const branch = execSync('git rev-parse --abbrev-ref HEAD', { + cwd: package_root, + stdio: ['ignore', 'pipe', 'ignore'], + }) + .toString() + .trim(); + const remote_url = execSync('git config --get remote.origin.url', { + cwd: package_root, + stdio: ['ignore', 'pipe', 'ignore'], + }) + .toString() + .trim(); + const commit_timestamp = execSync('git show -s --format=%ci HEAD', { + cwd: package_root, + stdio: ['ignore', 'pipe', 'ignore'], + }) + .toString() + .trim(); + cached_git_info = { commit_hash, branch, remote_url, commit_timestamp }; + return cached_git_info; + } + catch (error) { + logger.debug(`Error getting git info: ${error.message}`); + cached_git_info = null; + return null; + } +}; +export const _log_pretty_path = (input) => { + if (!input) { + return ''; + } + if (typeof input !== 'string') { + return `<${input.constructor?.name || typeof input}>`; + } + const normalized = input.trim(); + if (!normalized) { + return ''; + } + let pretty_path = normalized.replace(os.homedir(), '~'); + pretty_path = pretty_path.replace(process.cwd(), '.'); + return pretty_path.includes(' ') ? `"${pretty_path}"` : pretty_path; +}; +export const _log_pretty_url = (value, max_len = 22) => { + let sanitized = value + .replace('https://', '') + .replace('http://', '') + .replace('www.', ''); + if (max_len !== null && sanitized.length > max_len) { + sanitized = `${sanitized.slice(0, max_len)}…`; + } + return sanitized; +}; +export const log_pretty_path = _log_pretty_path; +export const log_pretty_url = _log_pretty_url; +export const uuid7str = () => { + const timestamp = Buffer.alloc(6); + const now = Date.now(); + timestamp.writeUIntBE(now, 0, 6); + const random = crypto.randomBytes(10); + const bytes = Buffer.concat([timestamp, random]); + bytes[6] = (bytes[6] & 0x0f) | 0x70; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + const hex = bytes.toString('hex'); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +}; +/** + * Retry an async function with configurable attempts and delays + * Implements exponential backoff with jitter + * + * @param fn - The async function to retry + * @param options - Retry configuration + * @returns The result of the function + * @throws The last error if all retries fail + * + * @example + * const result = await retryAsync( + * async () => await fetchData(), + * { maxAttempts: 3, delayMs: 1000, backoffMultiplier: 2 } + * ); + */ +export async function retryAsync(fn, options = {}) { + const { maxAttempts = 3, delayMs = 1000, backoffMultiplier = 1, maxDelayMs = 30000, shouldRetry = () => true, onRetry, } = options; + let lastError = null; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await fn(); + } + catch (error) { + lastError = error; + // Check if we should retry + const isLastAttempt = attempt === maxAttempts; + if (isLastAttempt || !shouldRetry(lastError, attempt)) { + throw lastError; + } + // Calculate delay with exponential backoff and jitter + const baseDelay = delayMs * Math.pow(backoffMultiplier, attempt - 1); + const jitter = Math.random() * 0.3 * baseDelay; // Add up to 30% jitter + const nextDelay = Math.min(baseDelay + jitter, maxDelayMs); + // Notify about retry + if (onRetry) { + onRetry(lastError, attempt, nextDelay); + } + // Wait before next attempt + await new Promise((resolve) => setTimeout(resolve, nextDelay)); + } + } + // This should never be reached, but TypeScript requires it + throw lastError || new Error('Retry failed with unknown error'); +} +/** + * Create a semaphore for limiting concurrent operations + * + * @example + * const semaphore = createSemaphore(3); // Allow max 3 concurrent operations + * await semaphore.acquire(); + * try { + * await doWork(); + * } finally { + * semaphore.release(); + * } + */ +export function createSemaphore(maxConcurrent) { + let activeCount = 0; + const queue = []; + return { + /** + * Acquire a semaphore slot + * Waits if max concurrent operations are already running + */ + async acquire() { + if (activeCount < maxConcurrent) { + activeCount++; + return; + } + await new Promise((resolve) => { + queue.push(resolve); + }); + }, + /** + * Release a semaphore slot + * Allows next queued operation to proceed + */ + release() { + const next = queue.shift(); + if (next) { + next(); + } + else { + activeCount--; + } + }, + /** + * Get current active count + */ + getActiveCount() { + return activeCount; + }, + /** + * Get queue length + */ + getQueueLength() { + return queue.length; + }, + }; +} +/** + * Check if a URL is a new tab page (about:blank/about:newtab/chrome://new-tab-page/chrome://newtab). + */ +export function is_new_tab_page(url) { + return (url === 'about:blank' || + url === 'about:newtab' || + url === 'chrome://new-tab-page/' || + url === 'chrome://new-tab-page' || + url === 'chrome://newtab/' || + url === 'chrome://newtab'); +} +/** + * Check if a URL matches a domain pattern. SECURITY CRITICAL. + * + * Supports optional glob patterns and schemes: + * - *.example.com will match sub.example.com and example.com + * - *google.com will match google.com, agoogle.com, and www.google.com + * - http*://example.com will match http://example.com, https://example.com + * - chrome-extension://* will match chrome-extension://aaaaaaaaaaaa and chrome-extension://bbbbbbbbbbbbb + * + * When no scheme is specified, https is used by default for security. + * For example, 'example.com' will match 'https://example.com' but not 'http://example.com'. + * + * Note: New tab pages (about:blank, chrome://new-tab-page) must be handled at the callsite, not inside this function. + */ +export function match_url_with_domain_pattern(url, domain_pattern, log_warnings = false) { + try { + // Note: new tab pages should be handled at the callsite, not here + if (is_new_tab_page(url)) { + return false; + } + const parsed_url = new URL(url); + // Extract only the hostname and scheme components + const scheme = parsed_url.protocol.replace(':', '').toLowerCase(); + const domain = parsed_url.hostname.toLowerCase(); + if (!scheme || !domain) { + return false; + } + // Normalize the domain pattern + const normalizedPattern = domain_pattern.toLowerCase(); + // Handle pattern with scheme + let pattern_scheme; + let pattern_domain; + if (normalizedPattern.includes('://')) { + const parts = normalizedPattern.split('://'); + pattern_scheme = parts[0]; + pattern_domain = parts[1]; + } + else { + pattern_scheme = 'https'; // Default to matching only https for security + pattern_domain = normalizedPattern; + } + // Handle port in pattern (we strip ports from patterns since we already extracted only the hostname from the URL) + if (pattern_domain.includes(':') && !pattern_domain.startsWith(':')) { + pattern_domain = pattern_domain.split(':')[0]; + } + // If scheme doesn't match using minimatch, return false + if (!minimatch(scheme, pattern_scheme)) { + return false; + } + // Check for exact match + if (pattern_domain === '*' || domain === pattern_domain) { + return true; + } + // Handle glob patterns + if (pattern_domain.includes('*')) { + // Check for unsafe glob patterns + // First, check for patterns like *.*.domain which are unsafe + if ((pattern_domain.match(/\*\./g) || []).length > 1 || + (pattern_domain.match(/\.\*/g) || []).length > 1) { + if (log_warnings) { + console.error(`⛔️ Multiple wildcards in pattern=[${domain_pattern}] are not supported`); + } + return false; // Don't match unsafe patterns + } + // Check for wildcards in TLD part (example.*) + if (pattern_domain.endsWith('.*')) { + if (log_warnings) { + console.error(`⛔️ Wildcard TLDs like in pattern=[${domain_pattern}] are not supported for security`); + } + return false; // Don't match unsafe patterns + } + // Then check for embedded wildcards + const bare_domain = pattern_domain.replace('*.', ''); + if (bare_domain.includes('*')) { + if (log_warnings) { + console.error(`⛔️ Only *.domain style patterns are supported, ignoring pattern=[${domain_pattern}]`); + } + return false; // Don't match unsafe patterns + } + // Special handling so that *.google.com also matches bare google.com + if (pattern_domain.startsWith('*.')) { + const base = pattern_domain.slice(2); // Remove '*.' + if (domain === base || domain.endsWith('.' + base)) { + return true; + } + } + // Use minimatch for pattern matching + return minimatch(domain, pattern_domain); + } + // No match + return false; + } + catch (error) { + // Invalid URL or pattern + return false; + } +} From 2dfd1b8d4adb51ee7f1ab3fab6a0c929208af904 Mon Sep 17 00:00:00 2001 From: Aleksey Bykhun Date: Tue, 5 May 2026 20:10:24 -0700 Subject: [PATCH 04/14] fix(dom): include shadow DOM form elements when offsetWidth/Height are 0 Ports the equivalent of upstream Python PR #3793 to the TS in-page DOM walker. The Python fix targeted CDP DOMSnapshot omitting layout for shadow DOM nodes; the TS port uses live DOM and gates inclusion on isElementVisible (offsetWidth/Height > 0). Custom-styled shadow widgets (e.g. auth0 login) can still report 0 dimensions on the inner control while being functional. For input/button/select/textarea/a inside a shadow root, treat the element as visible+top so isInteractiveElement runs and it lands in selector_map. --- src/dom/dom_tree/index.js | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/dom/dom_tree/index.js b/src/dom/dom_tree/index.js index 9847ff78..db761804 100644 --- a/src/dom/dom_tree/index.js +++ b/src/dom/dom_tree/index.js @@ -988,6 +988,17 @@ return hasQuickInteractiveAttr; } + // Tags that should remain interactive even when offsetWidth/Height report 0 + // when they live inside a shadow root. Matches the upstream Python fix where + // shadow-DOM form elements lacking CDP DOMSnapshot layout still go into the + // selector_map (e.g. auth0-style login widgets). + const SHADOW_DOM_FORM_TAGS = new Set(['input', 'button', 'select', 'textarea', 'a']); + + function isInsideShadowDom(element) { + let root = element.getRootNode && element.getRootNode(); + return Boolean(root && root instanceof ShadowRoot); + } + // --- Define constants for distinct interaction check --- const DISTINCT_INTERACTIVE_TAGS = new Set([ 'a', 'button', 'input', 'select', 'textarea', 'summary', 'details', 'label', 'option' @@ -1321,13 +1332,25 @@ // Perform visibility, interactivity, and highlighting checks if (node.nodeType === Node.ELEMENT_NODE) { nodeData.isVisible = isElementVisible(node); // isElementVisible uses offsetWidth/Height, which is fine + + // Shadow-DOM exception: form tags inside a shadow root that report + // offsetWidth/Height = 0 are still functional. Treating them as visible + // here lets isInteractiveElement run and the element reach selector_map. + const isShadowDomFormElement = + !nodeData.isVisible && + SHADOW_DOM_FORM_TAGS.has(node.tagName.toLowerCase()) && + isInsideShadowDom(node); + if (isShadowDomFormElement) { + nodeData.isVisible = true; + } + if (nodeData.isVisible) { - nodeData.isTopElement = isTopElement(node); - + nodeData.isTopElement = isShadowDomFormElement ? true : isTopElement(node); + // Special handling for ARIA menu containers - check interactivity even if not top element const role = node.getAttribute('role'); const isMenuContainer = role === 'menu' || role === 'menubar' || role === 'listbox'; - + if (nodeData.isTopElement || isMenuContainer) { nodeData.isInteractive = isInteractiveElement(node); // Call the dedicated highlighting function From 278341b9de48360e32bde62e402a540063db5136 Mon Sep 17 00:00:00 2001 From: Aleksey Bykhun Date: Tue, 5 May 2026 20:11:42 -0700 Subject: [PATCH 05/14] feat(prompts): add generic retry/loop-breaking rules to bu-flash prompt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The browser-use provider auto-flips flash_mode (service.ts:599), which selects the 15-line minimal prompt. That prompt lacked any guidance for loop avoidance, retry strategy, or autocomplete handling — gaps the fine-tuned bu-2-0 model was supposed to internalize but doesn't always. Append a short block (5 generic rules, no URL or provider matching) covering: same-action-3x, stuck-URL, dead clicks, autocomplete value mismatch, and missing credentials. Keeps the prompt small (15 -> 23 lines) without re-inflating to the full 269-line variant. --- src/agent/system_prompt_browser_use_flash.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/agent/system_prompt_browser_use_flash.md b/src/agent/system_prompt_browser_use_flash.md index 0a04cf5e..0179b3cf 100644 --- a/src/agent/system_prompt_browser_use_flash.md +++ b/src/agent/system_prompt_browser_use_flash.md @@ -4,6 +4,14 @@ You are a browser-use agent operating in flash mode. You automate browser tasks Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative. + +- If you have taken the same action 3+ times without visible progress, do NOT repeat it. Try a different element, a different action type, or scroll to surface new options. +- If the URL has not changed for 3+ steps and the page state looks identical, you are stuck. Switch strategy: scroll, open a different element, or reload. +- If a click or input did nothing, do NOT click the same index again — re-read the element list, the target may have moved or a new element may now match better (look for *[index] markers). +- If the field's actual value differs from what you typed (page reformatted/autocompleted), wait one step for suggestions, then click a suggestion instead of pressing Enter. +- If credentials are needed and not provided, stop and report; do NOT invent emails, passwords, or codes. + + You must respond with a valid JSON in this exact format: {{ From af21630f40c736ab488f7245feedc0a6d8f84cac Mon Sep 17 00:00:00 2001 From: Aleksey Bykhun Date: Wed, 6 May 2026 11:24:13 -0700 Subject: [PATCH 06/14] fix(controller): coerce booleans to ints for index fields (pydantic parity) Empirical observation: bu-2-0 (browser-use cloud LLM) occasionally emits `{input_text: {index: true, text: "..."}}` (boolean) where the schema expects an integer. python upstream silently coerces `True -> 1` / `False -> 0` via pydantic's default lax mode. zod (TS) hard-rejects with `expected number, received boolean`, the agent retries with the same broken output, and bails at max_failures. observed bail mode in production for every auth0-style form fill (daytona, zeroentropy, kernel, browserbase). This patch ports the lax-coercion behavior to TS at the validation boundary. A `lenientInt(min)` helper preprocesses booleans into numbers before delegating to `z.number().int().min(min)`. Same shape for `lenientNumber()` covering `num_pages` / `pages` floats. Helper is applied only to LLM-emitted index/element-index/page-count fields where pydantic's silent coercion is documented behavior. Fields where bool->0/1 would be semantically wrong (timeout, delay, max_results, coordinate_x/y) are left strict to avoid masking a different model bug. This is a graceful-degradation patch, not a fix to the model. bu-2-0 should not emit booleans for integer fields. With this patch the agent now progresses + relies on existing retry-feedback (PR #34) to self-correct on bad index choice rather than looping to max_failures. Helpers + per-schema regression coverage in `test/coerce-boolean-to-int.test.ts` (23 tests). Co-Authored-By: Claude Sonnet 4.6 --- src/controller/views.ts | 38 +++++-- test/coerce-boolean-to-int.test.ts | 172 +++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+), 9 deletions(-) create mode 100644 test/coerce-boolean-to-int.test.ts diff --git a/src/controller/views.ts b/src/controller/views.ts index 91e8e893..4e936b92 100644 --- a/src/controller/views.ts +++ b/src/controller/views.ts @@ -1,5 +1,25 @@ import { z } from 'zod'; +// pydantic-parity: lax boolean->number coercion at the LLM-facing boundary. +// bu-2-0 occasionally emits booleans for integer fields (token-level confusion); +// python upstream accepts this silently via pydantic's default lax mode. +// Without this, every action that expects a numeric index hard-rejects and the +// agent loops to max_failures. See: https://docs.pydantic.dev/latest/concepts/strict_mode/ +export const lenientInt = (min?: number) => { + let schema = z.number().int(); + if (min !== undefined) schema = schema.min(min); + return z.preprocess( + (v) => (typeof v === 'boolean' ? Number(v) : v), + schema, + ); +}; + +export const lenientNumber = () => + z.preprocess( + (v) => (typeof v === 'boolean' ? Number(v) : v), + z.number(), + ); + export const SearchGoogleActionSchema = z.object({ query: z.string(), }); @@ -23,21 +43,21 @@ export const WaitActionSchema = z.object({ export type WaitAction = z.infer; export const ClickElementActionSchema = z.object({ - index: z.number().int().min(1).optional(), + index: lenientInt(1).optional(), coordinate_x: z.number().int().optional(), coordinate_y: z.number().int().optional(), }); export type ClickElementAction = z.infer; export const ClickElementActionIndexOnlySchema = z.object({ - index: z.number().int().min(1), + index: lenientInt(1), }); export type ClickElementActionIndexOnly = z.infer< typeof ClickElementActionIndexOnlySchema >; export const InputTextActionSchema = z.object({ - index: z.number().int().min(0), + index: lenientInt(0), text: z.string(), clear: z.boolean().default(true), }); @@ -82,9 +102,9 @@ export type CloseTabAction = z.infer; export const ScrollActionSchema = z.object({ down: z.boolean().default(true), // Default to scroll down - num_pages: z.number().default(1), // Default to 1 page - pages: z.number().optional(), // Alias for num_pages - index: z.number().int().optional(), + num_pages: lenientNumber().default(1), // Default to 1 page + pages: lenientNumber().optional(), // Alias for num_pages + index: lenientInt().optional(), }); export type ScrollAction = z.infer; @@ -94,7 +114,7 @@ export const SendKeysActionSchema = z.object({ export type SendKeysAction = z.infer; export const UploadFileActionSchema = z.object({ - index: z.number().int(), + index: lenientInt(), path: z.string(), }); export type UploadFileAction = z.infer; @@ -181,12 +201,12 @@ export const ScrollToTextActionSchema = z.object({ export type ScrollToTextAction = z.infer; export const DropdownOptionsActionSchema = z.object({ - index: z.number().int().min(1), + index: lenientInt(1), }); export type DropdownOptionsAction = z.infer; export const SelectDropdownActionSchema = z.object({ - index: z.number().int().min(1), + index: lenientInt(1), text: z.string(), }); export type SelectDropdownAction = z.infer; diff --git a/test/coerce-boolean-to-int.test.ts b/test/coerce-boolean-to-int.test.ts new file mode 100644 index 00000000..a87f8e3f --- /dev/null +++ b/test/coerce-boolean-to-int.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, it } from 'vitest'; +import { + ClickElementActionSchema, + DropdownOptionsActionSchema, + InputTextActionSchema, + ScrollActionSchema, + SelectDropdownActionSchema, + UploadFileActionSchema, + lenientInt, + lenientNumber, +} from '../src/controller/views.js'; + +describe('lenientInt helper (pydantic-parity boolean->int coercion)', () => { + it('coerces true to 1', () => { + const result = lenientInt().safeParse(true); + expect(result.success).toBe(true); + if (result.success) expect(result.data).toBe(1); + }); + + it('coerces false to 0', () => { + const result = lenientInt().safeParse(false); + expect(result.success).toBe(true); + if (result.success) expect(result.data).toBe(0); + }); + + it('passes through plain integers', () => { + const result = lenientInt().safeParse(5); + expect(result.success).toBe(true); + if (result.success) expect(result.data).toBe(5); + }); + + it('does not coerce strings (matches pydantic strict-string behavior)', () => { + const result = lenientInt().safeParse('5'); + expect(result.success).toBe(false); + }); + + it('rejects non-integer numbers', () => { + const result = lenientInt().safeParse(3.14); + expect(result.success).toBe(false); + }); + + it('passes false (=>0) when min=0', () => { + const result = lenientInt(0).safeParse(false); + expect(result.success).toBe(true); + if (result.success) expect(result.data).toBe(0); + }); + + it('rejects false (=>0) when min=1', () => { + const result = lenientInt(1).safeParse(false); + expect(result.success).toBe(false); + }); + + it('accepts true (=>1) when min=1', () => { + const result = lenientInt(1).safeParse(true); + expect(result.success).toBe(true); + if (result.success) expect(result.data).toBe(1); + }); +}); + +describe('lenientNumber helper (pydantic-parity boolean->number coercion)', () => { + it('coerces true to 1', () => { + const result = lenientNumber().safeParse(true); + expect(result.success).toBe(true); + if (result.success) expect(result.data).toBe(1); + }); + + it('coerces false to 0', () => { + const result = lenientNumber().safeParse(false); + expect(result.success).toBe(true); + if (result.success) expect(result.data).toBe(0); + }); + + it('passes through floats', () => { + const result = lenientNumber().safeParse(2.5); + expect(result.success).toBe(true); + if (result.success) expect(result.data).toBe(2.5); + }); +}); + +describe('InputTextActionSchema lenient index coercion', () => { + it('accepts {index: true, text: "x"} as {index: 1, text: "x"}', () => { + const result = InputTextActionSchema.safeParse({ index: true, text: 'x' }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.index).toBe(1); + expect(result.data.text).toBe('x'); + } + }); + + it('accepts {index: false, text: "x"} as {index: 0, text: "x"}', () => { + const result = InputTextActionSchema.safeParse({ index: false, text: 'x' }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.index).toBe(0); + expect(result.data.text).toBe('x'); + } + }); + + it('accepts plain integer indices', () => { + const result = InputTextActionSchema.safeParse({ index: 5, text: 'x' }); + expect(result.success).toBe(true); + if (result.success) expect(result.data.index).toBe(5); + }); +}); + +describe('ClickElementActionSchema lenient index coercion', () => { + it('accepts {index: true} as {index: 1}', () => { + const result = ClickElementActionSchema.safeParse({ index: true }); + expect(result.success).toBe(true); + if (result.success) expect(result.data.index).toBe(1); + }); + + it('rejects {index: false} because min=1', () => { + const result = ClickElementActionSchema.safeParse({ index: false }); + expect(result.success).toBe(false); + }); +}); + +describe('UploadFileActionSchema lenient index coercion', () => { + it('accepts {index: true, path: "/tmp/x"} as {index: 1, ...}', () => { + const result = UploadFileActionSchema.safeParse({ + index: true, + path: '/tmp/x', + }); + expect(result.success).toBe(true); + if (result.success) expect(result.data.index).toBe(1); + }); +}); + +describe('DropdownOptionsActionSchema lenient index coercion', () => { + it('accepts {index: true} as {index: 1}', () => { + const result = DropdownOptionsActionSchema.safeParse({ index: true }); + expect(result.success).toBe(true); + if (result.success) expect(result.data.index).toBe(1); + }); + + it('rejects {index: false} because min=1', () => { + const result = DropdownOptionsActionSchema.safeParse({ index: false }); + expect(result.success).toBe(false); + }); +}); + +describe('SelectDropdownActionSchema lenient index coercion', () => { + it('accepts {index: true, text: "opt"} as {index: 1, ...}', () => { + const result = SelectDropdownActionSchema.safeParse({ + index: true, + text: 'opt', + }); + expect(result.success).toBe(true); + if (result.success) expect(result.data.index).toBe(1); + }); +}); + +describe('ScrollActionSchema lenient num_pages/pages/index coercion', () => { + it('accepts {num_pages: true} as {num_pages: 1}', () => { + const result = ScrollActionSchema.safeParse({ num_pages: true }); + expect(result.success).toBe(true); + if (result.success) expect(result.data.num_pages).toBe(1); + }); + + it('accepts {pages: false} as {pages: 0}', () => { + const result = ScrollActionSchema.safeParse({ pages: false }); + expect(result.success).toBe(true); + if (result.success) expect(result.data.pages).toBe(0); + }); + + it('accepts {index: true} as {index: 1}', () => { + const result = ScrollActionSchema.safeParse({ index: true }); + expect(result.success).toBe(true); + if (result.success) expect(result.data.index).toBe(1); + }); +}); From f83076fb2c472c6307b868876991b69aa2d0e489 Mon Sep 17 00:00:00 2001 From: Aleksey Bykhun Date: Wed, 6 May 2026 11:32:15 -0700 Subject: [PATCH 07/14] test(fixture): capture raw bu-2-0 response showing index: true emission Hand-captured request/response pair from a real BA invocation against the daytona.io auth0 login page. The model emits {input_text: {index: true}} which silently passes pydantic in upstream python (True -> 1) but hard-fails zod in this TS port. Useful for upstream model-bug reporting and as test data for the boolean->int coercion fix. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/fixtures/bu-2-0-boolean-index-bug.json | 1481 +++++++++++++++++++ 1 file changed, 1481 insertions(+) create mode 100644 test/fixtures/bu-2-0-boolean-index-bug.json diff --git a/test/fixtures/bu-2-0-boolean-index-bug.json b/test/fixtures/bu-2-0-boolean-index-bug.json new file mode 100644 index 00000000..7024dd03 --- /dev/null +++ b/test/fixtures/bu-2-0-boolean-index-bug.json @@ -0,0 +1,1481 @@ +{ + "description": "bu-2-0 model emits {input_text: {index: true}} (boolean) instead of integer when handling auth0-style login pages. Reproducible against app.daytona.io login page on the first input_text within a multi-action batch. Python pydantic silently coerces True->1; TS zod hard-rejects without preprocess.", + "captured_at": "2026-05-06T18:30:51.991Z", + "model": "bu-2-0", + "endpoint": "https://llm.api.browser-use.com/v1/chat/completions", + "capture_method": "Instrumented dist/llm/browser-use/chat.js makeRequest() via stderr writes (BU-FIXTURE-REQUEST/HEADERS/STATUS/RESPONSE) while running canary-env/browser/cli/run.ts against https://app.daytona.io/login.", + "reproduction_target": "https://app.daytona.io/login", + "request": { + "method": "POST", + "headers": { + "Authorization": "Bearer ", + "Content-Type": "application/json" + }, + "body": { + "model": "bu-2-0", + "messages": [ + { + "role": "system", + "content": "You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in .\nDefault: English. Match user's language.\nUltimate objective. Specific tasks: follow each step. Open-ended: plan approach.\nElements: [index]text. Only [indexed] are interactive. Indentation=child. *[=new.\n- PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. When writing CSV, use double quotes for commas. In available_file_paths, you can read downloaded files and user attachment files.\n\nYou are allowed to use a maximum of 5 actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.\n\nYou must respond with a valid JSON in this exact format:\n{{\n \"memory\": \"Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its opvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.\",\n \"action\":[{{\"navigate\": {{ \"url\": \"url_value\"}}}}]\n}}\nBefore calling `done` with `success=true`: re-read the user request, verify every requirement is met (correct count, filters applied, format matched), confirm actions actually completed via page state/screenshot, and ensure no data was fabricated. If anything is unmet or uncertain, set `success` to `false`. BLOCKING ERROR CHECK: if you encountered an unresolved blocking error (payment declined, login failed with no credentials, email verification wall, access denied not bypassed, required paywall) you MUST set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups) do not count.\nDATA GROUNDING: Only report data observed in browser state or tool outputs. Never fabricate URLs, prices, or values — including \"representative\" ones. If not found, say so.\n\n" + }, + { + "role": "user", + "content": [ + { + "text": "\nAgent initialized\n [ROLE]\nYou are a user emulator helping a coding agent finish a task with daytona.\nYour role is REACTIVE: act when the coding agent reports it cannot proceed (authentication required, login needed, credentials missing, an error you can resolve) OR when the coding agent leaves TODO items you can fulfill (URL/API key/config value the user needs to fetch from a provider dashboard).\nRunComplete means the task is end-to-end working — not just \"files written.\" If the coding agent's message contains TODO-style items (\"to finish setup\", \"you need to\", \"get this from dashboard\", \"set X in config\"), the task is NOT done. Read each TODO: if it's something YOU can fetch (dashboard URL, API key, project ID, secret), act on it via Login*/AlreadyLoggedIn with credentials. If it's user-domain (deploy, configure DNS, change billing), emit Status describing the gap.\nDo NOT proactively log in, create credentials, or take action the coding agent did not signal as needed. Reply with exactly one structured-output variant.\n\n[BROWSER OPS HINTS]\n\nTip: navigator.clipboard.readText() does NOT work in this environment. After clicking a Copy button or creating a token, read the token directly from the DOM — use document.querySelector to find the value from input fields, code blocks, or elements near the Copy button.\n\nOn GitHub OAuth authorization pages, the Authorize button starts disabled (anti-clickjacking protection). Move the mouse around the page first to trigger GitHub's interaction detection, wait 1-2 seconds for the button to become enabled, then click the button with name=\"authorize\" and value=\"1\" (value=\"0\" is Cancel).\n\nAfter an OAuth redirect or login completion, Chrome may destroy page frames and become unresponsive (you can't read page content or interact with elements). If this happens, refresh the page by navigating to the current URL again, wait 2 seconds, then retry your action.\n\n[DECISION RULES]\n- Read the EVENT first. Three categories:\n (a) coding agent is asking for help, hitting an auth wall, or reporting an error → act.\n (b) coding agent left TODO items you can fetch (dashboard URL, API key, project ID, secret, config value annotated \"get this from \") → act on the fetchable ones.\n (c) coding agent is genuinely done — task is end-to-end working, no handoff guide, no TODO list, no \"to finish setup\" → emit RunComplete.\n (d) coding agent is narrating progress with no blocker and no TODO → emit Noop.\n- The [LOGIN HINT] section is a resource: credentials to USE if authentication is needed. It is NOT an instruction to log in. Only log in when the coding agent reports it cannot proceed without auth, OR when fulfilling a TODO that requires dashboard access.\n- RunComplete is strict: it means end-to-end working completion. \"CA wrote files + handoff guide\" is NOT RunComplete — that's an unfinished task. Look for explicit signals that the integration runs and does what was asked.\n- If you'd attach `credentials`, compare them to `delivered_creds` in STATE:\n - exact match (same keys, same values) → emit RunComplete (task done) or Noop (agent just narrating); do NOT re-emit Login*/AlreadyLoggedIn with creds already delivered.\n - different (new key, rotated value, additional scope) → emit the appropriate variant with the NEW credentials.\n- If `logged_in: true` and the agent is just narrating progress, emit Noop.\n- OAuth completion means the task is DONE — emit RunComplete (or AlreadyLoggedIn if you happened to land on a logged-in dashboard). Pages like \"Login successful\", \"Return to the terminal\", \"You may close this tab\", \"You are now logged in\", or an unreachable localhost:PORT/callback — the token was already delivered via the localhost callback. Do NOT try to find a token in the page or navigate to the dashboard.\n- auth_url events MUST NOT emit RunComplete (this is a navigation event, not task closure).\n- Most common mistake: re-emitting LoginSucceeded with credentials you already delivered. Re-read `delivered_creds` before attaching `credentials`.\n\n[LOGIN HINT]\n / \n\n[VARIANTS] (choose exactly one)\n- Noop — nothing to do; the coding agent doesn't need anything from you.\n- LoginSucceeded(credentials?, note?) — you just logged in. `credentials` is an optional string→string map of newly-visible creds; omit if you're not seeing fresh creds. `note` is freeform context.\n- LoginFailed(reason) — captcha, MFA you can't solve, server error, account disabled, etc.\n- AlreadyLoggedIn(credentials?) — page shows you're already logged in (persistent cookie/session). `credentials` is optional — include only fresh creds visible on the page.\n- RunComplete — coding agent's task is done (or it's working fine with what it has) — close the run. Forbidden after auth_url.\n- Status(message) — neutral progress chatter to forward.\n\n[STATE]\n(no prior state)\n\n[EVENT]\nThe coding agent is working with daytona and stopped with stop_reason=\"end_turn\".\nLast message:\n\n\"Go to https://app.daytona.io/login and log in with the provided credentials so the agent can fetch a DAYTONA_API_KEY.\" \n\nThe coding agent requested a login to Daytona to fetch an API key. I am starting by navigating to the Daytona login page as instructed.\nResult\nNavigated to https://app.daytona.io/login\n\nThe initial attempt to reach the login page resulted in a 404 error. I will try navigating to the base URL to find the correct login path.\nResult\nNavigated to https://app.daytona.io/\n\n\n\n\n(persistent session — task set per event)\nExpected output format: StructuredOutput\n{\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"anyOf\": [\n {\n \"type\": \"string\"\n },\n {\n \"default\": {\n \"action\": \"noop\"\n },\n \"oneOf\": [\n {\n \"type\": \"object\",\n \"properties\": {\n \"action\": {\n \"type\": \"string\",\n \"const\": \"noop\"\n }\n },\n \"required\": [\n \"action\"\n ],\n \"additionalProperties\": false\n },\n {\n \"type\": \"object\",\n \"properties\": {\n \"action\": {\n \"type\": \"string\",\n \"const\": \"status\"\n },\n \"message\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"action\",\n \"message\"\n ],\n \"additionalProperties\": false\n },\n {\n \"type\": \"object\",\n \"properties\": {\n \"action\": {\n \"type\": \"string\",\n \"const\": \"login_succeeded\"\n },\n \"credentials\": {\n \"anyOf\": [\n {\n \"type\": \"object\",\n \"propertyNames\": {\n \"type\": \"string\"\n },\n \"additionalProperties\": {\n \"type\": \"string\"\n }\n },\n {\n \"type\": \"null\"\n }\n ]\n },\n \"note\": {\n \"default\": \"\",\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"action\",\n \"note\"\n ],\n \"additionalProperties\": false\n },\n {\n \"type\": \"object\",\n \"properties\": {\n \"action\": {\n \"type\": \"string\",\n \"const\": \"login_failed\"\n },\n \"reason\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"action\",\n \"reason\"\n ],\n \"additionalProperties\": false\n },\n {\n \"type\": \"object\",\n \"properties\": {\n \"action\": {\n \"type\": \"string\",\n \"const\": \"already_logged_in\"\n },\n \"credentials\": {\n \"anyOf\": [\n {\n \"type\": \"object\",\n \"propertyNames\": {\n \"type\": \"string\"\n },\n \"additionalProperties\": {\n \"type\": \"string\"\n }\n },\n {\n \"type\": \"null\"\n }\n ]\n }\n },\n \"required\": [\n \"action\"\n ],\n \"additionalProperties\": false\n },\n {\n \"type\": \"object\",\n \"properties\": {\n \"action\": {\n \"type\": \"string\",\n \"const\": \"run_complete\"\n }\n },\n \"required\": [\n \"action\"\n ],\n \"additionalProperties\": false\n }\n ]\n }\n ]\n}\n [ROLE]\nYou are a user emulator helping a coding agent finish a task with daytona.\nYour role is REACTIVE: act when the coding agent reports it cannot proceed (authentication required, login needed, credentials missing, an error you can resolve) OR when the coding agent leaves TODO items you can fulfill (URL/API key/config value the user needs to fetch from a provider dashboard).\nRunComplete means the task is end-to-end working — not just \"files written.\" If the coding agent's message contains TODO-style items (\"to finish setup\", \"you need to\", \"get this from dashboard\", \"set X in config\"), the task is NOT done. Read each TODO: if it's something YOU can fetch (dashboard URL, API key, project ID, secret), act on it via Login*/AlreadyLoggedIn with credentials. If it's user-domain (deploy, configure DNS, change billing), emit Status describing the gap.\nDo NOT proactively log in, create credentials, or take action the coding agent did not signal as needed. Reply with exactly one structured-output variant.\n\n[BROWSER OPS HINTS]\n\nTip: navigator.clipboard.readText() does NOT work in this environment. After clicking a Copy button or creating a token, read the token directly from the DOM — use document.querySelector to find the value from input fields, code blocks, or elements near the Copy button.\n\nOn GitHub OAuth authorization pages, the Authorize button starts disabled (anti-clickjacking protection). Move the mouse around the page first to trigger GitHub's interaction detection, wait 1-2 seconds for the button to become enabled, then click the button with name=\"authorize\" and value=\"1\" (value=\"0\" is Cancel).\n\nAfter an OAuth redirect or login completion, Chrome may destroy page frames and become unresponsive (you can't read page content or interact with elements). If this happens, refresh the page by navigating to the current URL again, wait 2 seconds, then retry your action.\n\n[DECISION RULES]\n- Read the EVENT first. Three categories:\n (a) coding agent is asking for help, hitting an auth wall, or reporting an error → act.\n (b) coding agent left TODO items you can fetch (dashboard URL, API key, project ID, secret, config value annotated \"get this from \") → act on the fetchable ones.\n (c) coding agent is genuinely done — task is end-to-end working, no handoff guide, no TODO list, no \"to finish setup\" → emit RunComplete.\n (d) coding agent is narrating progress with no blocker and no TODO → emit Noop.\n- The [LOGIN HINT] section is a resource: credentials to USE if authentication is needed. It is NOT an instruction to log in. Only log in when the coding agent reports it cannot proceed without auth, OR when fulfilling a TODO that requires dashboard access.\n- RunComplete is strict: it means end-to-end working completion. \"CA wrote files + handoff guide\" is NOT RunComplete — that's an unfinished task. Look for explicit signals that the integration runs and does what was asked.\n- If you'd attach `credentials`, compare them to `delivered_creds` in STATE:\n - exact match (same keys, same values) → emit RunComplete (task done) or Noop (agent just narrating); do NOT re-emit Login*/AlreadyLoggedIn with creds already delivered.\n - different (new key, rotated value, additional scope) → emit the appropriate variant with the NEW credentials.\n- If `logged_in: true` and the agent is just narrating progress, emit Noop.\n- OAuth completion means the task is DONE — emit RunComplete (or AlreadyLoggedIn if you happened to land on a logged-in dashboard). Pages like \"Login successful\", \"Return to the terminal\", \"You may close this tab\", \"You are now logged in\", or an unreachable localhost:PORT/callback — the token was already delivered via the localhost callback. Do NOT try to find a token in the page or navigate to the dashboard.\n- auth_url events MUST NOT emit RunComplete (this is a navigation event, not task closure).\n- Most common mistake: re-emitting LoginSucceeded with credentials you already delivered. Re-read `delivered_creds` before attaching `credentials`.\n\n[LOGIN HINT]\n / \n\n[VARIANTS] (choose exactly one)\n- Noop — nothing to do; the coding agent doesn't need anything from you.\n- LoginSucceeded(credentials?, note?) — you just logged in. `credentials` is an optional string→string map of newly-visible creds; omit if you're not seeing fresh creds. `note` is freeform context.\n- LoginFailed(reason) — captcha, MFA you can't solve, server error, account disabled, etc.\n- AlreadyLoggedIn(credentials?) — page shows you're already logged in (persistent cookie/session). `credentials` is optional — include only fresh creds visible on the page.\n- RunComplete — coding agent's task is done (or it's working fine with what it has) — close the run. Forbidden after auth_url.\n- Status(message) — neutral progress chatter to forward.\n\n[STATE]\n(no prior state)\n\n[EVENT]\nThe coding agent is working with daytona and stopped with stop_reason=\"end_turn\".\nLast message:\n\n\"Go to https://app.daytona.io/login and log in with the provided credentials so the agent can fetch a DAYTONA_API_KEY.\" \n\n\n\n\n\n[empty todo.md, fill it when applicable]\n\nStep3 maximum:30\nToday:2026-05-06\n\n\n2 links, 9 interactive, 0 iframes, 1 images, 57 total elements\nCurrent tab: 0000\nAvailable tabs:\nTab 0000: https://daytonaio.us.auth0.com/u/login?state=hKFo2SAxWVdTLVRMb1...1442697 - Log in | Daytona\nTab 0001: about:blank - about:blank\nTab 0002: https://accounts.google.com/v3/signin/challenge/pwd?TL=APouJz4tOx8luQ31aZB0Y...abc4674 - https://accounts.google.com/v3\n\n0.0 above, 0.4 below \n\nInteractive elements:\n[Start of page]\nWelcome\nSANDBOXES FOR AI GENERATED CODE\n[0]