diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 79482f9685..63b08082f5 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -1,8 +1,13 @@ # Use Node.js runtime for production FROM node:20-slim -# Install required packages (wget for healthcheck, curl/bash for bun install, libc compat for prisma) -RUN apt-get update && apt-get install -y --no-install-recommends wget curl openssl && rm -rf /var/lib/apt/lists/* +# Install required packages: +# - wget for healthcheck, curl/bash for bun install, openssl for prisma +# - fontconfig + fonts-dejavu-core so sharp/librsvg can render text in the +# browser-automation screenshot overlay (node:*-slim ships with zero fonts) +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget curl openssl fontconfig fonts-dejavu-core \ + && rm -rf /var/lib/apt/lists/* WORKDIR /app diff --git a/apps/api/Dockerfile.multistage b/apps/api/Dockerfile.multistage index 90e1c5f38d..1830d7c426 100644 --- a/apps/api/Dockerfile.multistage +++ b/apps/api/Dockerfile.multistage @@ -84,8 +84,11 @@ RUN groupadd --system nestjs && useradd --system --gid nestjs --create-home nest WORKDIR /app RUN chown nestjs:nestjs /app -# Install runtime dependencies + AWS RDS CA certificate bundle -RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates \ +# Install runtime dependencies + AWS RDS CA certificate bundle. +# fontconfig + fonts-dejavu-core are needed so sharp/librsvg can render text +# in the browser-automation screenshot overlay (node:*-slim has no fonts). +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget ca-certificates fontconfig fonts-dejavu-core \ && wget -q -O /usr/local/share/aws-rds-ca-bundle.pem https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ && apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/apps/api/src/browserbase/browserbase.controller.spec.ts b/apps/api/src/browserbase/browserbase.controller.spec.ts new file mode 100644 index 0000000000..1713edeffa --- /dev/null +++ b/apps/api/src/browserbase/browserbase.controller.spec.ts @@ -0,0 +1,105 @@ +// apps/api/src/browserbase/browserbase.controller.spec.ts +jest.mock('@db', () => ({ + db: {}, + Prisma: { + PrismaClientKnownRequestError: class PrismaClientKnownRequestError extends Error { + code: string; + constructor(message: string, { code }: { code: string }) { + super(message); + this.code = code; + } + }, + }, +})); + +jest.mock('../auth/auth.server', () => ({ + auth: { + api: { + getSession: jest.fn(), + }, + }, +})); + +jest.mock('@trycompai/auth', () => ({ + statement: {}, + BUILT_IN_ROLE_PERMISSIONS: {}, +})); + +import { Test } from '@nestjs/testing'; +import { NotFoundException } from '@nestjs/common'; +import type { Response } from 'express'; +import { BrowserbaseController } from './browserbase.controller'; +import { BrowserbaseService } from './browserbase.service'; +import { HybridAuthGuard } from '../auth/hybrid-auth.guard'; +import { PermissionGuard } from '../auth/permission.guard'; + +describe('BrowserbaseController.redirectToScreenshot', () => { + let controller: BrowserbaseController; + let service: jest.Mocked>; + + beforeEach(async () => { + service = { + getScreenshotRedirectUrl: jest.fn(), + } as jest.Mocked>; + + const moduleRef = await Test.createTestingModule({ + controllers: [BrowserbaseController], + providers: [{ provide: BrowserbaseService, useValue: service }], + }) + .overrideGuard(HybridAuthGuard) + .useValue({ canActivate: () => true }) + .overrideGuard(PermissionGuard) + .useValue({ canActivate: () => true }) + .compile(); + + controller = moduleRef.get(BrowserbaseController); + }); + + const makeRes = () => { + const res: Partial = { redirect: jest.fn() }; + return res as Response & { redirect: jest.Mock }; + }; + + it('302-redirects to the freshly minted presigned URL', async () => { + service.getScreenshotRedirectUrl.mockResolvedValue( + 'https://s3.example.com/fresh-signed', + ); + const res = makeRes(); + + await controller.redirectToScreenshot('bar_1', 'org_1', res); + + expect(service.getScreenshotRedirectUrl).toHaveBeenCalledWith({ + runId: 'bar_1', + organizationId: 'org_1', + download: false, + }); + expect(res.redirect).toHaveBeenCalledWith(302, 'https://s3.example.com/fresh-signed'); + }); + + it('passes download=true to the service when the query param is "true"', async () => { + service.getScreenshotRedirectUrl.mockResolvedValue( + 'https://s3.example.com/fresh-signed-attachment', + ); + const res = makeRes(); + + await controller.redirectToScreenshot('bar_1', 'org_1', res, 'true'); + + expect(service.getScreenshotRedirectUrl).toHaveBeenCalledWith({ + runId: 'bar_1', + organizationId: 'org_1', + download: true, + }); + }); + + it('propagates NotFoundException when the service throws', async () => { + service.getScreenshotRedirectUrl.mockRejectedValue( + new NotFoundException('Screenshot not found'), + ); + const res = makeRes(); + + await expect( + controller.redirectToScreenshot('bar_missing', 'org_1', res), + ).rejects.toBeInstanceOf(NotFoundException); + expect(res.redirect).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/browserbase/browserbase.controller.ts b/apps/api/src/browserbase/browserbase.controller.ts index cac2b7aef1..a0c7047fee 100644 --- a/apps/api/src/browserbase/browserbase.controller.ts +++ b/apps/api/src/browserbase/browserbase.controller.ts @@ -6,8 +6,11 @@ import { Param, Patch, Post, + Query, + Res, UseGuards, } from '@nestjs/common'; +import type { Response } from 'express'; import { ApiOperation, ApiParam, @@ -371,4 +374,28 @@ export class BrowserbaseController { runId, )) as BrowserAutomationRunResponseDto | null; } + + @Get('runs/:runId/screenshot') + @RequirePermission('task', 'read') + @ApiOperation({ + summary: 'Redirect to a freshly signed screenshot URL', + description: + 'Issues a 302 redirect to a newly signed S3 URL so that "Open full size" links never serve an expired URL. Pass ?download=true to force an attachment download.', + }) + @ApiParam({ name: 'runId', description: 'Run ID' }) + @ApiResponse({ status: 302, description: 'Redirect to signed S3 URL' }) + @ApiResponse({ status: 404, description: 'Run or screenshot not found' }) + async redirectToScreenshot( + @Param('runId') runId: string, + @OrganizationId() organizationId: string, + @Res() res: Response, + @Query('download') download?: string, + ): Promise { + const url = await this.browserbaseService.getScreenshotRedirectUrl({ + runId, + organizationId, + download: download === 'true' || download === '1', + }); + res.redirect(302, url); + } } diff --git a/apps/api/src/browserbase/browserbase.service.spec.ts b/apps/api/src/browserbase/browserbase.service.spec.ts new file mode 100644 index 0000000000..5a06feaf39 --- /dev/null +++ b/apps/api/src/browserbase/browserbase.service.spec.ts @@ -0,0 +1,128 @@ +// apps/api/src/browserbase/browserbase.service.spec.ts +import { Test } from '@nestjs/testing'; +import { NotFoundException } from '@nestjs/common'; +import { BrowserbaseService } from './browserbase.service'; + +jest.mock('@db', () => ({ + db: { + browserAutomationRun: { + findUnique: jest.fn(), + }, + }, +})); + +jest.mock('@/app/s3', () => ({ + getSignedUrl: jest.fn().mockResolvedValue('https://s3.example.com/signed'), + s3Client: { send: jest.fn() }, + BUCKET_NAME: 'test-bucket', +})); + +import { db } from '@db'; +import { getSignedUrl } from '@/app/s3'; + +describe('BrowserbaseService.getScreenshotRedirectUrl', () => { + let service: BrowserbaseService; + + beforeEach(async () => { + jest.clearAllMocks(); + const moduleRef = await Test.createTestingModule({ + providers: [BrowserbaseService], + }).compile(); + service = moduleRef.get(BrowserbaseService); + }); + + it('returns a freshly minted presigned URL for an in-scope run', async () => { + (db.browserAutomationRun.findUnique as jest.Mock).mockResolvedValue({ + id: 'bar_1', + screenshotUrl: 'browser-automations/org_1/bau_1/bar_1.jpg', + automation: { task: { organizationId: 'org_1' } }, + }); + + const url = await service.getScreenshotRedirectUrl({ + runId: 'bar_1', + organizationId: 'org_1', + }); + + expect(url).toBe('https://s3.example.com/signed'); + expect(db.browserAutomationRun.findUnique).toHaveBeenCalledWith({ + where: { id: 'bar_1' }, + include: { automation: { include: { task: true } } }, + }); + }); + + it('throws NotFoundException when the run does not exist', async () => { + (db.browserAutomationRun.findUnique as jest.Mock).mockResolvedValue(null); + + await expect( + service.getScreenshotRedirectUrl({ + runId: 'bar_missing', + organizationId: 'org_1', + }), + ).rejects.toBeInstanceOf(NotFoundException); + }); + + it('throws NotFoundException when the run belongs to a different org', async () => { + (db.browserAutomationRun.findUnique as jest.Mock).mockResolvedValue({ + id: 'bar_1', + screenshotUrl: 'browser-automations/org_2/bau_1/bar_1.jpg', + automation: { task: { organizationId: 'org_2' } }, + }); + + await expect( + service.getScreenshotRedirectUrl({ + runId: 'bar_1', + organizationId: 'org_1', + }), + ).rejects.toBeInstanceOf(NotFoundException); + }); + + it('throws NotFoundException when the run has no screenshot', async () => { + (db.browserAutomationRun.findUnique as jest.Mock).mockResolvedValue({ + id: 'bar_1', + screenshotUrl: null, + automation: { task: { organizationId: 'org_1' } }, + }); + + await expect( + service.getScreenshotRedirectUrl({ + runId: 'bar_1', + organizationId: 'org_1', + }), + ).rejects.toBeInstanceOf(NotFoundException); + }); + + it('signs the URL without Content-Disposition when download is falsy', async () => { + (db.browserAutomationRun.findUnique as jest.Mock).mockResolvedValue({ + id: 'bar_1', + screenshotUrl: 'browser-automations/org_1/bau_1/bar_1.jpg', + automation: { task: { organizationId: 'org_1' } }, + }); + + await service.getScreenshotRedirectUrl({ + runId: 'bar_1', + organizationId: 'org_1', + }); + + const command = (getSignedUrl as jest.Mock).mock.calls[0][1]; + expect(command.input.ResponseContentDisposition).toBeUndefined(); + }); + + it('signs the URL with attachment Content-Disposition when download is true', async () => { + (db.browserAutomationRun.findUnique as jest.Mock).mockResolvedValue({ + id: 'bar_1', + screenshotUrl: 'browser-automations/org_1/bau_1/bar_1.jpg', + automation: { task: { organizationId: 'org_1' } }, + }); + + await service.getScreenshotRedirectUrl({ + runId: 'bar_1', + organizationId: 'org_1', + download: true, + }); + + const command = (getSignedUrl as jest.Mock).mock.calls[0][1]; + expect(command.input.ResponseContentDisposition).toBe( + 'attachment; filename="screenshot-bar_1.jpg"', + ); + }); +}); diff --git a/apps/api/src/browserbase/browserbase.service.ts b/apps/api/src/browserbase/browserbase.service.ts index 2cfb919f80..8e388fbc13 100644 --- a/apps/api/src/browserbase/browserbase.service.ts +++ b/apps/api/src/browserbase/browserbase.service.ts @@ -1,16 +1,14 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable, Logger, NotFoundException } from '@nestjs/common'; import Browserbase from '@browserbasehq/sdk'; // Lazy-imported in createStagehand() to avoid Node v25 crash // (SlowBuffer.prototype was removed — @browserbasehq/stagehand bundles buffer-equal-constant-time which uses it) type Stagehand = import('@browserbasehq/stagehand').Stagehand; import { db } from '@db'; import { z } from 'zod'; -import { - GetObjectCommand, - PutObjectCommand, - S3Client, -} from '@aws-sdk/client-s3'; -import { getSignedUrl } from '@/app/s3'; +import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { BUCKET_NAME, getSignedUrl, s3Client } from '@/app/s3'; +import { renderOverlay } from './screenshot-overlay'; +import { isNoPageError, toRunErrorMessage } from './run-error-formatter'; const BROWSER_WIDTH = 1440; const BROWSER_HEIGHT = 900; @@ -23,6 +21,13 @@ const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const PENDING_CONTEXT_ID = '__PENDING__'; +/** Empty strings become null; any actual text is trimmed and kept. */ +const normalizeCriteria = (value: string | null | undefined): string | null => { + if (value == null) return null; + const trimmed = value.trim(); + return trimmed.length === 0 ? null : trimmed; +}; + const isPrismaUniqueConstraintError = (error: unknown): boolean => { if (typeof error !== 'object' || error === null) return false; if (!('code' in error)) return false; @@ -33,14 +38,23 @@ const isPrismaUniqueConstraintError = (error: unknown): boolean => { @Injectable() export class BrowserbaseService { private readonly logger = new Logger(BrowserbaseService.name); - private readonly s3Client: S3Client; - private readonly bucketName: string; - constructor() { - this.s3Client = new S3Client({ - region: process.env.AWS_REGION || 'us-east-1', - }); - this.bucketName = process.env.APP_AWS_BUCKET_NAME || 'comp-attachments'; + private get s3Client(): S3Client { + if (!s3Client) { + throw new Error( + 'S3 client not configured — set APP_AWS_ACCESS_KEY_ID, APP_AWS_SECRET_ACCESS_KEY, APP_AWS_REGION, APP_AWS_BUCKET_NAME in apps/api/.env', + ); + } + return s3Client; + } + + private get bucketName(): string { + if (!BUCKET_NAME) { + throw new Error( + 'APP_AWS_BUCKET_NAME is not set — configure S3 credentials in apps/api/.env', + ); + } + return BUCKET_NAME; } private getBrowserbase() { @@ -319,12 +333,7 @@ export class BrowserbaseService { username: result.username, }; } catch (err) { - const message = err instanceof Error ? err.message : String(err); - const isNoPage = - message.includes('awaitActivePage') || - message.includes('no page available') || - message.includes('No page found'); - if (isNoPage) { + if (isNoPageError(err)) { throw new Error( 'Browser session ended before we could verify login status. Please retry.', ); @@ -343,6 +352,7 @@ export class BrowserbaseService { description?: string; targetUrl: string; instruction: string; + evaluationCriteria?: string; schedule?: string; }) { return db.browserAutomation.create({ @@ -352,6 +362,7 @@ export class BrowserbaseService { description: data.description, targetUrl: data.targetUrl, instruction: data.instruction, + evaluationCriteria: normalizeCriteria(data.evaluationCriteria), schedule: data.schedule, isEnabled: true, // Enable by default so scheduled runs work }, @@ -390,13 +401,20 @@ export class BrowserbaseService { description?: string; targetUrl?: string; instruction?: string; + evaluationCriteria?: string; schedule?: string; isEnabled?: boolean; }, ) { + const { evaluationCriteria, ...rest } = data; return db.browserAutomation.update({ where: { id: automationId }, - data, + data: { + ...rest, + ...(evaluationCriteria !== undefined + ? { evaluationCriteria: normalizeCriteria(evaluationCriteria) } + : {}), + }, }); } @@ -506,6 +524,7 @@ export class BrowserbaseService { { title: automation.task.title, description: automation.task.description, + evaluationCriteria: automation.evaluationCriteria, }, ); @@ -534,7 +553,7 @@ export class BrowserbaseService { }; } - // Upload screenshot to S3 (only taken if evaluation passed) + // Upload screenshot to S3 let screenshotKey: string | undefined; let presignedUrl: string | undefined; if (result.screenshot) { @@ -547,7 +566,8 @@ export class BrowserbaseService { presignedUrl = await this.getPresignedUrl(screenshotKey); } - // Update run as completed + // Update run as completed. Only persist an evaluation verdict when + // the caller's automation had criteria configured. await db.browserAutomationRun.update({ where: { id: runId }, data: { @@ -556,7 +576,7 @@ export class BrowserbaseService { durationMs: run.startedAt ? Date.now() - run.startedAt.getTime() : 0, screenshotUrl: screenshotKey, evaluationStatus: result.evaluationStatus ?? null, - evaluationReason: result.evaluationReason ?? 'Screenshot captured', + evaluationReason: result.evaluationReason ?? null, }, }); @@ -568,6 +588,7 @@ export class BrowserbaseService { }; } catch (err) { this.logger.error('Failed to execute automation on session', err); + const { userFacing, needsReauth } = toRunErrorMessage(err); await db.browserAutomationRun.update({ where: { id: runId }, @@ -575,13 +596,14 @@ export class BrowserbaseService { status: 'failed', completedAt: new Date(), durationMs: run.startedAt ? Date.now() - run.startedAt.getTime() : 0, - error: err instanceof Error ? err.message : 'Unknown error', + error: userFacing, }, }); return { success: false, - error: err instanceof Error ? err.message : 'Unknown error', + error: userFacing, + needsReauth: needsReauth ? true : undefined, }; } } @@ -646,6 +668,7 @@ export class BrowserbaseService { { title: automation.task.title, description: automation.task.description, + evaluationCriteria: automation.evaluationCriteria, }, ); @@ -686,7 +709,8 @@ export class BrowserbaseService { presignedUrl = await this.getPresignedUrl(screenshotKey); } - // Update run as completed + // Update run as completed. Only persist an evaluation verdict when + // the automation had criteria configured. await db.browserAutomationRun.update({ where: { id: run.id }, data: { @@ -695,7 +719,7 @@ export class BrowserbaseService { durationMs: Date.now() - run.startedAt!.getTime(), screenshotUrl: screenshotKey, evaluationStatus: result.evaluationStatus ?? null, - evaluationReason: result.evaluationReason ?? 'Screenshot captured', + evaluationReason: result.evaluationReason ?? null, }, }); @@ -719,6 +743,7 @@ export class BrowserbaseService { } } catch (err) { this.logger.error('Failed to run browser automation', err); + const { userFacing, needsReauth } = toRunErrorMessage(err); // Update run as failed await db.browserAutomationRun.update({ @@ -727,14 +752,15 @@ export class BrowserbaseService { status: 'failed', completedAt: new Date(), durationMs: run.startedAt ? Date.now() - run.startedAt.getTime() : 0, - error: err instanceof Error ? err.message : 'Unknown error', + error: userFacing, }, }); return { runId: run.id, success: false, - error: err instanceof Error ? err.message : 'Unknown error', + error: userFacing, + needsReauth: needsReauth ? true : undefined, }; } } @@ -743,7 +769,11 @@ export class BrowserbaseService { sessionId: string, targetUrl: string, instruction: string, - taskContext?: { title: string; description?: string | null }, + taskContext?: { + title: string; + description?: string | null; + evaluationCriteria?: string | null; + }, ): Promise<{ success: boolean; screenshot?: string; @@ -802,32 +832,71 @@ export class BrowserbaseService { // Always take a screenshot at the end (no pass/fail criteria gate) page = await this.ensureActivePage(stagehand); - const screenshot = await page.screenshot({ + const sourceUrl = page.url(); + const rawScreenshot = await page.screenshot({ type: 'jpeg', quality: 80, fullPage: false, }); + let finalBuffer: Buffer = rawScreenshot; + try { + finalBuffer = await renderOverlay({ + buffer: rawScreenshot, + instruction, + sourceUrl, + capturedAt: new Date(), + }); + } catch (overlayErr) { + this.logger.warn('Screenshot overlay render failed; uploading raw image', { + error: + overlayErr instanceof Error ? overlayErr.message : String(overlayErr), + }); + } + + // Optional evaluation: if the automation was configured with + // natural-language criteria, ask Stagehand to inspect the page and + // produce a pass/fail verdict with a short reason. + let evaluationStatus: 'pass' | 'fail' | undefined; + let evaluationReason: string | undefined; + const criteria = taskContext?.evaluationCriteria?.trim(); + if (criteria) { + try { + const evalSchema = z.object({ + pass: z.boolean(), + reason: z.string(), + }); + const evaluation = (await stagehand.extract( + `You are an auditor reviewing the current page after an automation has finished navigating. Decide whether the page clearly satisfies this criteria. Only return pass=true if the evidence is unambiguously present and visible. If it is ambiguous, missing, or contradicted, return pass=false. Always provide a short reason (max 220 characters).\n\nCriteria: ${criteria}`, + evalSchema as any, + )) as { pass: boolean; reason: string }; + + evaluationStatus = evaluation.pass ? 'pass' : 'fail'; + evaluationReason = evaluation.reason; + } catch (evalErr) { + this.logger.warn( + 'Evaluation step failed; returning screenshot without verdict', + { + error: + evalErr instanceof Error ? evalErr.message : String(evalErr), + }, + ); + } + } + return { success: true, - screenshot: screenshot.toString('base64'), - evaluationReason: taskContext - ? `Navigation completed for "${taskContext.title}". Screenshot captured.` - : 'Navigation completed. Screenshot captured.', + screenshot: finalBuffer.toString('base64'), + evaluationStatus, + evaluationReason, }; } catch (err) { this.logger.error('Failed to execute automation', err); - const message = err instanceof Error ? err.message : String(err); - const isNoPage = - message.includes('awaitActivePage') || - message.includes('no page available') || - message.includes('No page found'); + const { userFacing, needsReauth } = toRunErrorMessage(err); return { success: false, - needsReauth: isNoPage ? true : undefined, - error: isNoPage - ? 'Browser session ended before we could capture evidence. Please retry.' - : message, + needsReauth: needsReauth ? true : undefined, + error: userFacing, }; } finally { await this.safeCloseStagehand(stagehand); @@ -856,12 +925,57 @@ export class BrowserbaseService { return key; } - async getPresignedUrl(key: string, expiresIn = 3600): Promise { + async getPresignedUrl( + key: string, + options: { expiresIn?: number; responseContentDisposition?: string } = {}, + ): Promise { const command = new GetObjectCommand({ Bucket: this.bucketName, Key: key, + ResponseContentDisposition: options.responseContentDisposition, + }); + return getSignedUrl(this.s3Client, command, { + expiresIn: options.expiresIn ?? 3600, + }); + } + + /** + * Resolve a run's S3 screenshot key to a freshly signed presigned URL, + * scoped to the caller's organization. Used by the controller's + * GET runs/:runId/screenshot redirect endpoint so that the "Open full size" + * UI link never serves an expired URL. + * + * When `download` is true, the presigned URL is signed with an + * attachment Content-Disposition so the browser downloads the image + * instead of rendering it inline. + */ + async getScreenshotRedirectUrl(input: { + runId: string; + organizationId: string; + download?: boolean; + }): Promise { + const { runId, organizationId, download } = input; + + const run = await db.browserAutomationRun.findUnique({ + where: { id: runId }, + include: { automation: { include: { task: true } } }, + }); + + if (!run || !run.screenshotUrl) { + throw new NotFoundException('Screenshot not found'); + } + + if (run.automation.task.organizationId !== organizationId) { + throw new NotFoundException('Screenshot not found'); + } + + const responseContentDisposition = download + ? `attachment; filename="screenshot-${runId}.jpg"` + : undefined; + + return this.getPresignedUrl(run.screenshotUrl, { + responseContentDisposition, }); - return getSignedUrl(this.s3Client, command, { expiresIn }); } async getRunWithPresignedUrl(runId: string) { diff --git a/apps/api/src/browserbase/dto/browserbase.dto.ts b/apps/api/src/browserbase/dto/browserbase.dto.ts index f0bfffbf49..00b637cf20 100644 --- a/apps/api/src/browserbase/dto/browserbase.dto.ts +++ b/apps/api/src/browserbase/dto/browserbase.dto.ts @@ -82,6 +82,14 @@ export class CreateBrowserAutomationDto { @IsNotEmpty() instruction: string; + @ApiPropertyOptional({ + description: + 'Optional natural-language criteria used to evaluate the automation result. When set, the run gets a pass/fail verdict.', + }) + @IsString() + @IsOptional() + evaluationCriteria?: string; + @ApiPropertyOptional({ description: 'Cron schedule expression' }) @IsString() @IsOptional() @@ -111,6 +119,14 @@ export class UpdateBrowserAutomationDto { @IsOptional() instruction?: string; + @ApiPropertyOptional({ + description: + 'Optional natural-language criteria used to evaluate the automation result. Pass an empty string to clear.', + }) + @IsString() + @IsOptional() + evaluationCriteria?: string; + @ApiPropertyOptional({ description: 'Cron schedule expression' }) @IsString() @IsOptional() diff --git a/apps/api/src/browserbase/run-error-formatter.spec.ts b/apps/api/src/browserbase/run-error-formatter.spec.ts new file mode 100644 index 0000000000..657f790203 --- /dev/null +++ b/apps/api/src/browserbase/run-error-formatter.spec.ts @@ -0,0 +1,64 @@ +import { isNoPageError, toRunErrorMessage } from './run-error-formatter'; + +describe('toRunErrorMessage', () => { + it('classifies an isNoPage error with needsReauth=true', () => { + const err = new Error('No Page found for awaitActivePage'); + const result = toRunErrorMessage(err); + expect(result.needsReauth).toBe(true); + expect(result.userFacing).toContain('Browser session ended'); + }); + + it('classifies a timeout error without needsReauth', () => { + const err = new Error('operation timed out after 30s'); + const result = toRunErrorMessage(err); + expect(result.needsReauth).toBe(false); + expect(result.userFacing).toContain('timed out'); + }); + + it('uppercase Timeout also matches', () => { + const err = new Error('Timeout exceeded'); + const result = toRunErrorMessage(err); + expect(result.userFacing).toContain('timed out'); + }); + + it('falls back to generic message for other errors', () => { + const err = new Error('some very specific Stagehand crash stack'); + const result = toRunErrorMessage(err); + expect(result.needsReauth).toBe(false); + expect(result.userFacing).toBe( + 'Automation failed to complete. Please retry — see run error details for specifics.', + ); + expect(result.userFacing).not.toContain('Stagehand'); + }); + + it('handles non-Error values (string, unknown)', () => { + expect(toRunErrorMessage('raw string').userFacing).toBe( + 'Automation failed to complete. Please retry — see run error details for specifics.', + ); + expect(toRunErrorMessage({ nested: 'object' }).userFacing).toBe( + 'Automation failed to complete. Please retry — see run error details for specifics.', + ); + expect(toRunErrorMessage(null).userFacing).toBe( + 'Automation failed to complete. Please retry — see run error details for specifics.', + ); + }); +}); + +describe('isNoPageError', () => { + it('returns true for awaitActivePage', () => { + expect(isNoPageError(new Error('awaitActivePage: no page available'))).toBe(true); + }); + + it('returns true for No page found', () => { + expect(isNoPageError(new Error('No page found for stagehand'))).toBe(true); + }); + + it('returns false for an unrelated error', () => { + expect(isNoPageError(new Error('network reset'))).toBe(false); + }); + + it('returns false for non-Error values', () => { + expect(isNoPageError('no page available' as unknown)).toBe(false); + expect(isNoPageError(null)).toBe(false); + }); +}); diff --git a/apps/api/src/browserbase/run-error-formatter.ts b/apps/api/src/browserbase/run-error-formatter.ts new file mode 100644 index 0000000000..88262e6e0e --- /dev/null +++ b/apps/api/src/browserbase/run-error-formatter.ts @@ -0,0 +1,49 @@ +export interface RunErrorMessage { + userFacing: string; + needsReauth: boolean; +} + +const SESSION_ENDED_MESSAGE = + 'Browser session ended before we could capture evidence. Please retry.'; +const TIMEOUT_MESSAGE = + 'Automation timed out before completing. Please retry — if this keeps happening, simplify the instruction or check the target site.'; +const GENERIC_MESSAGE = + 'Automation failed to complete. Please retry — see run error details for specifics.'; + +/** + * Check whether an error was thrown because Browserbase/Stagehand's active page + * went away (session closed, page navigated, CDP died). These strings are tied + * to upstream SDK error text — if the SDK renames them, this predicate needs + * to be updated and callers may silently fall through to generic handling. + */ +export function isNoPageError(err: unknown): boolean { + const message = err instanceof Error ? err.message : ''; + return ( + message.includes('awaitActivePage') || + message.includes('no page available') || + message.includes('No page found') + ); +} + +/** + * Translate a thrown error into a short, user-readable message for the + * BrowserAutomationRun.error field. The raw error is still logged upstream + * for server-side debugging. + */ +export function toRunErrorMessage(err: unknown): RunErrorMessage { + if (isNoPageError(err)) { + return { userFacing: SESSION_ENDED_MESSAGE, needsReauth: true }; + } + + const message = err instanceof Error ? err.message : ''; + const isTimeout = + message.includes('timeout') || + message.includes('Timeout') || + message.includes('timed out'); + + if (isTimeout) { + return { userFacing: TIMEOUT_MESSAGE, needsReauth: false }; + } + + return { userFacing: GENERIC_MESSAGE, needsReauth: false }; +} diff --git a/apps/api/src/browserbase/screenshot-overlay.spec.ts b/apps/api/src/browserbase/screenshot-overlay.spec.ts new file mode 100644 index 0000000000..65c020c0d6 --- /dev/null +++ b/apps/api/src/browserbase/screenshot-overlay.spec.ts @@ -0,0 +1,106 @@ +// apps/api/src/browserbase/screenshot-overlay.spec.ts +import sharp from 'sharp'; +import { renderOverlay, OVERLAY_HEIGHT_PX } from './screenshot-overlay'; + +describe('renderOverlay', () => { + const makeSolidJpeg = async (width = 800, height = 600) => { + return sharp({ + create: { + width, + height, + channels: 3, + background: { r: 240, g: 240, b: 240 }, + }, + }) + .jpeg({ quality: 80 }) + .toBuffer(); + }; + + it('adds a bottom banner that increases image height by OVERLAY_HEIGHT_PX', async () => { + const input = await makeSolidJpeg(800, 600); + const out = await renderOverlay({ + buffer: input, + instruction: 'Verify MFA is enforced', + sourceUrl: 'https://github.com/settings/security', + capturedAt: new Date('2026-04-22T14:32:07Z'), + }); + const meta = await sharp(out).metadata(); + expect(meta.width).toBe(800); + expect(meta.height).toBe(600 + OVERLAY_HEIGHT_PX); + expect(meta.format).toBe('jpeg'); + }); + + it('preserves non-800 widths (narrow image)', async () => { + const input = await makeSolidJpeg(400, 300); + const out = await renderOverlay({ + buffer: input, + instruction: 'Check', + sourceUrl: 'https://example.com', + capturedAt: new Date('2026-04-22T14:32:07Z'), + }); + const meta = await sharp(out).metadata(); + expect(meta.width).toBe(400); + expect(meta.height).toBe(300 + OVERLAY_HEIGHT_PX); + }); + + it('preserves wide widths (4000px)', async () => { + const input = await makeSolidJpeg(4000, 1200); + const out = await renderOverlay({ + buffer: input, + instruction: 'Check', + sourceUrl: 'https://example.com', + capturedAt: new Date('2026-04-22T14:32:07Z'), + }); + const meta = await sharp(out).metadata(); + expect(meta.width).toBe(4000); + expect(meta.height).toBe(1200 + OVERLAY_HEIGHT_PX); + }); + + it('paints a dark banner on the bottom (top-left pixel is light source color; bottom-center is dark banner)', async () => { + const input = await makeSolidJpeg(800, 600); + const out = await renderOverlay({ + buffer: input, + instruction: 'Check', + sourceUrl: 'https://example.com', + capturedAt: new Date('2026-04-22T14:32:07Z'), + }); + const raw = await sharp(out).raw().toBuffer({ resolveWithObject: true }); + const { data, info } = raw; + // Top-left pixel: source color, ~240 + const topLeft = { r: data[0], g: data[1], b: data[2] }; + expect(topLeft.r).toBeGreaterThan(200); + // Bottom-center pixel (in the banner region): dark + const bottomRow = info.height - Math.floor(OVERLAY_HEIGHT_PX / 2); + const midCol = Math.floor(info.width / 2); + const idx = (bottomRow * info.width + midCol) * info.channels; + const bottomMid = { r: data[idx], g: data[idx + 1], b: data[idx + 2] }; + expect(bottomMid.r).toBeLessThan(40); + expect(bottomMid.g).toBeLessThan(40); + expect(bottomMid.b).toBeLessThan(40); + }); + + it('truncates very long instruction text without throwing', async () => { + const input = await makeSolidJpeg(800, 600); + const longInstruction = 'a'.repeat(500); + const out = await renderOverlay({ + buffer: input, + instruction: longInstruction, + sourceUrl: 'https://example.com', + capturedAt: new Date('2026-04-22T14:32:07Z'), + }); + const meta = await sharp(out).metadata(); + expect(meta.height).toBe(600 + OVERLAY_HEIGHT_PX); + }); + + it('handles unicode in instruction and URL', async () => { + const input = await makeSolidJpeg(800, 600); + const out = await renderOverlay({ + buffer: input, + instruction: 'Vérifier MFA — 🔐', + sourceUrl: 'https://exämple.com/café', + capturedAt: new Date('2026-04-22T14:32:07Z'), + }); + const meta = await sharp(out).metadata(); + expect(meta.height).toBe(600 + OVERLAY_HEIGHT_PX); + }); +}); diff --git a/apps/api/src/browserbase/screenshot-overlay.ts b/apps/api/src/browserbase/screenshot-overlay.ts new file mode 100644 index 0000000000..27d09b3c80 --- /dev/null +++ b/apps/api/src/browserbase/screenshot-overlay.ts @@ -0,0 +1,163 @@ +// apps/api/src/browserbase/screenshot-overlay.ts +import sharp from 'sharp'; + +export const OVERLAY_HEIGHT_PX = 88; +const OVERLAY_BG = '#0A0A0A'; +const OVERLAY_SURFACE = '#111113'; +const OVERLAY_BORDER = '#1F1F23'; +const OVERLAY_TEXT = '#FAFAFA'; +const OVERLAY_MUTED = '#8B8B92'; +const OVERLAY_ACCENT = '#22C55E'; +const BRAND_PILLAR_WIDTH = 168; +const MAX_INSTRUCTION_CHARS = 120; +const MAX_URL_CHARS = 140; + +// Comp AI hex logo mark, viewBox 0 0 56 56. Fill is overridden at render time. +const COMP_LOGO_PATH = + 'M41 13.3327L39.3682 12.16L28.5853 4.41866C28.2368 4.16845 27.7675 4.16844 27.419 4.41863L2.41685 22.3661C2.15517 22.5539 2 22.8563 2 23.1784V32.8194C2 33.1415 2.15515 33.4439 2.41681 33.6317L27.4189 51.5813C27.7675 51.8315 28.2368 51.8315 28.5854 51.5812L53.5833 33.6317C53.8449 33.4439 54 33.1415 54 32.8194V23.1784C54 22.8563 53.8448 22.5539 53.5832 22.3661L41 13.3327ZM27.419 9.11825C27.7675 8.86804 28.2368 8.86802 28.5854 9.11822L34.9638 13.6969C35.5198 14.096 35.5195 14.9232 34.9633 15.322L31.9378 17.4913C31.7156 17.6504 31.4167 17.6502 31.1947 17.4908L28.5853 15.6178C28.2368 15.3676 27.7675 15.3676 27.419 15.6178L18.7338 21.8529C18.178 22.2519 18.178 23.0787 18.7339 23.4776L21.1661 25.2235L24.4382 27.5755L27.4188 29.7149C27.7674 29.9651 28.2368 29.965 28.5854 29.7146L37.2698 23.4751C37.8251 23.0761 37.8252 22.25 37.2699 21.8509L35.2116 20.3717C35.0294 20.2407 35.0296 19.9695 35.2121 19.8389L38.7873 17.2754C39.1358 17.0255 39.6049 17.0257 39.9532 17.2758L46.3287 21.8531C46.8844 22.252 46.8844 23.0786 46.3288 23.4777L43.3017 25.6513L28.5875 36.2167C28.239 36.467 27.7696 36.467 27.4211 36.2168L19.9104 30.8253L16.6382 28.4777L12.7026 25.6535L9.67725 23.48C9.12177 23.0809 9.12192 22.2544 9.67753 21.8555L27.419 9.11825Z'; + +export interface RenderOverlayInput { + buffer: Buffer; + instruction: string; + sourceUrl: string; + capturedAt: Date; +} + +/** + * Composite an audit metadata banner onto the bottom of a screenshot. + * The banner adds OVERLAY_HEIGHT_PX to the total image height. + * Failure-mode contract: throws on malformed input; callers should handle + * and fall back to the raw image. + */ +export async function renderOverlay(input: RenderOverlayInput): Promise { + const { buffer, instruction, sourceUrl, capturedAt } = input; + + const sourceMeta = await sharp(buffer).metadata(); + const width = sourceMeta.width; + const height = sourceMeta.height; + if (!width) { + throw new Error('renderOverlay: source image has no width'); + } + if (!height) { + throw new Error('renderOverlay: source image has no height'); + } + + const instructionText = truncate(instruction, MAX_INSTRUCTION_CHARS); + const sourceUrlText = truncate(sourceUrl, MAX_URL_CHARS); + const timestampText = formatUtc(capturedAt); + + const bannerSvg = buildBannerSvg({ + width, + height: OVERLAY_HEIGHT_PX, + instruction: instructionText, + sourceUrl: sourceUrlText, + timestamp: timestampText, + }); + + const bannerBuffer = await sharp(Buffer.from(bannerSvg)) + .png() + .toBuffer(); + + const extended = await sharp(buffer) + .extend({ + bottom: OVERLAY_HEIGHT_PX, + background: OVERLAY_BG, + }) + .composite([ + { + input: bannerBuffer, + top: height, + left: 0, + }, + ]) + .jpeg({ quality: 85 }) + .toBuffer(); + + return extended; +} + +function truncate(value: string, max: number): string { + if (value.length <= max) return value; + return value.slice(0, max - 1) + '…'; +} + +function formatUtc(date: Date): string { + const pad = (n: number) => String(n).padStart(2, '0'); + const y = date.getUTCFullYear(); + const m = pad(date.getUTCMonth() + 1); + const d = pad(date.getUTCDate()); + const hh = pad(date.getUTCHours()); + const mm = pad(date.getUTCMinutes()); + const ss = pad(date.getUTCSeconds()); + return `${y}-${m}-${d} ${hh}:${mm}:${ss} UTC`; +} + +function escapeXml(value: string): string { + // Strip XML 1.0 illegal control chars (keep tab 0x09, LF 0x0A, CR 0x0D) + const stripped = value.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, ''); + return stripped + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +interface BannerArgs { + width: number; + height: number; + instruction: string; + sourceUrl: string; + timestamp: string; +} + +function buildBannerSvg(args: BannerArgs): string { + const { width, height, instruction, sourceUrl, timestamp } = args; + const rowFontSize = 12; + const labelFontSize = 9; + // Explicit Linux-safe font stack. The production container only ships + // DejaVu Sans (via fonts-dejavu-core); Apple/Segoe/Roboto aren't available + // so librsvg would render .notdef glyphs ("□ □ □") on servers. + const fontFamily = '"DejaVu Sans", sans-serif'; + + const w = Math.floor(width); + const h = Math.floor(height); + const pillarW = Math.min(BRAND_PILLAR_WIDTH, Math.floor(w / 3)); + const infoX = pillarW + 20; + const accentStripeH = 2; + const logoSize = 28; + const logoX = 18; + const logoY = Math.floor((h - accentStripeH - logoSize) / 2); + const brandTextX = logoX + logoSize + 12; + const brandTextY = logoY + 14; + const brandTaglineY = logoY + 27; + + return ` + + + + + + + + + + Comp AI + AUDIT TRAIL + + + + REQUIREMENT + ${escapeXml(instruction)} + + + CAPTURED + ${escapeXml(timestamp)} + + + SOURCE + ${escapeXml(sourceUrl)} + + +`; +} diff --git a/apps/app/src/app/(app)/[orgId]/tasks/[taskId]/components/browser-automations/BrowserAutomationConfigDialog.tsx b/apps/app/src/app/(app)/[orgId]/tasks/[taskId]/components/browser-automations/BrowserAutomationConfigDialog.tsx index a71c038dee..07dc6ee42e 100644 --- a/apps/app/src/app/(app)/[orgId]/tasks/[taskId]/components/browser-automations/BrowserAutomationConfigDialog.tsx +++ b/apps/app/src/app/(app)/[orgId]/tasks/[taskId]/components/browser-automations/BrowserAutomationConfigDialog.tsx @@ -23,6 +23,7 @@ const automationConfigSchema = z.object({ name: z.string().trim().min(1, { message: 'Name is required' }), targetUrl: z.string().trim().url({ message: 'Starting URL must be a valid URL' }), instruction: z.string().trim().min(1, { message: 'Instruction is required' }), + evaluationCriteria: z.string().trim().optional(), }); type AutomationConfigFormData = z.infer; @@ -30,7 +31,10 @@ type AutomationConfigFormData = z.infer; interface BrowserAutomationConfigDialogProps { isOpen: boolean; mode: 'create' | 'edit'; - initialValues?: Pick; + initialValues?: Pick< + BrowserAutomation, + 'id' | 'name' | 'targetUrl' | 'instruction' | 'evaluationCriteria' + >; isSaving: boolean; onClose: () => void; onCreate: (data: AutomationConfigFormData) => Promise; @@ -57,6 +61,7 @@ export function BrowserAutomationConfigDialog({ name: '', targetUrl: '', instruction: '', + evaluationCriteria: '', }, }); @@ -68,15 +73,16 @@ export function BrowserAutomationConfigDialog({ name: initialValues.name ?? '', targetUrl: initialValues.targetUrl ?? '', instruction: initialValues.instruction ?? '', + evaluationCriteria: initialValues.evaluationCriteria ?? '', }); return; } - reset({ name: '', targetUrl: '', instruction: '' }); + reset({ name: '', targetUrl: '', instruction: '', evaluationCriteria: '' }); }, [isOpen, mode, initialValues, reset]); const handleClose = () => { - reset({ name: '', targetUrl: '', instruction: '' }); + reset({ name: '', targetUrl: '', instruction: '', evaluationCriteria: '' }); onClose(); }; @@ -157,6 +163,23 @@ export function BrowserAutomationConfigDialog({ )} +
+ +