From 9a4aadda2fad795a32b5a177c1f6c201f74ccbc9 Mon Sep 17 00:00:00 2001
From: Aiden Bai <aiden.bai05@gmail.com>
Date: Sat, 4 Apr 2026 21:24:02 -0700
Subject: [PATCH 1/3] feat: add video-transcript prototype for generating tests
 from screen recordings
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds apps/video-transcript — a standalone CLI that extracts structured
interaction transcripts from screen recordings using Gemini 2.5 Flash
via AI SDK. Designed to compound with git diff context so the test
agent gets both "what changed" and "how the feature works."

Pipeline: video → ffmpeg idle-time cutting → Gemini transcript extraction

Updates the video-transcript spec to use AI SDK (@ai-sdk/google) instead
of @google/genai, matching the existing codebase dependency surface.
---
 .specs/video-transcript.md                    |   2 +-
 apps/video-transcript/package.json            |  42 ++++
 .../video-transcript/src/activity-analyzer.ts | 209 ++++++++++++++++++
 apps/video-transcript/src/constants.ts        |  14 ++
 .../src/extract-transcript.ts                 |  36 +++
 apps/video-transcript/src/index.ts            | 128 +++++++++++
 .../video-transcript/src/transcript-prompt.ts |  38 ++++
 apps/video-transcript/src/types.ts            |   7 +
 apps/video-transcript/tsconfig.json           |   9 +
 pnpm-lock.yaml                                |  83 +++++++
 10 files changed, 567 insertions(+), 1 deletion(-)
 create mode 100644 apps/video-transcript/package.json
 create mode 100644 apps/video-transcript/src/activity-analyzer.ts
 create mode 100644 apps/video-transcript/src/constants.ts
 create mode 100644 apps/video-transcript/src/extract-transcript.ts
 create mode 100644 apps/video-transcript/src/index.ts
 create mode 100644 apps/video-transcript/src/transcript-prompt.ts
 create mode 100644 apps/video-transcript/src/types.ts
 create mode 100644 apps/video-transcript/tsconfig.json

diff --git a/.specs/video-transcript.md b/.specs/video-transcript.md
index 7db25f972..eeacff58d 100644
--- a/.specs/video-transcript.md
+++ b/.specs/video-transcript.md
@@ -81,7 +81,7 @@ const { text } = await generateText({
   messages: [{
     role: "user",
     content: [
-      { type: "file", data: readFileSync(videoPath), mimeType: "video/mp4" },
+      { type: "file", data: readFileSync(videoPath), mediaType: "video/mp4" },
       { type: "text", text: transcriptPrompt },
     ],
   }],
diff --git a/apps/video-transcript/package.json b/apps/video-transcript/package.json
new file mode 100644
index 000000000..7082286be
--- /dev/null
+++ b/apps/video-transcript/package.json
@@ -0,0 +1,42 @@
+{
+  "name": "@expect/video-transcript",
+  "version": "0.0.1",
+  "private": true,
+  "description": "Extract structured interaction transcripts from screen recordings",
+  "type": "module",
+  "bin": {
+    "video-transcript": "./dist/index.js"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "vp pack",
+    "dev": "vp pack --watch",
+    "lint": "vp lint && tsc --noEmit",
+    "format": "vp fmt",
+    "format:check": "vp fmt --check",
+    "check": "vp check",
+    "test": "vp test",
+    "typecheck": "tsgo --noEmit"
+  },
+  "dependencies": {
+    "@ai-sdk/google": "^3.0.53",
+    "ai": "^6.0.146",
+    "commander": "^13.1.0",
+    "effect": "4.0.0-beta.35",
+    "picocolors": "^1.1.1"
+  },
+  "devDependencies": {
+    "@types/node": "^22.15.0",
+    "typescript": "^5.7.0"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}
diff --git a/apps/video-transcript/src/activity-analyzer.ts b/apps/video-transcript/src/activity-analyzer.ts
new file mode 100644
index 000000000..3b742ff61
--- /dev/null
+++ b/apps/video-transcript/src/activity-analyzer.ts
@@ -0,0 +1,209 @@
+import { execFile } from "node:child_process";
+import { readFileSync, mkdtempSync, rmSync, readdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import {
+  FRAME_DIFF_IDLE_THRESHOLD,
+  FRAMES_PER_SECOND,
+  IDLE_CUT_THRESHOLD_SECONDS,
+  MIN_ACTIVE_SEGMENT_SECONDS,
+  SCENE_CHANGE_THRESHOLD,
+} from "./constants";
+import type { ActivitySegment, ActivityTimeline } from "./types";
+
+const execFileAsync = (
+  command: string,
+  args: readonly string[],
+): Promise<{ stdout: string; stderr: string }> =>
+  new Promise((resolve, reject) => {
+    execFile(command, args, { maxBuffer: 10 * 1024 * 1024 }, (error, stdout, stderr) => {
+      if (error) reject(error);
+      else resolve({ stdout, stderr });
+    });
+  });
+
+export const checkFfmpegAvailable = async (): Promise<boolean> => {
+  try {
+    await execFileAsync("ffmpeg", ["-version"]);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+const extractFrames = async (videoPath: string, outputDir: string): Promise<number> => {
+  await execFileAsync("ffmpeg", [
+    "-i",
+    videoPath,
+    "-vf",
+    `fps=${FRAMES_PER_SECOND}`,
+    "-vsync",
+    "vfr",
+    "-f",
+    "rawvideo",
+    "-pix_fmt",
+    "gray",
+    "-s",
+    "320x180",
+    path.join(outputDir, "frame_%05d.raw"),
+  ]);
+
+  const files = readdirSync(outputDir).filter((file) => file.startsWith("frame_"));
+  return files.length;
+};
+
+const computeFrameDiff = (frameA: Buffer, frameB: Buffer): number => {
+  const length = Math.min(frameA.length, frameB.length);
+  if (length === 0) return 0;
+
+  let totalDiff = 0;
+  for (let index = 0; index < length; index++) {
+    totalDiff += Math.abs(frameA[index]! - frameB[index]!) / 255;
+  }
+
+  return totalDiff / length;
+};
+
+const classifySegments = (diffs: readonly number[]): ActivityTimeline => {
+  const rawClassification: Array<"active" | "idle" | "scene_change"> = [];
+
+  for (const diff of diffs) {
+    if (diff >= SCENE_CHANGE_THRESHOLD) {
+      rawClassification.push("scene_change");
+    } else if (diff > FRAME_DIFF_IDLE_THRESHOLD) {
+      rawClassification.push("active");
+    } else {
+      rawClassification.push("idle");
+    }
+  }
+
+  const segments: ActivitySegment[] = [];
+  let currentType = rawClassification[0];
+  let segmentStart = 0;
+
+  if (!currentType) return [];
+
+  for (let index = 1; index <= rawClassification.length; index++) {
+    const nextType = rawClassification[index];
+    if (nextType !== currentType || index === rawClassification.length) {
+      segments.push({
+        type: currentType,
+        startSeconds: segmentStart,
+        endSeconds: index,
+      });
+      if (nextType) {
+        currentType = nextType;
+        segmentStart = index;
+      }
+    }
+  }
+
+  return mergeShortSegments(segments);
+};
+
+const mergeShortSegments = (segments: readonly ActivitySegment[]): ActivityTimeline => {
+  const merged: ActivitySegment[] = [];
+
+  for (const segment of segments) {
+    const duration = segment.endSeconds - segment.startSeconds;
+
+    if (segment.type === "active" && duration < MIN_ACTIVE_SEGMENT_SECONDS) {
+      const previous = merged[merged.length - 1];
+      if (previous && previous.type === "idle") {
+        merged[merged.length - 1] = { ...previous, endSeconds: segment.endSeconds };
+      } else {
+        merged.push(segment);
+      }
+      continue;
+    }
+
+    if (segment.type === "idle" && duration <= IDLE_CUT_THRESHOLD_SECONDS) {
+      const previous = merged[merged.length - 1];
+      if (previous && previous.type === "active") {
+        merged[merged.length - 1] = { ...previous, endSeconds: segment.endSeconds };
+        continue;
+      }
+    }
+
+    merged.push(segment);
+  }
+
+  return merged;
+};
+
+export const analyzeActivity = async (videoPath: string): Promise<ActivityTimeline> => {
+  const framesDir = mkdtempSync(path.join(tmpdir(), "expect-frames-"));
+
+  try {
+    const frameCount = await extractFrames(videoPath, framesDir);
+    if (frameCount < 2) return [{ type: "active", startSeconds: 0, endSeconds: frameCount }];
+
+    const frameSize = 320 * 180;
+    const diffs: number[] = [];
+
+    for (let index = 1; index < frameCount; index++) {
+      const prevPath = path.join(framesDir, `frame_${String(index).padStart(5, "0")}.raw`);
+      const currPath = path.join(framesDir, `frame_${String(index + 1).padStart(5, "0")}.raw`);
+
+      const prevFrame = readFileSync(prevPath);
+      const currFrame = readFileSync(currPath);
+      diffs.push(computeFrameDiff(prevFrame, currFrame));
+    }
+
+    return classifySegments(diffs);
+  } finally {
+    rmSync(framesDir, { recursive: true, force: true });
+  }
+};
+
+export const buildTrimmedVideo = async (
+  videoPath: string,
+  timeline: ActivityTimeline,
+): Promise<string> => {
+  const activeSegments = timeline.filter(
+    (segment) => segment.type === "active" || segment.type === "scene_change",
+  );
+
+  if (activeSegments.length === 0) return videoPath;
+
+  const outputDir = mkdtempSync(path.join(tmpdir(), "expect-trimmed-"));
+  const outputPath = path.join(outputDir, "trimmed.mp4");
+
+  const filterParts = activeSegments.map(
+    (segment) => `between(t,${segment.startSeconds},${segment.endSeconds})`,
+  );
+  const selectFilter = filterParts.join("+");
+
+  await execFileAsync("ffmpeg", [
+    "-i",
+    videoPath,
+    "-vf",
+    `select='${selectFilter}',setpts=N/FRAME_RATE/TB`,
+    "-af",
+    `aselect='${selectFilter}',asetpts=N/SR/TB`,
+    "-y",
+    outputPath,
+  ]);
+
+  return outputPath;
+};
+
+export const formatTimeline = (timeline: ActivityTimeline): string => {
+  const formatTime = (seconds: number): string => {
+    const minutes = Math.floor(seconds / 60);
+    const secs = seconds % 60;
+    return `${String(minutes).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
+  };
+
+  const lines = timeline.map((segment) => {
+    const label =
+      segment.type === "scene_change"
+        ? "scene change (likely navigation)"
+        : segment.type === "idle"
+          ? "idle"
+          : "active";
+    return `- [${formatTime(segment.startSeconds)}–${formatTime(segment.endSeconds)}] ${label}`;
+  });
+
+  return lines.join("\n");
+};
diff --git a/apps/video-transcript/src/constants.ts b/apps/video-transcript/src/constants.ts
new file mode 100644
index 000000000..816487dd2
--- /dev/null
+++ b/apps/video-transcript/src/constants.ts
@@ -0,0 +1,14 @@
+export const FRAME_DIFF_IDLE_THRESHOLD = 0.005;
+export const IDLE_CUT_THRESHOLD_SECONDS = 3;
+export const SCENE_CHANGE_THRESHOLD = 0.15;
+export const MIN_ACTIVE_SEGMENT_SECONDS = 1;
+export const FRAMES_PER_SECOND = 1;
+
+export const SUPPORTED_VIDEO_EXTENSIONS = [".mp4", ".webm", ".mov", ".avi", ".mkv"] as const;
+export const SUPPORTED_MIME_TYPES: Record<string, string> = {
+  ".mp4": "video/mp4",
+  ".webm": "video/webm",
+  ".mov": "video/quicktime",
+  ".avi": "video/x-msvideo",
+  ".mkv": "video/x-matroska",
+};
diff --git a/apps/video-transcript/src/extract-transcript.ts b/apps/video-transcript/src/extract-transcript.ts
new file mode 100644
index 000000000..d6e305639
--- /dev/null
+++ b/apps/video-transcript/src/extract-transcript.ts
@@ -0,0 +1,36 @@
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { google } from "@ai-sdk/google";
+import { generateText } from "ai";
+import { SUPPORTED_MIME_TYPES } from "./constants";
+import { buildTranscriptPrompt } from "./transcript-prompt";
+import type { ActivityTimeline } from "./types";
+
+const getMimeType = (videoPath: string): string => {
+  const extension = path.extname(videoPath).toLowerCase();
+  return SUPPORTED_MIME_TYPES[extension] ?? "video/mp4";
+};
+
+export const extractTranscript = async (
+  videoPath: string,
+  timeline: ActivityTimeline | undefined,
+): Promise<string> => {
+  const mimeType = getMimeType(videoPath);
+  const videoData = readFileSync(videoPath);
+  const prompt = buildTranscriptPrompt(timeline);
+
+  const { text } = await generateText({
+    model: google("gemini-2.5-flash"),
+    messages: [
+      {
+        role: "user",
+        content: [
+          { type: "file", data: videoData, mediaType: mimeType },
+          { type: "text", text: prompt },
+        ],
+      },
+    ],
+  });
+
+  return text;
+};
diff --git a/apps/video-transcript/src/index.ts b/apps/video-transcript/src/index.ts
new file mode 100644
index 000000000..6e2a0d359
--- /dev/null
+++ b/apps/video-transcript/src/index.ts
@@ -0,0 +1,128 @@
+#!/usr/bin/env node
+import { existsSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { Command } from "commander";
+import pc from "picocolors";
+import { SUPPORTED_VIDEO_EXTENSIONS } from "./constants";
+import {
+  analyzeActivity,
+  buildTrimmedVideo,
+  checkFfmpegAvailable,
+  formatTimeline,
+} from "./activity-analyzer";
+import { extractTranscript } from "./extract-transcript";
+
+const program = new Command()
+  .name("video-transcript")
+  .description("Extract structured interaction transcripts from screen recordings")
+  .version("0.0.1")
+  .argument("<video>", "path to the screen recording file")
+  .option("-o, --output <path>", "write transcript to a file instead of stdout")
+  .option("--no-preprocess", "skip ffmpeg idle-time cutting (upload raw video)")
+  .option("--timeline-only", "only output the activity timeline, skip transcript extraction")
+  .option("--verbose", "show detailed progress information");
+
+program.action(
+  async (
+    videoArg: string,
+    options: {
+      output?: string;
+      preprocess: boolean;
+      timelineOnly: boolean;
+      verbose: boolean;
+    },
+  ) => {
+    const videoPath = path.resolve(videoArg);
+
+    if (!existsSync(videoPath)) {
+      console.error(pc.red(`Error: Video file not found: ${videoPath}`));
+      process.exit(1);
+    }
+
+    const extension = path.extname(videoPath).toLowerCase();
+    if (
+      !SUPPORTED_VIDEO_EXTENSIONS.includes(extension as (typeof SUPPORTED_VIDEO_EXTENSIONS)[number])
+    ) {
+      console.error(
+        pc.red(
+          `Error: Unsupported video format "${extension}". Supported: ${SUPPORTED_VIDEO_EXTENSIONS.join(", ")}`,
+        ),
+      );
+      process.exit(1);
+    }
+
+    if (!options.timelineOnly && !process.env["GOOGLE_GENERATIVE_AI_API_KEY"]) {
+      console.error(
+        pc.red(
+          "Error: GOOGLE_GENERATIVE_AI_API_KEY environment variable is required for transcript extraction.",
+        ),
+      );
+      process.exit(1);
+    }
+
+    let processedVideoPath = videoPath;
+    let timeline: Awaited<ReturnType<typeof analyzeActivity>> | undefined;
+
+    if (options.preprocess) {
+      const hasFfmpeg = await checkFfmpegAvailable();
+
+      if (hasFfmpeg) {
+        if (options.verbose) console.error(pc.dim("Analyzing video for idle segments..."));
+
+        timeline = await analyzeActivity(videoPath);
+
+        if (options.verbose) {
+          console.error(pc.dim("Activity timeline:"));
+          console.error(pc.dim(formatTimeline(timeline)));
+        }
+
+        const activeCount = timeline.filter(
+          (segment) => segment.type === "active" || segment.type === "scene_change",
+        ).length;
+        const idleCount = timeline.filter((segment) => segment.type === "idle").length;
+
+        console.error(
+          pc.cyan(
+            `Found ${activeCount} active segment${activeCount !== 1 ? "s" : ""}, ${idleCount} idle segment${idleCount !== 1 ? "s" : ""}`,
+          ),
+        );
+
+        if (options.timelineOnly) {
+          const output = formatTimeline(timeline);
+          if (options.output) {
+            writeFileSync(options.output, output);
+            console.error(pc.green(`Timeline written to ${options.output}`));
+          } else {
+            console.log(output);
+          }
+          return;
+        }
+
+        if (idleCount > 0) {
+          if (options.verbose) console.error(pc.dim("Building trimmed video..."));
+          processedVideoPath = await buildTrimmedVideo(videoPath, timeline);
+          console.error(pc.cyan("Trimmed idle segments from video"));
+        }
+      } else {
+        console.error(
+          pc.yellow(
+            "Warning: ffmpeg not found. Uploading raw video without idle-time cutting. Install ffmpeg for optimized processing.",
+          ),
+        );
+      }
+    }
+
+    console.error(pc.cyan("Extracting transcript via Gemini 2.5 Flash..."));
+
+    const transcript = await extractTranscript(processedVideoPath, timeline);
+
+    if (options.output) {
+      writeFileSync(options.output, transcript);
+      console.error(pc.green(`Transcript written to ${options.output}`));
+    } else {
+      console.log(transcript);
+    }
+  },
+);
+
+program.parse();
diff --git a/apps/video-transcript/src/transcript-prompt.ts b/apps/video-transcript/src/transcript-prompt.ts
new file mode 100644
index 000000000..ce931c71f
--- /dev/null
+++ b/apps/video-transcript/src/transcript-prompt.ts
@@ -0,0 +1,38 @@
+import type { ActivityTimeline } from "./types";
+import { formatTimeline } from "./activity-analyzer";
+
+const BASE_PROMPT = `You are analyzing a screen recording of a developer interacting with a web application.
+Produce a structured interaction transcript describing every user action and observable
+UI response. This transcript will be used to generate automated browser tests.
+
+Format each interaction as a timestamped entry:
+
+[MM:SS] ACTION: <what the user did>
+        TARGET: <what UI element they interacted with — be specific: button label, input field name, menu item, link text>
+        RESULT: <what happened in the UI after the action — page navigation, content change, modal appearance, error message, loading state, etc.>
+        URL: <the URL visible in the browser, if observable>
+
+Group related interactions into logical steps separated by blank lines.
+Add a one-line summary at the start of each group describing the workflow phase
+(e.g., "## Login", "## Add item to cart", "## Complete checkout").
+
+Rules:
+- Describe what you SEE, not what you infer. If you can read text on screen, quote it.
+- Include the URL or route when visible in the address bar.
+- Note form field values the user types (mask passwords as "***").
+- Note any error messages, toasts, validation messages, or loading spinners.
+- Skip idle periods where nothing happens. If there's a gap, note it as [MM:SS] IDLE: ~Ns.
+- For the final state, describe what the screen shows — this is the expected end state for the test.`;
+
+export const buildTranscriptPrompt = (timeline: ActivityTimeline | undefined): string => {
+  if (!timeline || timeline.length === 0) return BASE_PROMPT;
+
+  return [
+    BASE_PROMPT,
+    "",
+    "Activity timeline (from frame analysis):",
+    formatTimeline(timeline),
+    "",
+    "Focus your transcript on the active segments. The idle segments have been cut from the video.",
+  ].join("\n");
+};
diff --git a/apps/video-transcript/src/types.ts b/apps/video-transcript/src/types.ts
new file mode 100644
index 000000000..f44f29a5d
--- /dev/null
+++ b/apps/video-transcript/src/types.ts
@@ -0,0 +1,7 @@
+export interface ActivitySegment {
+  readonly type: "active" | "idle" | "scene_change";
+  readonly startSeconds: number;
+  readonly endSeconds: number;
+}
+
+export type ActivityTimeline = readonly ActivitySegment[];
diff --git a/apps/video-transcript/tsconfig.json b/apps/video-transcript/tsconfig.json
new file mode 100644
index 000000000..27d6a8668
--- /dev/null
+++ b/apps/video-transcript/tsconfig.json
@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": ".",
+    "types": ["node"]
+  },
+  "include": ["src"]
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 05a588bc8..937010ef3 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -153,6 +153,31 @@ importers:
         specifier: ^11.19.1
         version: 11.19.1
 
+  apps/video-transcript:
+    dependencies:
+      '@ai-sdk/google':
+        specifier: ^3.0.53
+        version: 3.0.58(zod@4.3.6)
+      ai:
+        specifier: ^6.0.146
+        version: 6.0.146(zod@4.3.6)
+      commander:
+        specifier: ^13.1.0
+        version: 13.1.0
+      effect:
+        specifier: 4.0.0-beta.35
+        version: 4.0.0-beta.35
+      picocolors:
+        specifier: ^1.1.1
+        version: 1.1.1
+    devDependencies:
+      '@types/node':
+        specifier: ^22.15.0
+        version: 22.19.15
+      typescript:
+        specifier: ^5.7.0
+        version: 5.9.3
+
   apps/website:
     dependencies:
       '@base-ui/react':
@@ -502,6 +527,24 @@ packages:
     peerDependencies:
       zod: ^3.25.0 || ^4.0.0
 
+  '@ai-sdk/gateway@3.0.88':
+    resolution: {integrity: sha512-AFoj7xdWAtCQcy0jJ235ENSakYM8D28qBX+rB+/rX4r8qe/LXgl0e5UivOqxAlIM5E9jnQdYxIPuj3XFtGk/yg==}
+    engines: {node: '>=18'}
+    peerDependencies:
+      zod: ^3.25.76 || ^4.1.8
+
+  '@ai-sdk/google@3.0.58':
+    resolution: {integrity: sha512-7P7s8g/FoIxesx2y32eK8idAMLOFHN2f4gs5KYi8q2QaScuubXFjgFMFqbjYF5bc92akiOd/C6OG0vIDlV7t2Q==}
+    engines: {node: '>=18'}
+    peerDependencies:
+      zod: ^3.25.76 || ^4.1.8
+
+  '@ai-sdk/provider-utils@4.0.22':
+    resolution: {integrity: sha512-B2OTFcRw/Pdka9ZTjpXv6T6qZ6RruRuLokyb8HwW+aoW9ndJ3YasA3/mVswyJw7VMBF8ofXgqvcrCt9KYvFifg==}
+    engines: {node: '>=18'}
+    peerDependencies:
+      zod: ^3.25.76 || ^4.1.8
+
   '@ai-sdk/provider@3.0.8':
     resolution: {integrity: sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ==}
     engines: {node: '>=18'}
@@ -3076,6 +3119,10 @@ packages:
       vue-router:
         optional: true
 
+  '@vercel/oidc@3.1.0':
+    resolution: {integrity: sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w==}
+    engines: {node: '>= 20'}
+
   '@voidzero-dev/vite-plus-core@0.1.12':
     resolution: {integrity: sha512-j8YNe7A+8JcSoddztf5whvom/yJ7OKUO3Y5a3UoLIUmOL8YEKVv5nPANrxJ7eaFfHJoMnBEwzBpq1YVZ+H3uPA==}
     engines: {node: ^20.19.0 || >=22.12.0}
@@ -3279,6 +3326,12 @@ packages:
     resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==}
     engines: {node: '>= 14'}
 
+  ai@6.0.146:
+    resolution: {integrity: sha512-70DE8k1rR0N3mXxyyfjYAx/FxRln/kQ5ym18lt1ys1eUklcPuoIXGbUBwdfCbmkt6YF3jCDZ5+OgkWieP/NGDw==}
+    engines: {node: '>=18'}
+    peerDependencies:
+      zod: ^3.25.76 || ^4.1.8
+
   ajv-formats@3.0.1:
     resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==}
     peerDependencies:
@@ -7122,6 +7175,26 @@ snapshots:
     dependencies:
       zod: 4.3.6
 
+  '@ai-sdk/gateway@3.0.88(zod@4.3.6)':
+    dependencies:
+      '@ai-sdk/provider': 3.0.8
+      '@ai-sdk/provider-utils': 4.0.22(zod@4.3.6)
+      '@vercel/oidc': 3.1.0
+      zod: 4.3.6
+
+  '@ai-sdk/google@3.0.58(zod@4.3.6)':
+    dependencies:
+      '@ai-sdk/provider': 3.0.8
+      '@ai-sdk/provider-utils': 4.0.22(zod@4.3.6)
+      zod: 4.3.6
+
+  '@ai-sdk/provider-utils@4.0.22(zod@4.3.6)':
+    dependencies:
+      '@ai-sdk/provider': 3.0.8
+      '@standard-schema/spec': 1.1.0
+      eventsource-parser: 3.0.6
+      zod: 4.3.6
+
   '@ai-sdk/provider@3.0.8':
     dependencies:
       json-schema: 0.4.0
@@ -9597,6 +9670,8 @@ snapshots:
       next: 16.2.0(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)
       react: 19.2.4
 
+  '@vercel/oidc@3.1.0': {}
+
   '@voidzero-dev/vite-plus-core@0.1.12(@types/node@22.19.15)(esbuild@0.27.4)(jiti@2.6.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2)':
     dependencies:
       '@oxc-project/runtime': 0.115.0
@@ -9731,6 +9806,14 @@ snapshots:
 
   agent-base@7.1.4: {}
 
+  ai@6.0.146(zod@4.3.6):
+    dependencies:
+      '@ai-sdk/gateway': 3.0.88(zod@4.3.6)
+      '@ai-sdk/provider': 3.0.8
+      '@ai-sdk/provider-utils': 4.0.22(zod@4.3.6)
+      '@opentelemetry/api': 1.9.0
+      zod: 4.3.6
+
   ajv-formats@3.0.1(ajv@8.18.0):
     optionalDependencies:
       ajv: 8.18.0

From ce7912b3953c1509540f0f2ba414e1397cb0006e Mon Sep 17 00:00:00 2001
From: Aiden Bai <aiden.bai05@gmail.com>
Date: Sat, 4 Apr 2026 21:26:32 -0700
Subject: [PATCH 2/3] switch to Vercel AI Gateway and add tests

- Replace @ai-sdk/google with @ai-sdk/gateway for provider-agnostic
  model routing via AI_GATEWAY_API_KEY
- Add 24 tests covering activity-analyzer (frame diff, segment
  classification, timeline formatting), transcript-prompt (base prompt,
  timeline appending), and extract-transcript (gateway model, file parts,
  timeline inclusion, response handling)
---
 apps/video-transcript/package.json            |   5 +-
 .../src/activity-analyzer.test.ts             | 107 ++++++++++++++++++
 .../video-transcript/src/activity-analyzer.ts |   4 +-
 .../src/extract-transcript.test.ts            |  76 +++++++++++++
 .../src/extract-transcript.ts                 |   6 +-
 apps/video-transcript/src/index.ts            |   6 +-
 .../src/transcript-prompt.test.ts             |  50 ++++++++
 apps/video-transcript/tsconfig.json           |   2 +-
 pnpm-lock.yaml                                |  37 +++---
 9 files changed, 260 insertions(+), 33 deletions(-)
 create mode 100644 apps/video-transcript/src/activity-analyzer.test.ts
 create mode 100644 apps/video-transcript/src/extract-transcript.test.ts
 create mode 100644 apps/video-transcript/src/transcript-prompt.test.ts

diff --git a/apps/video-transcript/package.json b/apps/video-transcript/package.json
index 7082286be..e0744c051 100644
--- a/apps/video-transcript/package.json
+++ b/apps/video-transcript/package.json
@@ -26,7 +26,7 @@
     "typecheck": "tsgo --noEmit"
   },
   "dependencies": {
-    "@ai-sdk/google": "^3.0.53",
+    "@ai-sdk/gateway": "^3.0.88",
     "ai": "^6.0.146",
     "commander": "^13.1.0",
     "effect": "4.0.0-beta.35",
@@ -34,7 +34,8 @@
   },
   "devDependencies": {
     "@types/node": "^22.15.0",
-    "typescript": "^5.7.0"
+    "typescript": "^5.7.0",
+    "vitest": "*"
   },
   "engines": {
     "node": ">=18"
diff --git a/apps/video-transcript/src/activity-analyzer.test.ts b/apps/video-transcript/src/activity-analyzer.test.ts
new file mode 100644
index 000000000..029601158
--- /dev/null
+++ b/apps/video-transcript/src/activity-analyzer.test.ts
@@ -0,0 +1,107 @@
+import { describe, it, expect } from "vitest";
+import { computeFrameDiff, classifySegments, formatTimeline } from "./activity-analyzer";
+
+describe("computeFrameDiff", () => {
+  it("returns 0 for identical frames", () => {
+    const frame = Buffer.from([0, 128, 255, 64]);
+    expect(computeFrameDiff(frame, frame)).toBe(0);
+  });
+
+  it("returns 1 for maximally different frames", () => {
+    const black = Buffer.from([0, 0, 0, 0]);
+    const white = Buffer.from([255, 255, 255, 255]);
+    expect(computeFrameDiff(black, white)).toBeCloseTo(1, 5);
+  });
+
+  it("returns 0 for empty buffers", () => {
+    expect(computeFrameDiff(Buffer.alloc(0), Buffer.alloc(0))).toBe(0);
+  });
+
+  it("computes correct diff for known values", () => {
+    const frameA = Buffer.from([0, 0, 0, 0]);
+    const frameB = Buffer.from([255, 0, 0, 0]);
+    expect(computeFrameDiff(frameA, frameB)).toBeCloseTo(0.25, 5);
+  });
+
+  it("handles frames of different sizes using the shorter length", () => {
+    const short = Buffer.from([0, 0]);
+    const long = Buffer.from([255, 255, 255, 255]);
+    expect(computeFrameDiff(short, long)).toBeCloseTo(1, 5);
+  });
+});
+
+describe("classifySegments", () => {
+  it("returns empty for no diffs", () => {
+    expect(classifySegments([])).toEqual([]);
+  });
+
+  it("classifies all-idle diffs as a single idle segment", () => {
+    const diffs = [0.001, 0.002, 0.001, 0.003, 0.002];
+    const result = classifySegments(diffs);
+    expect(result.length).toBe(1);
+    expect(result[0]!.type).toBe("idle");
+    expect(result[0]!.startSeconds).toBe(0);
+    expect(result[0]!.endSeconds).toBe(5);
+  });
+
+  it("classifies all-active diffs as a single active segment", () => {
+    const diffs = [0.05, 0.08, 0.06, 0.07];
+    const result = classifySegments(diffs);
+    expect(result.length).toBe(1);
+    expect(result[0]!.type).toBe("active");
+  });
+
+  it("detects scene changes from high diffs", () => {
+    const diffs = [0.01, 0.02, 0.5, 0.01, 0.02];
+    const result = classifySegments(diffs);
+    const sceneChanges = result.filter((s) => s.type === "scene_change");
+    expect(sceneChanges.length).toBeGreaterThanOrEqual(1);
+  });
+
+  it("merges short idle gaps between active segments", () => {
+    const diffs = [0.05, 0.06, 0.07, 0.001, 0.002, 0.05, 0.06, 0.07];
+    const result = classifySegments(diffs);
+    const idleSegments = result.filter((s) => s.type === "idle");
+    expect(idleSegments.length).toBe(0);
+    expect(result.every((s) => s.type === "active")).toBe(true);
+  });
+
+  it("preserves long idle gaps between active segments", () => {
+    const diffs = [0.05, 0.06, 0.001, 0.002, 0.001, 0.002, 0.05, 0.06];
+    const result = classifySegments(diffs);
+    const idleSegments = result.filter((s) => s.type === "idle");
+    expect(idleSegments.length).toBeGreaterThanOrEqual(1);
+  });
+});
+
+describe("formatTimeline", () => {
+  it("formats a simple timeline", () => {
+    const timeline = [
+      { type: "idle" as const, startSeconds: 0, endSeconds: 3 },
+      { type: "active" as const, startSeconds: 3, endSeconds: 10 },
+      { type: "idle" as const, startSeconds: 10, endSeconds: 12 },
+    ];
+    const result = formatTimeline(timeline);
+    expect(result).toContain("00:00");
+    expect(result).toContain("00:03");
+    expect(result).toContain("active");
+    expect(result).toContain("idle");
+  });
+
+  it("formats scene changes", () => {
+    const timeline = [{ type: "scene_change" as const, startSeconds: 5, endSeconds: 6 }];
+    const result = formatTimeline(timeline);
+    expect(result).toContain("scene change (likely navigation)");
+  });
+
+  it("formats times with minutes", () => {
+    const timeline = [{ type: "active" as const, startSeconds: 65, endSeconds: 130 }];
+    const result = formatTimeline(timeline);
+    expect(result).toContain("01:05");
+    expect(result).toContain("02:10");
+  });
+
+  it("returns empty string for empty timeline", () => {
+    expect(formatTimeline([])).toBe("");
+  });
+});
diff --git a/apps/video-transcript/src/activity-analyzer.ts b/apps/video-transcript/src/activity-analyzer.ts
index 3b742ff61..f27bfe70e 100644
--- a/apps/video-transcript/src/activity-analyzer.ts
+++ b/apps/video-transcript/src/activity-analyzer.ts
@@ -52,7 +52,7 @@ const extractFrames = async (videoPath: string, outputDir: string): Promise<numb
   return files.length;
 };
 
-const computeFrameDiff = (frameA: Buffer, frameB: Buffer): number => {
+export const computeFrameDiff = (frameA: Buffer, frameB: Buffer): number => {
   const length = Math.min(frameA.length, frameB.length);
   if (length === 0) return 0;
 
@@ -64,7 +64,7 @@ const computeFrameDiff = (frameA: Buffer, frameB: Buffer): number => {
   return totalDiff / length;
 };
 
-const classifySegments = (diffs: readonly number[]): ActivityTimeline => {
+export const classifySegments = (diffs: readonly number[]): ActivityTimeline => {
   const rawClassification: Array<"active" | "idle" | "scene_change"> = [];
 
   for (const diff of diffs) {
diff --git a/apps/video-transcript/src/extract-transcript.test.ts b/apps/video-transcript/src/extract-transcript.test.ts
new file mode 100644
index 000000000..c289b70a8
--- /dev/null
+++ b/apps/video-transcript/src/extract-transcript.test.ts
@@ -0,0 +1,76 @@
+import { describe, it, expect, vi, beforeAll, beforeEach, afterAll } from "vitest";
+
+vi.mock("@ai-sdk/gateway", () => ({
+  gateway: (model: string) => ({ modelId: model, provider: "gateway" }),
+}));
+
+vi.mock("ai", () => ({
+  generateText: vi.fn().mockResolvedValue({
+    text: "## Login\n\n[00:03] ACTION: User clicks Sign In\n        TARGET: Sign In button\n        RESULT: Login form appears",
+  }),
+}));
+
+import { extractTranscript } from "./extract-transcript";
+import { generateText } from "ai";
+import path from "node:path";
+import { writeFileSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+
+describe("extractTranscript", () => {
+  let tempDir: string;
+  let videoPath: string;
+
+  beforeAll(() => {
+    tempDir = mkdtempSync(path.join(tmpdir(), "extract-test-"));
+    videoPath = path.join(tempDir, "test.mp4");
+    writeFileSync(videoPath, Buffer.from("fake video content"));
+  });
+
+  beforeEach(() => {
+    vi.mocked(generateText).mockClear();
+  });
+
+  afterAll(() => {
+    rmSync(tempDir, { recursive: true, force: true });
+  });
+
+  it("calls generateText with gateway model", async () => {
+    const result = await extractTranscript(videoPath, undefined);
+
+    expect(generateText).toHaveBeenCalledOnce();
+    const call = vi.mocked(generateText).mock.calls[0]![0];
+    expect((call.model as { modelId: string }).modelId).toBe("google/gemini-2.5-flash");
+    expect(result).toContain("Login");
+  });
+
+  it("passes video data as file part with correct media type", async () => {
+    await extractTranscript(videoPath, undefined);
+
+    const call = vi.mocked(generateText).mock.calls[0]![0];
+    const messages = call.messages as Array<{
+      content: Array<{ type: string; mediaType?: string }>;
+    }>;
+    const filePart = messages[0]!.content.find((part) => part.type === "file");
+    expect(filePart).toBeDefined();
+    expect(filePart!.mediaType).toBe("video/mp4");
+  });
+
+  it("includes timeline in prompt when provided", async () => {
+    const timeline = [
+      { type: "active" as const, startSeconds: 0, endSeconds: 10 },
+      { type: "idle" as const, startSeconds: 10, endSeconds: 15 },
+    ];
+
+    await extractTranscript(videoPath, timeline);
+
+    const call = vi.mocked(generateText).mock.calls[0]![0];
+    const messages = call.messages as Array<{ content: Array<{ type: string; text?: string }> }>;
+    const textPart = messages[0]!.content.find((part) => part.type === "text");
+    expect(textPart!.text).toContain("Activity timeline");
+  });
+
+  it("returns transcript text from generateText response", async () => {
+    const result = await extractTranscript(videoPath, undefined);
+    expect(result).toContain("ACTION: User clicks Sign In");
+  });
+});
diff --git a/apps/video-transcript/src/extract-transcript.ts b/apps/video-transcript/src/extract-transcript.ts
index d6e305639..0fc39dd07 100644
--- a/apps/video-transcript/src/extract-transcript.ts
+++ b/apps/video-transcript/src/extract-transcript.ts
@@ -1,11 +1,13 @@
 import { readFileSync } from "node:fs";
 import path from "node:path";
-import { google } from "@ai-sdk/google";
+import { gateway } from "@ai-sdk/gateway";
 import { generateText } from "ai";
 import { SUPPORTED_MIME_TYPES } from "./constants";
 import { buildTranscriptPrompt } from "./transcript-prompt";
 import type { ActivityTimeline } from "./types";
 
+const MODEL = "google/gemini-2.5-flash";
+
 const getMimeType = (videoPath: string): string => {
   const extension = path.extname(videoPath).toLowerCase();
   return SUPPORTED_MIME_TYPES[extension] ?? "video/mp4";
@@ -20,7 +22,7 @@ export const extractTranscript = async (
   const prompt = buildTranscriptPrompt(timeline);
 
   const { text } = await generateText({
-    model: google("gemini-2.5-flash"),
+    model: gateway(MODEL),
     messages: [
       {
         role: "user",
diff --git a/apps/video-transcript/src/index.ts b/apps/video-transcript/src/index.ts
index 6e2a0d359..d11da961f 100644
--- a/apps/video-transcript/src/index.ts
+++ b/apps/video-transcript/src/index.ts
@@ -51,10 +51,10 @@ program.action(
       process.exit(1);
     }
 
-    if (!options.timelineOnly && !process.env["GOOGLE_GENERATIVE_AI_API_KEY"]) {
+    if (!options.timelineOnly && !process.env["AI_GATEWAY_API_KEY"]) {
       console.error(
         pc.red(
-          "Error: GOOGLE_GENERATIVE_AI_API_KEY environment variable is required for transcript extraction.",
+          "Error: AI_GATEWAY_API_KEY environment variable is required for transcript extraction.",
         ),
       );
       process.exit(1);
@@ -112,7 +112,7 @@ program.action(
       }
     }
 
-    console.error(pc.cyan("Extracting transcript via Gemini 2.5 Flash..."));
+    console.error(pc.cyan("Extracting transcript via Vercel AI Gateway (Gemini 2.5 Flash)..."));
 
     const transcript = await extractTranscript(processedVideoPath, timeline);
 
diff --git a/apps/video-transcript/src/transcript-prompt.test.ts b/apps/video-transcript/src/transcript-prompt.test.ts
new file mode 100644
index 000000000..13706f096
--- /dev/null
+++ b/apps/video-transcript/src/transcript-prompt.test.ts
@@ -0,0 +1,50 @@
+import { describe, it, expect } from "vitest";
+import { buildTranscriptPrompt } from "./transcript-prompt";
+
+describe("buildTranscriptPrompt", () => {
+  it("returns base prompt when timeline is undefined", () => {
+    const result = buildTranscriptPrompt(undefined);
+    expect(result).toContain("screen recording");
+    expect(result).toContain("[MM:SS] ACTION:");
+    expect(result).not.toContain("Activity timeline");
+  });
+
+  it("returns base prompt when timeline is empty", () => {
+    const result = buildTranscriptPrompt([]);
+    expect(result).toContain("screen recording");
+    expect(result).not.toContain("Activity timeline");
+  });
+
+  it("appends timeline section when timeline is provided", () => {
+    const timeline = [
+      { type: "idle" as const, startSeconds: 0, endSeconds: 2 },
+      { type: "active" as const, startSeconds: 2, endSeconds: 10 },
+      { type: "idle" as const, startSeconds: 10, endSeconds: 12 },
+    ];
+    const result = buildTranscriptPrompt(timeline);
+    expect(result).toContain("Activity timeline (from frame analysis):");
+    expect(result).toContain("active");
+    expect(result).toContain("idle");
+    expect(result).toContain("Focus your transcript on the active segments");
+  });
+
+  it("includes scene change labels in timeline", () => {
+    const timeline = [
+      { type: "active" as const, startSeconds: 0, endSeconds: 5 },
+      { type: "scene_change" as const, startSeconds: 5, endSeconds: 6 },
+      { type: "active" as const, startSeconds: 6, endSeconds: 12 },
+    ];
+    const result = buildTranscriptPrompt(timeline);
+    expect(result).toContain("scene change (likely navigation)");
+  });
+
+  it("always includes the structured format instructions", () => {
+    const result = buildTranscriptPrompt([
+      { type: "active" as const, startSeconds: 0, endSeconds: 10 },
+    ]);
+    expect(result).toContain("ACTION:");
+    expect(result).toContain("TARGET:");
+    expect(result).toContain("RESULT:");
+    expect(result).toContain("URL:");
+  });
+});
diff --git a/apps/video-transcript/tsconfig.json b/apps/video-transcript/tsconfig.json
index 27d6a8668..842a22c12 100644
--- a/apps/video-transcript/tsconfig.json
+++ b/apps/video-transcript/tsconfig.json
@@ -3,7 +3,7 @@
   "compilerOptions": {
     "outDir": "./dist",
     "rootDir": ".",
-    "types": ["node"]
+    "types": ["node", "vitest/globals"]
   },
   "include": ["src"]
 }
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 937010ef3..65adb1036 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -155,9 +155,9 @@ importers:
 
   apps/video-transcript:
     dependencies:
-      '@ai-sdk/google':
-        specifier: ^3.0.53
-        version: 3.0.58(zod@4.3.6)
+      '@ai-sdk/gateway':
+        specifier: ^3.0.88
+        version: 3.0.88(zod@4.3.6)
       ai:
         specifier: ^6.0.146
         version: 6.0.146(zod@4.3.6)
@@ -177,6 +177,9 @@ importers:
       typescript:
         specifier: ^5.7.0
         version: 5.9.3
+      vitest:
+        specifier: npm:@voidzero-dev/vite-plus-test@^0.1.12
+        version: '@voidzero-dev/vite-plus-test@0.1.12(@opentelemetry/api@1.9.0)(@types/node@22.19.15)(esbuild@0.27.4)(jiti@2.6.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(vite@7.3.1(@types/node@22.19.15)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.2))(yaml@2.8.2)'
 
   apps/website:
     dependencies:
@@ -533,12 +536,6 @@ packages:
     peerDependencies:
       zod: ^3.25.76 || ^4.1.8
 
-  '@ai-sdk/google@3.0.58':
-    resolution: {integrity: sha512-7P7s8g/FoIxesx2y32eK8idAMLOFHN2f4gs5KYi8q2QaScuubXFjgFMFqbjYF5bc92akiOd/C6OG0vIDlV7t2Q==}
-    engines: {node: '>=18'}
-    peerDependencies:
-      zod: ^3.25.76 || ^4.1.8
-
   '@ai-sdk/provider-utils@4.0.22':
     resolution: {integrity: sha512-B2OTFcRw/Pdka9ZTjpXv6T6qZ6RruRuLokyb8HwW+aoW9ndJ3YasA3/mVswyJw7VMBF8ofXgqvcrCt9KYvFifg==}
     engines: {node: '>=18'}
@@ -7182,12 +7179,6 @@ snapshots:
       '@vercel/oidc': 3.1.0
       zod: 4.3.6
 
-  '@ai-sdk/google@3.0.58(zod@4.3.6)':
-    dependencies:
-      '@ai-sdk/provider': 3.0.8
-      '@ai-sdk/provider-utils': 4.0.22(zod@4.3.6)
-      zod: 4.3.6
-
   '@ai-sdk/provider-utils@4.0.22(zod@4.3.6)':
     dependencies:
       '@ai-sdk/provider': 3.0.8
@@ -10682,8 +10673,8 @@ snapshots:
       '@next/eslint-plugin-next': 16.2.0
       eslint: 9.39.4(jiti@2.6.1)
       eslint-import-resolver-node: 0.3.9
-      eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.6.1))
-      eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1))
+      eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1))
+      eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1))
       eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.4(jiti@2.6.1))
       eslint-plugin-react: 7.37.5(eslint@9.39.4(jiti@2.6.1))
       eslint-plugin-react-hooks: 7.0.1(eslint@9.39.4(jiti@2.6.1))
@@ -10705,7 +10696,7 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.6.1)):
+  eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)):
     dependencies:
       '@nolyfill/is-core-module': 1.0.39
       debug: 4.4.3
@@ -10716,22 +10707,22 @@ snapshots:
       tinyglobby: 0.2.15
       unrs-resolver: 1.11.1
     optionalDependencies:
-      eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1))
+      eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1))
     transitivePeerDependencies:
       - supports-color
 
-  eslint-module-utils@2.12.1(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1)):
+  eslint-module-utils@2.12.1(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)):
     dependencies:
       debug: 3.2.7
     optionalDependencies:
       '@typescript-eslint/parser': 8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3)
       eslint: 9.39.4(jiti@2.6.1)
       eslint-import-resolver-node: 0.3.9
-      eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.6.1))
+      eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1))
     transitivePeerDependencies:
       - supports-color
 
-  eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1)):
+  eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)):
     dependencies:
       '@rtsao/scc': 1.1.0
       array-includes: 3.1.9
@@ -10742,7 +10733,7 @@ snapshots:
       doctrine: 2.1.0
       eslint: 9.39.4(jiti@2.6.1)
       eslint-import-resolver-node: 0.3.9
-      eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1))
+      eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.57.2(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1))
       hasown: 2.0.2
       is-core-module: 2.16.1
       is-glob: 4.0.3

From be73bdcccf4c68ae96d0bfe2a505b7511b72fc0f Mon Sep 17 00:00:00 2001
From: Aiden Bai <aiden.bai05@gmail.com>
Date: Sat, 4 Apr 2026 23:08:52 -0700
Subject: [PATCH 3/3] fix: add missing runtime-resolved deps to sdk and cli

The bundler inlines source from @expect/agent but leaves dynamic
require.resolve() calls intact. Consumers with strict node_modules
(pnpm) cannot resolve these at runtime unless they are declared as
dependencies in the published package.json.

Adds runtime-deps tests to both packages that scan the built dist
for require.resolve() targets and fail if any are undeclared.
---
 apps/cli/package.json                         |  3 +
 apps/cli/tests/runtime-deps.test.ts           | 76 +++++++++++++++++++
 packages/typescript-sdk/package.json          |  4 +
 .../typescript-sdk/tests/runtime-deps.test.ts | 57 ++++++++++++++
 pnpm-lock.yaml                                | 33 ++++++--
 5 files changed, 165 insertions(+), 8 deletions(-)
 create mode 100644 apps/cli/tests/runtime-deps.test.ts
 create mode 100644 packages/typescript-sdk/tests/runtime-deps.test.ts

diff --git a/apps/cli/package.json b/apps/cli/package.json
index 164148c56..70559423c 100644
--- a/apps/cli/package.json
+++ b/apps/cli/package.json
@@ -52,9 +52,12 @@
     "typecheck": "tsgo --noEmit"
   },
   "dependencies": {
+    "accessibility-checker-engine": "^4.0.16",
     "@agentclientprotocol/claude-agent-acp": "^0.24.2",
     "@effect/atom-react": "4.0.0-beta.35",
     "@effect/platform-node": "4.0.0-beta.35",
+    "@github/copilot": "^1.0.12",
+    "@google/gemini-cli": "^0.35.3",
     "@hono/node-server": "^1.19.11",
     "@tanstack/react-query": "^5.80.7",
     "@zed-industries/codex-acp": "^0.10.0",
diff --git a/apps/cli/tests/runtime-deps.test.ts b/apps/cli/tests/runtime-deps.test.ts
new file mode 100644
index 000000000..e21c04965
--- /dev/null
+++ b/apps/cli/tests/runtime-deps.test.ts
@@ -0,0 +1,76 @@
+import { describe, it, expect } from "vite-plus/test";
+import { readFileSync, readdirSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const cliRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
+const distDir = join(cliRoot, "dist");
+const packageJson = JSON.parse(readFileSync(join(cliRoot, "package.json"), "utf-8"));
+
+const declaredDeps = new Set([
+  ...Object.keys(packageJson.dependencies ?? {}),
+  ...Object.keys(packageJson.peerDependencies ?? {}),
+  ...Object.keys(packageJson.optionalDependencies ?? {}),
+]);
+
+/**
+ * Extracts packages that are resolved at runtime from the bundled dist.
+ *
+ * The bundler (vp pack) inlines source but leaves dynamic require.resolve()
+ * and resolvePackageBin() calls intact. These need to be resolvable from
+ * the consumer's node_modules, so they must be declared in package.json.
+ *
+ * Patterns matched:
+ *   - .resolve(`@scope/pkg/path`)       — minified makeRequire().resolve()
+ *   - .resolve("pkg/path")              — unminified require.resolve()
+ *   - resolvePackageBin(`@scope/pkg`)   — minified as varName(`@scope/pkg`)
+ *     detected via try/catch context: try:()=>{let t=Fn(`pkg`)
+ */
+const extractRuntimeResolvedPackages = (): string[] => {
+  const distFiles = readdirSync(distDir).filter(
+    (file) => file.endsWith(".js") && !file.endsWith(".map"),
+  );
+  const patterns = [
+    /\.resolve\(["']([^"']+)["']\)/g,
+    /\.resolve\(`([^`$]+)`\)/g,
+    // resolvePackageBin gets minified to a short var called inside try blocks
+    /try:\(\)=>\{let \w+=\w+\(`([^`]+)`\)/g,
+  ];
+  const packages = new Set<string>();
+
+  for (const file of distFiles) {
+    const content = readFileSync(join(distDir, file), "utf-8");
+    for (const pattern of patterns) {
+      let match: RegExpExecArray | null;
+      while ((match = pattern.exec(content)) !== null) {
+        const specifier = match[1];
+        const parts = specifier.startsWith("@")
+          ? specifier.split("/").slice(0, 2)
+          : specifier.split("/").slice(0, 1);
+        const packageName = parts.join("/");
+
+        if (specifier.startsWith(`${packageName}/package.json`)) continue;
+
+        packages.add(packageName);
+      }
+    }
+  }
+
+  return [...packages];
+};
+
+describe("runtime dependency safety", () => {
+  it("all runtime-resolved packages in dist are declared in package.json dependencies", () => {
+    const runtimePackages = extractRuntimeResolvedPackages();
+    const missing = runtimePackages.filter((pkg) => !declaredDeps.has(pkg));
+
+    if (missing.length > 0) {
+      throw new Error(
+        `Found runtime-resolved packages in dist/ not in dependencies, peerDependencies, or optionalDependencies:\n\n` +
+          missing.map((pkg) => `  - ${pkg}`).join("\n") +
+          `\n\nAdd them to "dependencies" in apps/cli/package.json ` +
+          `so consumers with strict node_modules (pnpm) can resolve them.`,
+      );
+    }
+  });
+});
diff --git a/packages/typescript-sdk/package.json b/packages/typescript-sdk/package.json
index c1d1801c2..55f5f4232 100644
--- a/packages/typescript-sdk/package.json
+++ b/packages/typescript-sdk/package.json
@@ -31,9 +31,13 @@
     "typecheck": "tsgo --noEmit"
   },
   "dependencies": {
+    "@agentclientprotocol/claude-agent-acp": "^0.24.2",
     "@effect/platform-node": "4.0.0-beta.35",
+    "@github/copilot": "^1.0.12",
+    "@google/gemini-cli": "^0.35.3",
     "@modelcontextprotocol/sdk": "^1.27.1",
     "@standard-schema/spec": "^1.1.0",
+    "@zed-industries/codex-acp": "^0.10.0",
     "effect": "4.0.0-beta.35"
   },
   "devDependencies": {
diff --git a/packages/typescript-sdk/tests/runtime-deps.test.ts b/packages/typescript-sdk/tests/runtime-deps.test.ts
new file mode 100644
index 000000000..fbf144cac
--- /dev/null
+++ b/packages/typescript-sdk/tests/runtime-deps.test.ts
@@ -0,0 +1,57 @@
+import { describe, it, expect } from "vite-plus/test";
+import { readFileSync, readdirSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const sdkRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
+const distDir = join(sdkRoot, "dist");
+const packageJson = JSON.parse(readFileSync(join(sdkRoot, "package.json"), "utf-8"));
+
+const declaredDeps = new Set([
+  ...Object.keys(packageJson.dependencies ?? {}),
+  ...Object.keys(packageJson.peerDependencies ?? {}),
+]);
+
+const extractRuntimeResolvedPackages = (): string[] => {
+  const distFiles = readdirSync(distDir).filter(
+    (file) => file.endsWith(".mjs") && !file.endsWith(".map"),
+  );
+  const patterns = [/require\.resolve\(["']([^"']+)["']\)/g, /\.resolve\(`([^`$]+)`\)/g];
+  const packages = new Set<string>();
+
+  for (const file of distFiles) {
+    const content = readFileSync(join(distDir, file), "utf-8");
+    for (const pattern of patterns) {
+      let match: RegExpExecArray | null;
+      while ((match = pattern.exec(content)) !== null) {
+        const specifier = match[1];
+        const parts = specifier.startsWith("@")
+          ? specifier.split("/").slice(0, 2)
+          : specifier.split("/").slice(0, 1);
+        const packageName = parts.join("/");
+
+        if (specifier.startsWith(`${packageName}/package.json`)) continue;
+
+        packages.add(packageName);
+      }
+    }
+  }
+
+  return [...packages];
+};
+
+describe("runtime dependency safety", () => {
+  it("all require.resolve() targets in dist are declared in package.json dependencies", () => {
+    const runtimePackages = extractRuntimeResolvedPackages();
+    const missing = runtimePackages.filter((pkg) => !declaredDeps.has(pkg));
+
+    if (missing.length > 0) {
+      throw new Error(
+        `Found require.resolve() calls in dist/ for packages not in dependencies or peerDependencies:\n\n` +
+          missing.map((pkg) => `  - ${pkg}`).join("\n") +
+          `\n\nAdd them to "dependencies" in packages/typescript-sdk/package.json ` +
+          `so consumers with strict node_modules (pnpm) can resolve them.`,
+      );
+    }
+  });
+});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 65adb1036..b0c4b3af0 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -42,6 +42,12 @@ importers:
       '@effect/platform-node':
         specifier: 4.0.0-beta.35
         version: 4.0.0-beta.35(effect@4.0.0-beta.35)(ioredis@5.10.0)
+      '@github/copilot':
+        specifier: ^1.0.12
+        version: 1.0.12
+      '@google/gemini-cli':
+        specifier: ^0.35.3
+        version: 0.35.3(@types/react@19.2.14)(express@5.2.1)
       '@hono/node-server':
         specifier: ^1.19.11
         version: 1.19.11(hono@4.12.9)
@@ -51,6 +57,9 @@ importers:
       '@zed-industries/codex-acp':
         specifier: ^0.10.0
         version: 0.10.0
+      accessibility-checker-engine:
+        specifier: ^4.0.16
+        version: 4.0.16
       cli-truncate:
         specifier: ^5.2.0
         version: 5.2.0
@@ -453,15 +462,27 @@ importers:
 
   packages/typescript-sdk:
     dependencies:
+      '@agentclientprotocol/claude-agent-acp':
+        specifier: ^0.24.2
+        version: 0.24.2
       '@effect/platform-node':
         specifier: 4.0.0-beta.35
         version: 4.0.0-beta.35(effect@4.0.0-beta.35)(ioredis@5.10.0)
+      '@github/copilot':
+        specifier: ^1.0.12
+        version: 1.0.12
+      '@google/gemini-cli':
+        specifier: ^0.35.3
+        version: 0.35.3(@types/react@19.2.14)(express@5.2.1)
       '@modelcontextprotocol/sdk':
         specifier: ^1.27.1
         version: 1.27.1(zod@4.3.6)
       '@standard-schema/spec':
         specifier: ^1.1.0
         version: 1.1.0
+      '@zed-industries/codex-acp':
+        specifier: ^0.10.0
+        version: 0.10.0
       effect:
         specifier: 4.0.0-beta.35
         version: 4.0.0-beta.35
@@ -7972,7 +7993,7 @@ snapshots:
       '@google-cloud/logging': 11.2.1
       '@google-cloud/opentelemetry-cloud-monitoring-exporter': 0.21.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@2.6.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@2.6.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-metrics@2.6.1(@opentelemetry/api@1.9.0))
       '@google-cloud/opentelemetry-cloud-trace-exporter': 3.0.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@2.6.1(@opentelemetry/api@1.9.0))(@opentelemetry/resources@2.6.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.0))
-      '@google/genai': 1.30.0(@modelcontextprotocol/sdk@1.27.1(zod@3.25.76))
+      '@google/genai': 1.30.0(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))
       '@grpc/grpc-js': 1.14.3
       '@iarna/toml': 2.2.5
       '@joshua.litt/get-ripgrep': 0.0.3
@@ -8059,7 +8080,7 @@ snapshots:
     dependencies:
       '@agentclientprotocol/sdk': 0.12.0(zod@3.25.76)
       '@google/gemini-cli-core': 0.35.3(express@5.2.1)
-      '@google/genai': 1.30.0(@modelcontextprotocol/sdk@1.27.1(zod@3.25.76))
+      '@google/genai': 1.30.0(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))
       '@iarna/toml': 2.2.5
       '@modelcontextprotocol/sdk': 1.27.1(zod@3.25.76)
       ansi-escapes: 7.3.0
@@ -8112,7 +8133,7 @@ snapshots:
       - tree-sitter
       - utf-8-validate
 
-  '@google/genai@1.30.0(@modelcontextprotocol/sdk@1.27.1(zod@3.25.76))':
+  '@google/genai@1.30.0(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))':
     dependencies:
       google-auth-library: 10.6.2
       ws: 8.19.0
@@ -8499,7 +8520,7 @@ snapshots:
       pkce-challenge: 5.0.1
       raw-body: 3.0.2
       zod: 4.3.6
-      zod-to-json-schema: 3.25.1(zod@4.3.6)
+      zod-to-json-schema: 3.25.1(zod@3.25.76)
     transitivePeerDependencies:
       - supports-color
 
@@ -14132,10 +14153,6 @@ snapshots:
     dependencies:
       zod: 3.25.76
 
-  zod-to-json-schema@3.25.1(zod@4.3.6):
-    dependencies:
-      zod: 4.3.6
-
   zod-validation-error@4.0.2(zod@4.3.6):
     dependencies:
       zod: 4.3.6