feat: video transcript prototype — generate tests from screen recordings

.specs/video-transcript.md

@@ -81,7 +81,7 @@ const { text } = await generateText({
   messages: [{
     role: "user",
     content: [
-      { type: "file", data: readFileSync(videoPath), mimeType: "video/mp4" },
+      { type: "file", data: readFileSync(videoPath), mediaType: "video/mp4" },
       { type: "text", text: transcriptPrompt },
     ],
   }],

apps/cli/package.json

@@ -52,9 +52,12 @@
     "typecheck": "tsgo --noEmit"
   },
   "dependencies": {
+    "accessibility-checker-engine": "^4.0.16",
     "@agentclientprotocol/claude-agent-acp": "^0.24.2",
     "@effect/atom-react": "4.0.0-beta.35",
     "@effect/platform-node": "4.0.0-beta.35",
+    "@github/copilot": "^1.0.12",
+    "@google/gemini-cli": "^0.35.3",
     "@hono/node-server": "^1.19.11",
     "@tanstack/react-query": "^5.80.7",
     "@zed-industries/codex-acp": "^0.10.0",

apps/cli/tests/runtime-deps.test.ts

@@ -0,0 +1,76 @@
+import { describe, it, expect } from "vite-plus/test";
+import { readFileSync, readdirSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const cliRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
+const distDir = join(cliRoot, "dist");
+const packageJson = JSON.parse(readFileSync(join(cliRoot, "package.json"), "utf-8"));
+
+const declaredDeps = new Set([
+  ...Object.keys(packageJson.dependencies ?? {}),
+  ...Object.keys(packageJson.peerDependencies ?? {}),
+  ...Object.keys(packageJson.optionalDependencies ?? {}),
+]);
+
+/**
+ * Extracts packages that are resolved at runtime from the bundled dist.
+ *
+ * The bundler (vp pack) inlines source but leaves dynamic require.resolve()
+ * and resolvePackageBin() calls intact. These need to be resolvable from
+ * the consumer's node_modules, so they must be declared in package.json.
+ *
+ * Patterns matched:
+ *   - .resolve(`@scope/pkg/path`)       — minified makeRequire().resolve()
+ *   - .resolve("pkg/path")              — unminified require.resolve()
+ *   - resolvePackageBin(`@scope/pkg`)   — minified as varName(`@scope/pkg`)
+ *     detected via try/catch context: try:()=>{let t=Fn(`pkg`)
+ */
+const extractRuntimeResolvedPackages = (): string[] => {
+  const distFiles = readdirSync(distDir).filter(
+    (file) => file.endsWith(".js") && !file.endsWith(".map"),
+  );
+  const patterns = [
+    /\.resolve\(["']([^"']+)["']\)/g,
+    /\.resolve\(`([^`$]+)`\)/g,
+    // resolvePackageBin gets minified to a short var called inside try blocks
+    /try:\(\)=>\{let \w+=\w+\(`([^`]+)`\)/g,
+  ];
+  const packages = new Set<string>();
+
+  for (const file of distFiles) {
+    const content = readFileSync(join(distDir, file), "utf-8");
+    for (const pattern of patterns) {
+      let match: RegExpExecArray | null;
+      while ((match = pattern.exec(content)) !== null) {
+        const specifier = match[1];
+        const parts = specifier.startsWith("@")
+          ? specifier.split("/").slice(0, 2)
+          : specifier.split("/").slice(0, 1);
+        const packageName = parts.join("/");
+
+        if (specifier.startsWith(`${packageName}/package.json`)) continue;
+
+        packages.add(packageName);
+      }
+    }
+  }
+
+  return [...packages];
+};
+
+describe("runtime dependency safety", () => {
+  it("all runtime-resolved packages in dist are declared in package.json dependencies", () => {
+    const runtimePackages = extractRuntimeResolvedPackages();
+    const missing = runtimePackages.filter((pkg) => !declaredDeps.has(pkg));
+
+    if (missing.length > 0) {
+      throw new Error(
+        `Found runtime-resolved packages in dist/ not in dependencies, peerDependencies, or optionalDependencies:\n\n` +
+          missing.map((pkg) => `  - ${pkg}`).join("\n") +
+          `\n\nAdd them to "dependencies" in apps/cli/package.json ` +
+          `so consumers with strict node_modules (pnpm) can resolve them.`,
+      );
+    }
+  });
+});

apps/video-transcript/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@expect/video-transcript",
+  "version": "0.0.1",
+  "private": true,
+  "description": "Extract structured interaction transcripts from screen recordings",
+  "type": "module",
+  "bin": {
+    "video-transcript": "./dist/index.js"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "vp pack",
+    "dev": "vp pack --watch",
+    "lint": "vp lint && tsc --noEmit",
+    "format": "vp fmt",
+    "format:check": "vp fmt --check",
+    "check": "vp check",
+    "test": "vp test",
+    "typecheck": "tsgo --noEmit"
+  },
+  "dependencies": {
+    "@ai-sdk/gateway": "^3.0.88",
+    "ai": "^6.0.146",
+    "commander": "^13.1.0",
+    "effect": "4.0.0-beta.35",
+    "picocolors": "^1.1.1"
+  },
+  "devDependencies": {
+    "@types/node": "^22.15.0",
+    "typescript": "^5.7.0",
+    "vitest": "*"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}

apps/video-transcript/src/activity-analyzer.test.ts

@@ -0,0 +1,107 @@
+import { describe, it, expect } from "vitest";
+import { computeFrameDiff, classifySegments, formatTimeline } from "./activity-analyzer";
+
+describe("computeFrameDiff", () => {
+  it("returns 0 for identical frames", () => {
+    const frame = Buffer.from([0, 128, 255, 64]);
+    expect(computeFrameDiff(frame, frame)).toBe(0);
+  });
+
+  it("returns 1 for maximally different frames", () => {
+    const black = Buffer.from([0, 0, 0, 0]);
+    const white = Buffer.from([255, 255, 255, 255]);
+    expect(computeFrameDiff(black, white)).toBeCloseTo(1, 5);
+  });
+
+  it("returns 0 for empty buffers", () => {
+    expect(computeFrameDiff(Buffer.alloc(0), Buffer.alloc(0))).toBe(0);
+  });
+
+  it("computes correct diff for known values", () => {
+    const frameA = Buffer.from([0, 0, 0, 0]);
+    const frameB = Buffer.from([255, 0, 0, 0]);
+    expect(computeFrameDiff(frameA, frameB)).toBeCloseTo(0.25, 5);
+  });
+
+  it("handles frames of different sizes using the shorter length", () => {
+    const short = Buffer.from([0, 0]);
+    const long = Buffer.from([255, 255, 255, 255]);
+    expect(computeFrameDiff(short, long)).toBeCloseTo(1, 5);
+  });
+});
+
+describe("classifySegments", () => {
+  it("returns empty for no diffs", () => {
+    expect(classifySegments([])).toEqual([]);
+  });
+
+  it("classifies all-idle diffs as a single idle segment", () => {
+    const diffs = [0.001, 0.002, 0.001, 0.003, 0.002];
+    const result = classifySegments(diffs);
+    expect(result.length).toBe(1);
+    expect(result[0]!.type).toBe("idle");
+    expect(result[0]!.startSeconds).toBe(0);
+    expect(result[0]!.endSeconds).toBe(5);
+  });
+
+  it("classifies all-active diffs as a single active segment", () => {
+    const diffs = [0.05, 0.08, 0.06, 0.07];
+    const result = classifySegments(diffs);
+    expect(result.length).toBe(1);
+    expect(result[0]!.type).toBe("active");
+  });
+
+  it("detects scene changes from high diffs", () => {
+    const diffs = [0.01, 0.02, 0.5, 0.01, 0.02];
+    const result = classifySegments(diffs);
+    const sceneChanges = result.filter((s) => s.type === "scene_change");
+    expect(sceneChanges.length).toBeGreaterThanOrEqual(1);
+  });
+
+  it("merges short idle gaps between active segments", () => {
+    const diffs = [0.05, 0.06, 0.07, 0.001, 0.002, 0.05, 0.06, 0.07];
+    const result = classifySegments(diffs);
+    const idleSegments = result.filter((s) => s.type === "idle");
+    expect(idleSegments.length).toBe(0);
+    expect(result.every((s) => s.type === "active")).toBe(true);
+  });
+
+  it("preserves long idle gaps between active segments", () => {
+    const diffs = [0.05, 0.06, 0.001, 0.002, 0.001, 0.002, 0.05, 0.06];
+    const result = classifySegments(diffs);
+    const idleSegments = result.filter((s) => s.type === "idle");
+    expect(idleSegments.length).toBeGreaterThanOrEqual(1);
+  });
+});
+
+describe("formatTimeline", () => {
+  it("formats a simple timeline", () => {
+    const timeline = [
+      { type: "idle" as const, startSeconds: 0, endSeconds: 3 },
+      { type: "active" as const, startSeconds: 3, endSeconds: 10 },
+      { type: "idle" as const, startSeconds: 10, endSeconds: 12 },
+    ];
+    const result = formatTimeline(timeline);
+    expect(result).toContain("00:00");
+    expect(result).toContain("00:03");
+    expect(result).toContain("active");
+    expect(result).toContain("idle");
+  });
+
+  it("formats scene changes", () => {
+    const timeline = [{ type: "scene_change" as const, startSeconds: 5, endSeconds: 6 }];
+    const result = formatTimeline(timeline);
+    expect(result).toContain("scene change (likely navigation)");
+  });
+
+  it("formats times with minutes", () => {
+    const timeline = [{ type: "active" as const, startSeconds: 65, endSeconds: 130 }];
+    const result = formatTimeline(timeline);
+    expect(result).toContain("01:05");
+    expect(result).toContain("02:10");
+  });
+
+  it("returns empty string for empty timeline", () => {
+    expect(formatTimeline([])).toBe("");
+  });
+});