feat(cli): Add init command to generate baseline eval config

packages/docs/src/content/docs/docs.mdx

@@ -36,6 +36,15 @@ Keep evals on their own command and Vitest config. The separate config keeps
 longer provider timeouts, eval-only includes, reporter setup, and replay
 defaults out of unit tests.
 
+Run the `init` command to generate the baseline config and add the eval scripts
+to `package.json` automatically:
+
+```bash
+pnpm exec vitest-evals init
+```
+
+Or add the files by hand:
+
 ```json title="package.json"
 {
   "scripts": {

packages/vitest-evals/README.md

@@ -26,6 +26,19 @@ For GitHub Actions summaries and annotations, emit Vitest JSON and use the
 native `getsentry/vitest-evals` action. No extra npm package is needed in the
 workflow.
 
+## Init
+
+Run `init` to generate the baseline Vitest config and add eval scripts to
+`package.json`:
+
+```sh
+pnpm exec vitest-evals init
+```
+
+This creates `vitest.evals.config.ts` and adds `evals` and `evals:record`
+scripts. Rerunning is safe — existing identical content is left unchanged.
+Pass `--force` to overwrite conflicting config or scripts.
+
 ## Core Model
 
 - `describeEval(...)` binds exactly one harness to a suite

packages/vitest-evals/package.json

@@ -17,9 +17,10 @@
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",
-  "files": [
-    "dist"
-  ],
+  "bin": {
+    "vitest-evals": "./dist/cli.js"
+  },
+  "files": ["dist"],
   "exports": {
     ".": {
       "source": "./src/index.ts",

packages/vitest-evals/src/cli.ts

@@ -0,0 +1,92 @@
+#!/usr/bin/env node
+import { runInitCommand } from "./cli/init";
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exitCode = 1;
+});
+
+async function main() {
+  const args = process.argv.slice(2);
+
+  if (args[0] === "--help" || args[0] === "-h" || args.length === 0) {
+    console.log(usage());
+    return;
+  }
+
+  const command = args[0];
+
+  if (command === "init") {
+    await dispatchInit(args.slice(1));
+    return;
+  }
+
+  console.error(`Unknown command: ${command}`);
+  console.error(usage());
+  process.exitCode = 1;
+}
+
+async function dispatchInit(args: string[]) {
+  const options = parseInitArgs(args);
+
+  if (options.help) {
+    console.log(initUsage());
+    return;
+  }
+
+  await runInitCommand({ cwd: options.cwd, force: options.force });
+}
+
+function parseInitArgs(args: string[]) {
+  let force = false;
+  let cwd: string | undefined;
+  let help = false;
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    switch (arg) {
+      case "--force":
+        force = true;
+        break;
+      case "--cwd": {
+        const value = args[++i];
+        if (!value) throw new Error("Missing value for --cwd");
+        cwd = value;
+        break;
+      }
+      case "--help":
+      case "-h":
+        help = true;
+        break;
+      default:
+        throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+
+  return { force, cwd, help };
+}
+
+function usage() {
+  return [
+    "Usage: vitest-evals <command>",
+    "",
+    "Commands:",
+    "  init    Generate a baseline eval config and add scripts to package.json",
+    "",
+    "Options:",
+    "  -h, --help    Print help",
+  ].join("\n");
+}
+
+function initUsage() {
+  return [
+    "Usage: vitest-evals init [options]",
+    "",
+    "Generate vitest.evals.config.ts and add eval scripts to package.json.",
+    "",
+    "Options:",
+    "  --force          Overwrite existing config and conflicting scripts",
+    "  --cwd <dir>      Target project directory (default: current directory)",
+    "  -h, --help       Print help",
+  ].join("\n");
+}

packages/vitest-evals/src/cli/init.test.ts

@@ -0,0 +1,182 @@
+import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { EVALS_CONFIG_CONTENT, EVALS_SCRIPTS, runInit } from "./init";
+
+function makeTmpDir() {
+  const dir = join(
+    tmpdir(),
+    `vitest-evals-init-test-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+  );
+  mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+function writePkg(dir: string, pkg: Record<string, unknown>) {
+  writeFileSync(join(dir, "package.json"), `${JSON.stringify(pkg, null, 2)}\n`);
+}
+
+function readPkg(dir: string): Record<string, unknown> {
+  return JSON.parse(readFileSync(join(dir, "package.json"), "utf8")) as Record<
+    string,
+    unknown
+  >;
+}
+
+function readConfig(dir: string): string {
+  return readFileSync(join(dir, "vitest.evals.config.ts"), "utf8");
+}
+
+describe("runInit", () => {
+  let dir: string;
+
+  beforeEach(() => {
+    dir = makeTmpDir();
+  });
+
+  afterEach(() => {
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("returns no-package-json when package.json is missing", () => {
+    const result = runInit({ cwd: dir });
+    expect(result).toEqual({ status: "no-package-json" });
+  });
+
+  it("creates config and adds scripts on a fresh project", () => {
+    writePkg(dir, { name: "my-app" });
+
+    const result = runInit({ cwd: dir });
+
+    expect(result).toEqual({
+      status: "ok",
+      wrote: [
+        "vitest.evals.config.ts",
+        "package.json scripts.evals",
+        "package.json scripts.evals:record",
+      ],
+      skipped: [],
+    });
+
+    expect(readConfig(dir)).toBe(EVALS_CONFIG_CONTENT);
+
+    const pkg = readPkg(dir);
+    const scripts = pkg.scripts as Record<string, string>;
+    expect(scripts.evals).toBe(EVALS_SCRIPTS.evals);
+    expect(scripts["evals:record"]).toBe(EVALS_SCRIPTS["evals:record"]);
+  });
+
+  it("preserves existing scripts when adding new ones", () => {
+    writePkg(dir, { name: "my-app", scripts: { test: "vitest" } });
+
+    runInit({ cwd: dir });
+
+    const pkg = readPkg(dir);
+    const scripts = pkg.scripts as Record<string, string>;
+    expect(scripts.test).toBe("vitest");
+    expect(scripts.evals).toBe(EVALS_SCRIPTS.evals);
+  });
+
+  it("is idempotent on a second run", () => {
+    writePkg(dir, { name: "my-app" });
+
+    runInit({ cwd: dir });
+    const result = runInit({ cwd: dir });
+
+    expect(result).toEqual({
+      status: "ok",
+      wrote: [],
+      skipped: [
+        "vitest.evals.config.ts",
+        "package.json scripts.evals",
+        "package.json scripts.evals:record",
+      ],
+    });
+  });
+
+  it("returns conflict when config exists with different content", () => {
+    writePkg(dir, { name: "my-app" });
+    writeFileSync(join(dir, "vitest.evals.config.ts"), "// custom config\n");
+
+    const result = runInit({ cwd: dir });
+
+    expect(result).toEqual({
+      status: "conflict",
+      conflicts: ["vitest.evals.config.ts"],
+    });
+  });
+
+  it("returns conflict when scripts have different values", () => {
+    writePkg(dir, {
+      name: "my-app",
+      scripts: { evals: "vitest run --config other.config.ts" },
+    });
+
+    const result = runInit({ cwd: dir });
+
+    expect(result).toEqual({
+      status: "conflict",
+      conflicts: ["package.json scripts.evals"],
+    });
+  });
+
+  it("returns all conflicts when multiple things differ", () => {
+    writePkg(dir, {
+      name: "my-app",
+      scripts: {
+        evals: "custom",
+        "evals:record": "custom-record",
+      },
+    });
+    writeFileSync(join(dir, "vitest.evals.config.ts"), "// different\n");
+
+    const result = runInit({ cwd: dir });
+    expect(result.status).toBe("conflict");
+    if (result.status === "conflict") {
+      expect(result.conflicts).toHaveLength(3);
+    }
+  });
+
+  it("overwrites with --force even when conflicts exist", () => {
+    writePkg(dir, {
+      name: "my-app",
+      scripts: { evals: "custom", "evals:record": "custom-record" },
+    });
+    writeFileSync(join(dir, "vitest.evals.config.ts"), "// different\n");
+
+    const result = runInit({ cwd: dir, force: true });
+
+    expect(result.status).toBe("ok");
+    expect(readConfig(dir)).toBe(EVALS_CONFIG_CONTENT);
+
+    const pkg = readPkg(dir);
+    const scripts = pkg.scripts as Record<string, string>;
+    expect(scripts.evals).toBe(EVALS_SCRIPTS.evals);
+    expect(scripts["evals:record"]).toBe(EVALS_SCRIPTS["evals:record"]);
+  });
+
+  it("--force preserves unrelated package.json fields and scripts", () => {
+    writePkg(dir, {
+      name: "my-app",
+      version: "1.2.3",
+      scripts: { test: "vitest", evals: "old-evals" },
+      dependencies: { lodash: "^4.17.21" },
+    });
+
+    runInit({ cwd: dir, force: true });
+
+    const pkg = readPkg(dir);
+    expect(pkg.name).toBe("my-app");
+    expect(pkg.version).toBe("1.2.3");
+    expect((pkg.dependencies as Record<string, string>).lodash).toBe(
+      "^4.17.21",
+    );
+    expect((pkg.scripts as Record<string, string>).test).toBe("vitest");
+  });
+
+  it("throws on invalid package.json JSON", () => {
+    writeFileSync(join(dir, "package.json"), "not valid json");
+    expect(() => runInit({ cwd: dir })).toThrow(/invalid JSON/);
+  });
+});