From 3d65ff5a0e25399a7b87e8c75b6f8eb90f0ef2f8 Mon Sep 17 00:00:00 2001 From: zeshi-du Date: Thu, 9 Jul 2026 15:29:24 -0700 Subject: [PATCH] release: v0.3.0 Private-Snapshot-RevId: d22d9f18f5d9712921097d5829c28d056e3ffd31 --- .gitattributes | 6 + .github/PULL_REQUEST_TEMPLATE.md | 4 +- .github/workflows/ci-nudge.yml | 129 ++++++ .github/workflows/issue-triage.yml | 23 +- .github/workflows/pr-triage.yml | 189 +++++++++ .github/workflows/stale.yml | 64 +++ CHANGELOG.md | 47 ++- CONTRIBUTING.md | 48 ++- DOCUMENTATION.md | 190 +++++++-- README.md | 52 ++- package.json | 2 +- skills/testsprite-onboard.skill.md | 21 + skills/testsprite-verify.skill.md | 33 ++ src/commands/agent.test.ts | 50 ++- src/commands/agent.ts | 21 +- src/commands/init.test.ts | 17 +- src/commands/init.ts | 27 +- src/commands/project.test.ts | 268 ++++++++++++- src/commands/project.ts | 376 +++++++++++++++++- src/commands/test.test.ts | 72 ++++ src/commands/test.ts | 87 +++- src/commands/usage.ts | 6 +- src/index.ts | 2 +- src/lib/agent-targets.test.ts | 4 +- src/lib/bundle.test.ts | 6 +- src/lib/credentials.test.ts | 14 +- src/lib/http.ts | 6 +- src/lib/skill-nudge.test.ts | 18 +- src/version.ts | 2 +- test/__snapshots__/help.snapshot.test.ts.snap | 52 ++- test/cli.subprocess.test.ts | 12 +- test/helpers/hermetic-env.ts | 41 ++ vitest.config.ts | 3 + 33 files changed, 1696 insertions(+), 196 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/ci-nudge.yml create mode 100644 .github/workflows/pr-triage.yml create mode 100644 .github/workflows/stale.yml create mode 100644 test/helpers/hermetic-env.ts diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..468c8ed --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Check out all text files with LF on every platform. Tests compare bytes +# from checked-out files (skill templates, snapshots); a CRLF working tree +# (core.autocrlf on Windows) broke those comparisons. +* text=auto eol=lf + +*.png binary diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c490f9b..1c165f6 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,7 +12,9 @@ agree on the approach before you invest time — see CONTRIBUTING.md. ## Related issue - + ## Type of change diff --git a/.github/workflows/ci-nudge.yml b/.github/workflows/ci-nudge.yml new file mode 100644 index 0000000..d01ede3 --- /dev/null +++ b/.github/workflows/ci-nudge.yml @@ -0,0 +1,129 @@ +# "Fix this and we merge" nudge (InsForge `agent-zhang-beihai` pattern, part 3). +# When a PR's CI finishes red, posts ONE sticky comment listing exactly which +# jobs failed and the local one-liner that reproduces/fixes each (the automated +# version of the hand-written "run `npm run format` and you're green" review +# comments). The comment flips to a green confirmation once all checks pass. +# +# Runs in base-repo context via workflow_run — no PR code is checked out, so +# fork PRs are safe. State is recomputed from check-runs each time, so the two +# CI workflows (CI + Test Coverage) can complete in any order. +# +# Bot identity: same App-token-first / GITHUB_TOKEN-fallback pattern as +# pr-triage.yml (the App needs the "Pull requests: Read & write" permission to +# post as testsprite-hob[bot]; until then comments come from github-actions[bot]). +name: CI failure nudge + +on: + workflow_run: + workflows: ['CI', 'Test Coverage'] + types: [completed] + +permissions: + checks: read # read the head SHA's check-run state + pull-requests: write + issues: write # PR comments ride the issues API + +env: + MARKER: '' + +jobs: + nudge: + # Public repo only; PR-triggered runs only (pushes to main have no PR to nudge). + if: >- + github.repository == 'TestSprite/testsprite-cli' && + github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + steps: + - id: app-token + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + with: + script: | + const { owner, repo } = context.repo; + const run = context.payload.workflow_run; + const marker = process.env.MARKER; + + const appClient = process.env.APP_TOKEN + ? require('@actions/github').getOctokit(process.env.APP_TOKEN) + : null; + async function write(fn) { + if (appClient) { + try { return await fn(appClient.rest); } + catch (e) { if (e.status !== 403 && e.status !== 404) throw e; } + } + return await fn(github.rest); + } + + // Resolve the PR for this run. `workflow_run.pull_requests` is empty + // for fork PRs, so fall back to the commit→PRs lookup. + let pr = (run.pull_requests || [])[0]; + if (!pr) { + const { data } = await github.rest.repos.listPullRequestsAssociatedWithCommit( + { owner, repo, commit_sha: run.head_sha }); + pr = data.find(p => p.state === 'open'); + } else { + pr = (await github.rest.pulls.get({ owner, repo, pull_number: pr.number })).data; + } + if (!pr || pr.state !== 'open' || pr.user.type === 'Bot') return; + + // Job name → the local command that reproduces/fixes it. Also the + // allowlist of CI jobs this nudge watches — keep in sync with ci.yml. + const FIX = { + 'Lint & Format': 'run `npm run lint:fix && npm run format`, then commit', + 'Typecheck': 'run `npm run typecheck` and fix the reported type errors', + 'Unit Tests': 'run `npm test` and fix the failing tests', + 'Build': 'run `npm run build` and fix the compile errors', + 'Local E2E Tests': 'run `npm run test:e2e` (it builds first)', + 'Coverage (>= 80%)': 'run `npm run test:coverage` — new code needs tests until every metric is back at 80%', + }; + + // Recompute full CI state from this SHA's check runs, narrowed to the + // CI jobs above — other workflows (e.g. pr-triage) also attach + // github-actions check runs to the PR head and must not count here. + const checks = await github.paginate(github.rest.checks.listForRef, + { owner, repo, ref: run.head_sha, per_page: 100 }); + const ours = checks.filter(c => c.app && c.app.slug === 'github-actions' && FIX[c.name]); + const failing = ours.filter(c => ['failure', 'timed_out'].includes(c.conclusion)); + const pending = ours.filter(c => c.status !== 'completed'); + + const comments = await github.paginate(github.rest.issues.listComments, + { owner, repo, issue_number: pr.number, per_page: 100 }); + const sticky = comments.find(c => (c.body || '').includes(marker)); + + if (failing.length === 0) { + // Only speak up on success if we previously flagged a failure, and + // only once everything has actually finished. + if (sticky && pending.length === 0 && !sticky.body.includes('all green')) { + await write(rest => rest.issues.updateComment({ owner, repo, comment_id: sticky.id, + body: `${marker}\n✅ CI is **all green** now — thanks, @${pr.user.login}!` })); + } + return; + } + + const lines = failing + .sort((a, b) => a.name.localeCompare(b.name)) + .map(c => { + const fix = FIX[c.name] || 'see the logs for details'; + return `- **${c.name}** — ${fix} ([logs](${c.html_url}))`; + }); + const body = `${marker}\nThanks, @${pr.user.login}! CI is red on this PR — ` + + `here's what failed and how to reproduce it locally:\n\n${lines.join('\n')}\n\n` + + `Everything runs on Node 22 after \`npm ci\`. Push a fix and this comment ` + + `flips green automatically once all checks pass.`; + + if (sticky) { + if (sticky.body !== body) { + await write(rest => rest.issues.updateComment( + { owner, repo, comment_id: sticky.id, body })); + } + } else { + await write(rest => rest.issues.createComment( + { owner, repo, issue_number: pr.number, body })); + } diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index 63323c0..6b6ce62 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -1,13 +1,15 @@ # P1-2 issue auto-assign + 3-slot-cap bot (InsForge `agent-zhang-beihai` pattern). -# Posts as the `alfheim-agent` GitHub App ⇒ `alfheim-agent[bot]`. +# Posts as the `testsprite-hob` GitHub App ⇒ `testsprite-hob[bot]`. # # Setup (one-time, by an org admin): -# 1. Create a GitHub App named `alfheim-agent` (org Settings → Developer settings → +# 1. Create a GitHub App named `testsprite-hob` (org Settings → Developer settings → # GitHub Apps → New). Permissions: Issues = Read & write, Metadata = Read. No webhook. # Generate a private key; install the App on the public testsprite-cli repo. # 2. Add two repo (or org) secrets: -# ALFHEIM_AGENT_APP_ID = the App's numeric App ID -# ALFHEIM_AGENT_PRIVATE_KEY = the App's .pem private key (full contents) +# TESTSPRITE_HOB_APP_ID = the App's numeric App ID +# TESTSPRITE_HOB_PRIVATE_KEY = the App's .pem private key (full contents) +# (The ALFHEIM_AGENT_* fallbacks are this App's original secret names from +# before it was renamed — same App ID + key; either naming works.) # Until those exist, the bot gracefully falls back to github-actions[bot] (still works). # # Fires on each new issue comment; assigns the commenter when they claim an issue @@ -30,23 +32,26 @@ env: jobs: triage: # issues only (issue_comment also fires on PRs), and never react to a bot's own - # comment — incl. our own alfheim-agent[bot], which (unlike GITHUB_TOKEN) would + # comment — incl. our own testsprite-hob[bot], which (unlike GITHUB_TOKEN) would # otherwise re-trigger this workflow. `type == 'Bot'` covers both bot identities. if: ${{ !github.event.issue.pull_request && github.event.comment.user.type != 'Bot' }} runs-on: ubuntu-latest steps: - # Mint a token for the alfheim-agent App so comments post as alfheim-agent[bot]. + # Mint a token for the testsprite-hob App so comments post as testsprite-hob[bot]. # continue-on-error: until the App + secrets exist this no-ops and we fall back below. - id: app-token continue-on-error: true uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: - app-id: ${{ secrets.ALFHEIM_AGENT_APP_ID }} - private-key: ${{ secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + # Scope the minted token to what this job uses (issues API only) instead + # of inheriting the App's full installation permissions. + permission-issues: write - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 with: - # App token when available (→ alfheim-agent[bot]); else the default + # App token when available (→ testsprite-hob[bot]); else the default # GITHUB_TOKEN (→ github-actions[bot]). Either way the logic is identical. github-token: ${{ steps.app-token.outputs.token || github.token }} script: | diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml new file mode 100644 index 0000000..a9cd23d --- /dev/null +++ b/.github/workflows/pr-triage.yml @@ -0,0 +1,189 @@ +# PR-side twin of issue-triage.yml (InsForge `agent-zhang-beihai` pattern, part 2). +# Enforces the issue-first workflow on community PRs: every non-docs PR must +# carry a closing link ("Closes #123") to an issue that is ASSIGNED to the PR +# author (claimed via `/assign`, which issue-triage.yml handles). Violations +# get a `needs-issue` label + a sticky comment, and — for PRs opened after +# GATE_SINCE — a failing check, so unclaimed work is visibly not review-ready. +# PRs opened before GATE_SINCE are grandfathered: nudge only, never a red check. +# +# Assignment happens on the ISSUE, so it cannot re-trigger this PR workflow; +# the comment tells the contributor to edit the PR description or push a +# commit (`edited` / `synchronize`) to re-run the gate. +# +# Runs with NO checkout — metadata-only, so it is safe under pull_request_target +# (which is required for fork PRs to get a write-capable token). +# +# Bot identity: posts as `testsprite-hob[bot]` when the App token works. The +# App currently has Issues R/W + Metadata only — commenting/labeling a PULL +# REQUEST needs the "Pull requests: Read & write" App permission (issues-API +# endpoints are permission-checked by target type). Until an org admin adds +# that permission and re-approves the installation, every call gracefully falls +# back to the default GITHUB_TOKEN and posts as github-actions[bot]. +# Secrets: TESTSPRITE_HOB_* preferred; the ALFHEIM_AGENT_* fallbacks are the +# original names from before the App was renamed (same App ID + key). +name: PR triage (issue-link gate) + +on: + pull_request_target: + types: [opened, edited, reopened, synchronize] + +permissions: + pull-requests: write # add/remove the needs-issue label + issues: write # create/update the nudge comment (PR comments ride the issues API) + +env: + LABEL: 'needs-issue' + MARKER: '' + # PRs created before this instant are grandfathered (nudge, no failing check). + GATE_SINCE: '2026-07-04T00:00:00Z' + +jobs: + gate: + # Public repo only — this file also lives in the private mirror, where PRs + # are internal work that never links public issues. Skip bot authors + # (dependabot etc.), maintainers, and docs-only changes (CONTRIBUTING + # exempts docs/small fixes from the issue-first ask). + if: >- + github.repository == 'TestSprite/testsprite-cli' && + github.event.pull_request.state == 'open' && + github.event.pull_request.user.type != 'Bot' && + !contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association) && + !startsWith(github.event.pull_request.title, 'docs') + runs-on: ubuntu-latest + steps: + # Mint an App token so actions post as testsprite-hob[bot]. Until the App + + # secrets + Pull-requests permission exist this no-ops / gets 403 and the + # script below falls back to the default token per-call. + - id: app-token + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + with: + # `github` client = default GITHUB_TOKEN: used for all reads (the App + # can't read PRs without the Pull-requests permission) and as the + # write fallback. App client (when mintable) is preferred for writes. + script: | + const { owner, repo } = context.repo; + const pr = context.payload.pull_request; + const label = process.env.LABEL; + const marker = process.env.MARKER; + const author = pr.user.login; + + const appClient = process.env.APP_TOKEN + ? require('@actions/github').getOctokit(process.env.APP_TOKEN) + : null; + // Prefer the App identity; fall back to github-actions[bot] when the + // App is absent or lacks the Pull-requests permission (403/404). + async function write(fn) { + if (appClient) { + try { return await fn(appClient.rest); } + catch (e) { if (e.status !== 403 && e.status !== 404) throw e; } + } + return await fn(github.rest); + } + + // Linked issues: GitHub-computed closing references carry number + + // assignees in one query. + const gql = await github.graphql( + `query($owner:String!,$repo:String!,$num:Int!){ + repository(owner:$owner,name:$repo){ + pullRequest(number:$num){ + closingIssuesReferences(first:10){ + nodes{ number assignees(first:10){ nodes{ login } } } + } + } + } + }`, { owner, repo, num: pr.number }); + const linkedIssues = gql.repository.pullRequest.closingIssuesReferences.nodes + .map(n => ({ number: n.number, assignees: n.assignees.nodes.map(a => a.login) })); + + // Body-text fallback for the brief window before GitHub computes the + // link (same-repo bare "#N" references only). + const bodyRefs = [...(pr.body || '') + .matchAll(/(close[sd]?|fix(e[sd])?|resolve[sd]?)\s*:?\s+#(\d+)/gi)] + .map(m => Number(m[3])) + .filter(n => !linkedIssues.some(i => i.number === n)) + .slice(0, 5); + for (const num of bodyRefs) { + try { + const { data } = await github.rest.issues.get({ owner, repo, issue_number: num }); + if (data.pull_request) continue; // "#N" pointed at a PR, not an issue + linkedIssues.push({ number: num, assignees: (data.assignees || []).map(a => a.login) }); + } catch (e) { /* unknown number — ignore */ } + } + + const linked = linkedIssues.length > 0; + const assignedToAuthor = linkedIssues.some(i => i.assignees.includes(author)); + const state = assignedToAuthor ? 'ok' : (linked ? 'unassigned' : 'unlinked'); + + const hasLabel = (pr.labels || []).some(l => l.name === label); + const comments = await github.paginate(github.rest.issues.listComments, + { owner, repo, issue_number: pr.number, per_page: 100 }); + const nudge = comments.find(c => (c.body || '').includes(marker)); + const stateLine = ``; + + const contributingUrl = + `https://github.com/${owner}/${repo}/blob/main/CONTRIBUTING.md#contribution-model`; + const rerunHint = 'After fixing it, edit the PR description or push a commit to re-run this check.'; + + let body; + if (state === 'ok') { + body = `${marker}\n${stateLine}\n` + + `✅ This PR is linked to an issue assigned to @${author} — thanks! ` + + `The \`${label}\` label has been removed.`; + } else if (state === 'unassigned') { + const list = linkedIssues.map(i => { + const holders = i.assignees.filter(a => a !== author); + return `#${i.number}` + (holders.length ? ` (currently assigned to @${holders.join(', @')})` : ' (unassigned)'); + }).join(', '); + body = `${marker}\n${stateLine}\n` + + `Thanks for the PR, @${author}! It links an issue, but that issue isn't assigned to you yet: ${list}. ` + + `Per our workflow, **claim the issue first by commenting \`/assign\` on it** (the triage bot assigns you automatically). ` + + `If it's already assigned to someone else, please coordinate with them or pick another issue — ` + + `unclaimed-issue PRs are not reviewed. ${rerunHint} ` + + `See [CONTRIBUTING → Contribution model](${contributingUrl}).`; + } else { + body = `${marker}\n${stateLine}\n` + + `Thanks for the PR, @${author}! A quick note on our workflow: for **features and behavior changes** ` + + `we require contributors to **open an issue first, claim it by commenting \`/assign\` on the issue, ` + + `then submit a PR that links it** (e.g. \`Closes #123\`). This PR isn't linked to any issue yet, ` + + `so it is not review-ready. ${rerunHint} ` + + `See [CONTRIBUTING → Contribution model](${contributingUrl}).`; + } + + // Sticky comment: create once, update when the state changes. + if (!nudge) { + if (state !== 'ok') { + await write(rest => rest.issues.createComment( + { owner, repo, issue_number: pr.number, body })); + } + } else if (!nudge.body.includes(stateLine)) { + await write(rest => rest.issues.updateComment( + { owner, repo, comment_id: nudge.id, body })); + } + + // Label tracks the gate state. + if (state === 'ok' && hasLabel) { + await write(rest => rest.issues.removeLabel( + { owner, repo, issue_number: pr.number, name: label })).catch(() => {}); + } + if (state !== 'ok' && !hasLabel) { + await write(rest => rest.issues.addLabels( + { owner, repo, issue_number: pr.number, labels: [label] })); + } + + // Hard gate for PRs opened after the cutoff; older PRs are nudged only. + const gated = new Date(pr.created_at) >= new Date(process.env.GATE_SINCE); + if (state !== 'ok' && gated) { + core.setFailed(state === 'unlinked' + ? 'No closing-linked issue. Open/claim an issue, add "Closes #" to the PR description, then re-run.' + : 'Linked issue is not assigned to the PR author. Comment /assign on the issue, then re-run.'); + } else if (state !== 'ok') { + core.notice(`Grandfathered PR (opened before ${process.env.GATE_SINCE}): gate not enforced, nudge only.`); + } diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..fbaa607 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,64 @@ +# P1-3 — stale-bot: nudge then close inactive issues / PRs (actions/stale). +# +# Generous windows for an open-source repo (first-response SLA is 5 business days, +# see CONTRIBUTING). Activity removes the `stale` label automatically, so a single +# reply resets the clock. Newcomer- and security-relevant work is exempt from +# closing. `good first issue` / `help wanted` stay open for whoever picks them up. +# +# This is a SYNCED asset (ships to the public mirror). It is scheduled, so unlike +# the event-gated triage bot it is fenced to the PUBLIC repo with a repository +# guard — on private atlas it is a no-op (no nagging internal issues/PRs). +name: Stale + +on: + schedule: + - cron: '30 1 * * *' # daily 01:30 UTC + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + stale: + if: ${{ github.repository == 'TestSprite/testsprite-cli' }} + runs-on: ubuntu-latest + permissions: + issues: write # comment + (un)label + close stale issues + pull-requests: write # comment + (un)label + close stale PRs + steps: + - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 + with: + # ── issues ────────────────────────────────────────────────────── + days-before-issue-stale: 60 + days-before-issue-close: 14 + stale-issue-label: stale + stale-issue-message: > + This issue has had no activity for 60 days, so it's been marked + `stale`. If it's still relevant, just leave a comment (or remove the + `stale` label) and we'll keep it open — otherwise it will be closed + in 14 days. Thanks for helping us keep the tracker tidy! + close-issue-message: > + Closing as `stale` after no activity. This isn't a judgement on the + idea — please reopen or open a fresh issue if it's still relevant. + + # ── pull requests (more grace — external contributors may be slow) ─ + days-before-pr-stale: 45 + days-before-pr-close: 21 + stale-pr-label: stale + stale-pr-message: > + This PR has had no activity for 45 days, so it's been marked `stale`. + Push a commit or leave a comment to keep it open — otherwise it will + be closed in 21 days. We'd still love to merge it; ping a maintainer + if you're blocked on a review. + close-pr-message: > + Closing as `stale` after no activity. Reopen any time you can pick it + back up — your work isn't lost. + + # ── shared behaviour ──────────────────────────────────────────── + exempt-issue-labels: 'pinned,security,in-progress,good first issue,help wanted,hackathon' + exempt-pr-labels: 'pinned,security,in-progress,hackathon' + exempt-draft-pr: true + remove-stale-when-updated: true + ascending: true # oldest first — fairest under the per-run op cap + operations-per-run: 60 + enable-statistics: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f1a0a8..e78347e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,48 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] +## [0.3.0] - 2026-07-08 + ### Added -- **JUnit XML report export for batch `--wait` runs.** `test run --all` and batch `test rerun` (`--all` or multiple test ids) accept `--report junit --report-file ` to write a CI-friendly XML sidecar after polling completes. `--output json` is unchanged; the report is written even when the batch exits non-zero. `--dry-run` writes a canned sample without network calls. -- **`testsprite test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs `, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal OFF (strict verbatim) so a healed drift can't mask a nondeterministic pass/fail. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness (`testsprite test flaky --runs 5 || exit 1`). Flags: `--runs ` (1–10), `--until-fail` (stop at the first non-passing attempt), `--timeout ` (per-attempt), and `--output json` for a machine-readable stability report. Frontend replays are free verbatim script replays; a one-line advisory is printed for backend tests, whose closure reruns may cost credits. +- **`testsprite doctor`** — one-command environment diagnostic that checks your Node version, credentials, endpoint reachability, and installed agent skills, and reports what's misconfigured. +- **`test scaffold`** — emit a schema-correct starter plan (frontend) or a backend test skeleton to bootstrap a new test without hand-writing the JSON. +- **`test lint`** — offline validator for plan / steps files; catches malformed test definitions before they are sent to the server. +- **`test diff `** — compare two runs of the same test to isolate what changed between a passing and a failing run. +- **`test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs`, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal off (strict verbatim) so a nondeterministic pass/fail can't be masked. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness. Flags: `--runs` (1–10), `--until-fail`, `--timeout`, `--output json`. +- **JUnit XML report export for batch runs.** `test run --all` and batch `test rerun` accept `--report junit --report-file ` to write a CI-friendly XML sidecar after `--wait` polling completes. The report is written even when the batch exits non-zero; `--output json` is unchanged; `--dry-run` writes a canned sample without network calls. +- **`test wait` is now variadic** — pass several run ids to attach to and poll multiple runs in a single invocation. +- **`agent status`** — report which TestSprite skills are installed for each agent target and whether they are current; installed skills are now stamped with a version/hash marker. +- **New `agent install` targets:** GitHub Copilot, Windsurf, and Kiro (experimental), alongside the existing Claude / Cursor / Cline / Codex / Antigravity targets. +- **`project credential` / `project auto-auth`** — configure a project's backend credentials (static credential, free) or a recurring auto-auth token (Pro) from the CLI, with surfaced auth warnings and managed-credential guidance. +- **Proxy support** — the CLI now honors `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` for use behind corporate and CI proxies. +- **`NO_COLOR` support** — colored output is suppressed when `NO_COLOR` is set, per no-color.org. +- **"New version available" notice** — a non-blocking, 24h-cached npm version check prints an upgrade hint on stderr. Opt out with the documented env var; automatically silenced in CI and under `--output json`. + +### Changed + +- **Node.js 20.19+, 22.13+, or 24+ is now the minimum supported runtime.** The CLI checks the running Node version at startup and exits with a clear message on an unsupported version; builds and CI run against Node 20 and 22. +- **Graceful shutdown** — the CLI handles termination signals cleanly and guards against broken-pipe (`EPIPE`) errors when its output is piped to a closing consumer (e.g. `| head`). +- **Interactive prompts and preamble now go to stderr**, keeping stdout pure for machine consumers even in interactive mode. +- **Empty environment variables are treated as unset** when resolving config, so `TESTSPRITE_API_URL=` no longer overrides the built-in default with an empty string. +- `agent install` defaults `--target` to `claude` in non-interactive / CI contexts (matching `setup`). +- The `usage` command no longer implies backend test runs are free. +- `setup`'s "Next steps" guidance no longer suggests `test list` before any project exists. + +### Fixed + +- **Timeouts & polling:** `RequestTimeoutError` is now classified as a timeout in the `--all --wait` fan-out; per-attempt timeout timers are cleared so they can't fire late; `run --all --wait` no longer polls still-queued runs past the shared deadline; a partial result is emitted on stdout when `run --wait` / `test wait` times out (so a redirected file is never zero-byte). +- **Batch rerun:** the exit code is preserved and auth errors escalate correctly; explicit ids combined with `--all` — or `--status` / `--skip-terminal` without `--all` — are rejected with a clear validation error; auto-minted idempotency keys are surfaced under `--output json`. +- **HTTP:** non-JSON `200` responses map to a typed error envelope instead of crashing the parser. +- **Failure bundles / artifacts:** artifact downloads retry on transient errors and guard the default run-id path; the `--out` directory no longer sweeps unrelated pre-existing files (data-loss fix); run-scoped per-step error text and step type are surfaced. +- **Input validation (fail fast, before any network call):** malformed API keys, invalid `--request-timeout`, directory `--code-file` / `--out` paths, blank or whitespace-only `--name` (test and project create/update), blank inline project passwords, fractional pagination flags / page sizes, and `--since` overflow are all rejected up front with `VALIDATION_ERROR` rather than crashing or failing late server-side. `--output` is validated uniformly across all command groups. +- **Setup / auth:** the endpoint is validated before the key check; the typed API-error envelope is preserved when key verification fails; the per-request timeout is honored during `configure`. +- **Misc:** cursor pagination no longer drops empty pages; trailing-dot hostnames are treated as loopback by the local-target guard; buffered input is preserved between interactive prompts; the Codex managed-section skill check requires a complete section; `code get` strips a leading BOM and rejects an empty `--out`. + +### Security + +- **INI injection:** CR/LF characters are stripped from credential values before they are written to `~/.testsprite/credentials`. +- **Symlink fail-close:** the own-file `agent install` path applies its symlink containment guard under `--dry-run` as well, so a planted symlink cannot place or clobber files outside `--dir`. ## [0.2.0] - 2026-06-29 @@ -149,6 +187,9 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for - Commander `help [command]` exits 0 (previously exited 5 on `test help` / `project help`). -[Unreleased]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.1...HEAD +[Unreleased]: https://github.com/TestSprite/testsprite-cli/compare/v0.3.0...HEAD +[0.3.0]: https://github.com/TestSprite/testsprite-cli/compare/v0.2.0...v0.3.0 +[0.2.0]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.2...v0.2.0 +[0.1.2]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.1...v0.1.2 [0.1.1]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.0...v0.1.1 [0.1.0]: https://github.com/TestSprite/testsprite-cli/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2110f0b..bed91fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,14 +20,54 @@ Discussions**, so the issue tracker stays a clean, actionable list. ## Contribution model -- **Small fixes and improvements:** just open a pull request. No issue required. -- **Large or breaking changes** (new commands, changed flags/output, new - dependencies, refactors): **open an issue first** to discuss the design. This - avoids wasted work on something we'd ask you to rework or that's out of scope. +- **Docs and small fixes** (typos, doc corrections, comment-only changes): + just open a pull request. No issue required (linking one is still + appreciated). +- **Features and behavior changes** (new commands or flags, changed output, + new dependencies, refactors) follow **issue-first**: + 1. Find an existing issue, or open a new one describing the change. + 2. Claim it by commenting `/assign` on the issue — the literal slash + command on its own line. The triage bot assigns you automatically; + free-text requests ("can I take this?") are **not** detected. + 3. If the issue is new, wait for triage — we check proposals against + [VISION.md](./VISION.md) and the [standing policies](#standing-scope-policies) + below before any code is written, so you don't invest in something + we'd ask you to rework or decline. + 4. Open your PR with a closing link (e.g. `Closes #123`) in the + description. +- **PR gate:** a bot checks every non-docs community PR for a closing-linked + issue that is **assigned to the PR author**. PRs that don't meet this get + the `needs-issue` label and a failing `PR triage` check, and **are not + reviewed** until it's fixed — file or claim the issue, add the closing + link, then edit the PR description (or push a commit) to re-run the check. +- Suspected **security vulnerabilities** are the exception to "file an + issue": report them privately per [SECURITY.md](./SECURITY.md) instead. - We **do** accept community code contributions — this is an actively maintained open-source CLI, not a read-only distribution mirror. - See [VISION.md](./VISION.md) for what is in and out of scope. +### Standing scope policies + +Pre-decided policies, so proposals don't have to relitigate them: + +- **Runtime dependencies are budgeted.** The CLI ships with a deliberately + tiny runtime dependency set (`commander`, `valibot`, plus `undici` — + approved for `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` support). Any new + runtime dependency needs explicit maintainer sign-off **in the issue, + before the PR**. Utility modules land only together with the consumer + that uses them — standalone libraries are declined. +- **`agent install` targets.** Shipped: `claude`, `antigravity`, `cursor`, + `cline`, `codex`, `kiro`, `windsurf`, `copilot`. Accepted and in progress: + `gemini`. + A proposal for a new target needs (1) the editor's official rules/skill + file mechanism, documented, and (2) the proposer prepared to maintain the + target going forward. +- **Outbound network calls.** The CLI talks only to the configured + TestSprite API endpoint. The one approved exception is an opt-out-able + npm registry version check (at most once per 24h, carrying nothing but + the package name, fully silenced by its env opt-out and in CI/JSON/dry-run + modes or when stderr is not a TTY). Any other outbound call is out of scope. + We aim to give every issue and PR a **first response within 5 business days** (best-effort). If something has gone quiet longer than that, a polite nudge on the thread or in Discord is welcome. diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 6b798ef..2f6cd6c 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -15,6 +15,7 @@ The full reference for the TestSprite CLI: install verification, manual setup, e - [Read commands](#read-commands) - [Write commands](#write-commands) - [Run commands](#run-commands) + - [Account & diagnostics](#account--diagnostics) - [Configuration](#configuration) - [Output & scripting](#output--scripting) - [Exit codes](#exit-codes) @@ -117,10 +118,15 @@ testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.m testsprite agent install kiro # .kiro/skills/testsprite-verify/SKILL.md testsprite agent install copilot # .github/instructions/testsprite-verify.instructions.md testsprite agent list # list all 8 targets with status + mode + path +testsprite agent status # check installed skills against this CLI version ``` Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental), `copilot` (experimental). +Omitting `--target` in a non-interactive shell (CI, agent subprocess) defaults to `claude` with an `[info]` note on stderr; in a terminal the CLI prompts (empty answer = `claude`). + +`agent status` checks every installed skill file against the current CLI version and reports one of `ok`, `stale`, `modified`, `unmarked`, `absent`, or `corrupt` per target. It exits `1` when anything needs attention, so `testsprite agent status && …` can gate a CI step; `--dir ` inspects a different project root. + The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf, copilot) backs up the existing file to `.bak` first. @@ -220,6 +226,15 @@ testsprite test result test_xxxxxxxx --history --source cli --since 7d --output testsprite test result test_xxxxxxxx --history --dry-run --output json ``` +#### `testsprite test diff ` + +Compare two runs of a test and print what regressed: verdict, `failureKind`, `failedStepIndex`, per-step status flips, and `codeVersion` drift. Exit `0` when the verdicts match, `1` when they differ — so a script can assert "this rerun behaves like the last known-good run" in one call. + +```bash +testsprite test diff run_aaaa run_bbbb --output json +testsprite test diff run_aaaa run_bbbb --dry-run --output json +``` + #### `testsprite test failure get ` The latest-failure agent entry point. Returns one consistent snapshot of the latest failing run as a self-contained bundle: the result, the failed step plus its immediate neighbors with screenshots and DOM snapshots, the test source, a video pointer, a root-cause hypothesis, a recommended fix target, and correlation metadata. For the bundle of a _specific_ run an agent just triggered, prefer `test artifact get ` — it is keyed by `runId` and cannot be raced by another run that lands afterward. @@ -253,7 +268,28 @@ testsprite test failure summary test_xxxxxxxx --dry-run --output json ### Write commands -Require the `write:tests` scope. +Require the `write:tests` scope (project commands require `write:projects`), except `test scaffold` and `test lint`, which are pure-local authoring helpers — no network, no credentials, no scope. + +#### `testsprite test scaffold` + +Emit a schema-correct starter test definition — a frontend plan JSON by default, or a backend Python skeleton with `--type backend`. Pure-local: no network, no credentials. Edit the scaffold, then create the test with `--plan-from` / `--code-file`. + +```bash +testsprite test scaffold > first-test.plan.json +testsprite test scaffold --type backend --out tests/health.py +testsprite test scaffold --out plan.json --force # overwrite an existing file +``` + +#### `testsprite test lint` + +Validate plan/steps files offline with the same validators `test create` runs, collecting **every** problem instead of stopping at the first. No network, no credentials. Exit `0` when all inputs are valid, `5` otherwise. + +```bash +testsprite test lint --plan-from ./checkout.plan.json +testsprite test lint --plan-from-dir ./plans/ # every *.json checked, all errors reported +testsprite test lint --plans ./plans.jsonl # one plan spec per line +testsprite test lint --steps ./refined.plan.json # the shape `test plan put` ingests +``` #### `testsprite test create` @@ -292,7 +328,7 @@ testsprite test update test_xxxxxxxx --dry-run --output json #### `testsprite test delete ` / `test delete-batch` -Soft-delete one test (or many). `--confirm` is required; absent it, the CLI exits 5 with a local validation error. +Permanently delete one test (or many) — there is **no restore window**. `--confirm` is required; absent it, the CLI exits 5 with a local validation error. ```bash testsprite test delete test_xxxxxxxx --confirm @@ -322,20 +358,66 @@ testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --dry-run --o #### `testsprite project create` / `project update` -Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. +Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. Note the asymmetry: `--description` is **create-only** — `project update` accepts `--name`, `--url`, `--username`, `--password`, `--password-file`, and `--instruction`, but not `--description`. ```bash testsprite project create --type frontend --name "Checkout" --url https://staging.example.com testsprite project update proj_xxxxxxxx --name "Checkout v2" ``` +#### `testsprite project credential ` + +Set the **static backend credential** injected into every backend test in the project (free tier). Supported types: `public` (no credential), `"Bearer token"`, `"API key"`, `"basic token"`. + +```bash +testsprite project credential proj_xxxxxxxx --type "Bearer token" --credential-file ./token.txt +testsprite project credential proj_xxxxxxxx --type public +testsprite project credential proj_xxxxxxxx --type "API key" --credential sk-live-... --dry-run --output json +``` + +`--credential ` or `--credential-file ` supplies the value (required unless `--type public`). Prefer `--credential-file` in scripts so the secret never lands in shell history. + +#### `testsprite project auto-auth ` + +Configure the **recurring-token (auto-refresh) login** for backend tests (Pro): a fresh token is fetched on each run and injected into every backend test, so long-lived suites survive token expiry. + +```bash +# Password login: POST the login endpoint, extract the token, inject as a Bearer header +testsprite project auto-auth proj_xxxxxxxx \ + --method password --inject bearer \ + --login-url https://api.example.com/login --login-method POST \ + --login-content-type application/json \ + --login-body-template '{"user":"{{username}}","pass":"{{password}}"}' \ + --username ci@example.com --password-file ./pw.txt \ + --token-path '$.data.accessToken' + +# OAuth refresh-token flow +testsprite project auto-auth proj_xxxxxxxx \ + --method refresh_token --inject header --inject-key X-Auth-Token \ + --token-endpoint https://auth.example.com/oauth/token \ + --client-id my-client --client-secret-file ./secret.txt \ + --refresh-token-file ./refresh.txt --scope api.read + +# AWS Cognito refresh +testsprite project auto-auth proj_xxxxxxxx \ + --method aws_cognito_refresh --inject bearer \ + --client-id my-app-client --refresh-token-file ./refresh.txt --region us-east-1 + +# Turn it off (stored config is kept) +testsprite project auto-auth proj_xxxxxxxx --disable +``` + +Required flags: `--method ` and `--inject ` (`--inject-key ` names the header/cookie when not `bearer`). Method-specific flags: password login uses `--login-url/--login-method/--login-content-type/--login-body-template/--username/--password[-file]/--token-path`; OAuth uses `--token-endpoint/--client-id/--client-secret[-file]/--refresh-token[-file]/--scope`; Cognito adds `--region`. File variants (`--password-file`, `--client-secret-file`, `--refresh-token-file`) keep secrets out of shell history. + ### Run commands Require the `run:tests` scope. #### `testsprite test run ` -Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", enqueuedAt, codeVersion, targetUrl }` and exits 0. With `--wait`, polls until terminal — exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on `--timeout` (with a `nextAction` pointing at `test wait ` so an agent can resume). +Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", enqueuedAt, codeVersion, targetUrl }` and exits 0. With `--wait`, polls until terminal — exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on `--timeout`. On a timeout the CLI still prints the partial run object (with `runId`) to stdout **before** exiting 7, plus a `nextAction` pointing at `test wait ` — so a script always has the id to resume with, and stdout is never empty. + +`--all --project ` runs every test in the project in wave order. On the current unified engine that means **all tests, frontend and backend**; on the legacy backend-only engine, frontend tests can't run — they are skipped and enumerated in `skippedFrontend` with a stderr advisory. ```bash # Trigger and return immediately @@ -348,7 +430,7 @@ testsprite test run test_xxxxxxxx --target-url https://staging.example.com \ # Dry-run prints a canned queued response (no network, no credentials) testsprite test run test_xxxxxxxx --dry-run --output json -# Batch BE run with JUnit XML for CI (sidecar; --output json unchanged) +# Batch run with JUnit XML for CI (sidecar; --output json unchanged) testsprite test run --all --project proj_xxxxxxxx --wait \ --report junit --report-file ./results.xml --output json @@ -427,16 +509,17 @@ Flags: `--output json` emits `{ testId, runs, passed, failed, stableRatio, verdict, failures: [{ attempt, runId, outcome, failureKind }] }`. Exit codes: **0** when every observed attempt passed (`stable`); **1** when any attempt did not pass (`flaky` or `failing`); **4** when the test has no replayable run (trigger `testsprite test run ` first); **5** on a validation error. -#### `testsprite test wait ` +#### `testsprite test wait ` -Block until a run reaches a terminal status. Same exit-code matrix as `test run --wait`. Used to resume polling after a timed-out `test run --wait`, or when an agent already has a `runId` from a previous invocation. +Block until one **or more** runs reach a terminal status. With a single `run-id` the behavior is unchanged: same exit-code matrix as `test run --wait`. With several ids, the runs are polled concurrently under one shared `--timeout` and the CLI prints a `{ results, summary }` envelope — the worst status wins the exit code — so every re-attach hint the CLI prints can be pasted back as one command. `--max-concurrency ` (1–100, default 10) caps concurrent polls. Used to resume polling after a timed-out `--wait`, or when an agent already holds `runId`s from previous invocations. ```bash testsprite test wait run_01hx3z9p8q4k2y7a --timeout 600 --output json +testsprite test wait run_aaaa run_bbbb run_cccc --timeout 900 --output json testsprite test wait run_01hx3z9p8q4k2y7a --dry-run --output json ``` -Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. +With several ids, a per-member poll error (e.g. one id not found) is recorded as `error:` in that run's row and folded into exit 7, rather than aborting the whole batch. Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. #### `testsprite test artifact get ` @@ -451,6 +534,30 @@ testsprite test artifact get run_01hx3z9p8q4k2y7a --dry-run --output json Returns 404 (CLI exit 4) when the run passed (`details.reason: "no_failing_run"`), is still in flight (`run_not_ready`), was cancelled (`cancelled_no_artifacts`), or its test was deleted (`no_code`). +### Account & diagnostics + +#### `testsprite usage` (alias: `testsprite credits`) + +Account pre-flight before a large batch: resolves the active key to its identity (`userId`, `keyId`, `env`) and surfaces the credit balance / plan fields when the backend supplies them. Useful right before a `test run --all` fan-out. + +```bash +testsprite usage --output json +testsprite credits +testsprite usage --dry-run --output json +``` + +#### `testsprite doctor` + +One-shot environment diagnostic. Runs a fixed checklist — CLI version, Node.js runtime, active profile, API endpoint, credentials, live connectivity + key validity (`GET /me`), and whether the verify skill is installed in the current project — and prints an OK/WARN/FAIL report. Exits non-zero only when a check **fails** (warnings, e.g. skill not installed, don't fail the process), so it can gate a CI step or an agent preflight: + +```bash +testsprite doctor +testsprite doctor --output json +testsprite doctor && testsprite test run test_xxxxxxxx --wait +``` + +Every check reuses the same helpers the real commands use, so the report reflects exactly what a subsequent command would resolve. + ## Configuration ### Profiles & credentials @@ -473,14 +580,17 @@ These apply to every command: ### Environment variables -| Variable | Purpose | -| ------------------------------- | --------------------------------------------------------------------------------------- | -| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | -| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | -| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | -| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | -| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | -| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | +| Variable | Purpose | +| ----------------------------------------- | ---------------------------------------------------------------------------------------- | +| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | +| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | +| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | +| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | +| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | +| `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` | Standard proxy support — API traffic is routed through the configured proxy | +| `TESTSPRITE_NO_SKILL_WARNING` | Any non-empty value silences the "verify skill not installed" reminder (CI / manual use) | +| `TESTSPRITE_PORTAL_URL` | Override the Portal origin used for `dashboardUrl` links (non-prod environments) | ### Update notice @@ -497,13 +607,14 @@ only outbound call the CLI makes besides your configured API endpoint. API-key scopes gate the write and run surfaces: -| Scope | Required by | -| --------------- | -------------------------------------------------------------------- | -| `read:me` | `auth whoami` | -| `read:projects` | `project list / get` | -| `read:tests` | every `test *` read command | -| `write:tests` | `test create / create-batch / update / delete / code put / plan put` | -| `run:tests` | `test run / rerun / wait / artifact get` | +| Scope | Required by | +| ---------------- | -------------------------------------------------------------------- | +| `read:me` | `auth status`, `usage`, `doctor` (connectivity check) | +| `read:projects` | `project list / get` | +| `read:tests` | every `test *` read command | +| `write:tests` | `test create / create-batch / update / delete / code put / plan put` | +| `write:projects` | `project create / update / credential / auto-auth` | +| `run:tests` | `test run / rerun / flaky / wait / artifact get` | New API keys include the full scope set. If a command returns `AUTH_FORBIDDEN`, the missing scope is named in `details.requiredScope` — regenerate your key from the dashboard to pick up new scopes. @@ -521,20 +632,25 @@ testsprite test wait "$RUN_ID" --timeout 600 --output json || echo "run did not ## Exit codes -| Code | Meaning | -| ---- | --------------------------------------- | -| `0` | Success | -| `1` | Generic failure / non-passed run status | -| `2` | Not yet implemented | -| `3` | Auth error | -| `4` | Not found | -| `5` | Validation error / payload too large | -| `6` | Conflict / precondition failed | -| `7` | Timeout / unsupported | -| `10` | Service unavailable | -| `11` | Rate limited (retriable) | -| `12` | Insufficient credits (non-retriable) | -| `13` | Feature gated (paid plan required) | +| Code | Meaning | +| --------------------- | --------------------------------------------------------------------------- | +| `0` | Success | +| `1` | Generic failure / non-passed run status | +| `2` | Not yet implemented | +| `3` | Auth error | +| `4` | Not found | +| `5` | Validation error / payload too large | +| `6` | Conflict / precondition failed | +| `7` | Timeout / unsupported | +| `10` | Service unavailable | +| `11` | Rate limited (retriable) | +| `12` | Insufficient credits (non-retriable) | +| `13` | Feature gated (paid plan required) | +| `129` / `130` / `143` | Interrupted by a signal (SIGHUP / SIGINT / SIGTERM) — `128 + signal number` | + +### Signals & pipes + +On SIGINT (Ctrl-C), SIGTERM, or SIGHUP the CLI prints `Interrupted (). Any run already started keeps executing on the server; check it with 'testsprite test list' or 'testsprite test wait '.` and exits `128 + signal`. **Ctrl-C does not cancel the server-side run** — execution (and any credit spend) continues; there is no cancel command today, so re-attach with `test wait ` instead of re-triggering. A closed stdout pipe (`EPIPE`, e.g. `testsprite test list | head`) exits `0` silently rather than crashing. ## Design principles diff --git a/README.md b/README.md index 404f237..78d293a 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,8 @@ TESTSPRITE_API_KEY=sk-... testsprite setup --from-env --yes --agent claude > **Pointing a coding agent (Claude Code, Cursor, Codex, Cline, …) at TestSprite?** Have it run `testsprite setup` first — that installs the verification skill, so the agent knows how to create, run, and triage tests on its own (instead of guessing from this README). New here? Start with the **[getting-started overview](https://docs.testsprite.com/cli/getting-started/overview)**. +> **Privacy note:** interactive runs check the npm registry at most once per 24 h to offer a "new version available" notice — package name only, never your key or data; `TESTSPRITE_NO_UPDATE_NOTIFIER=1` disables it. Details in [DOCUMENTATION.md → Update notice](./DOCUMENTATION.md#update-notice). + From there, the loop runs on its own — an example session, typed by the coding agent: ```bash @@ -89,28 +91,34 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- ## Commands -| Group | Command | What it does | -| --------- | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | -| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | -| | `auth remove` | Remove the active profile from the credentials file | -| **Read** | `project list` / `project get` | List projects / fetch one by id | -| | `test list` / `test get` | List tests under a project / fetch one by id | -| | `test code get` | Print (or write) the generated test source | -| | `test steps` | List the latest run's steps with screenshot / DOM pointers | -| | `test result` | Latest result; `--history` lists a test's prior runs | -| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | -| | `test failure summary` | One-screen triage card (no media download) | -| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | -| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | -| | `test code put` | Replace generated code (etag-guarded) | -| | `test plan put` | Replace a frontend test's plan-steps | -| | `project create` / `project update` | Manage projects | -| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | -| | `test wait` | Block on a `runId` until terminal | -| | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | +| Group | Command | What it does | +| --------- | --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | +| | `doctor` | Environment diagnostic — CLI/Node versions, profile, endpoint, credentials, connectivity, agent skill; exits non-zero on failure | +| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | +| | `auth remove` | Remove the active profile from the credentials file | +| | `usage` (alias `credits`) | Account pre-flight: identity, plus credit balance / plan info when the backend supplies them | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| | `test diff` | Compare two runs — verdict, failure kind, per-step status flips, code-version drift | +| **Write** | `test scaffold` / `test lint` | Author plans locally: emit a schema-correct starter, validate plan files offline — no network, no credentials | +| | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata / permanently delete (no restore window; `--confirm` required) | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` | Manage projects | +| | `project credential` / `project auto-auth` | Configure backend-test auth: a static injected credential, or auto-refresh login (Pro) | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test flaky` | Replay a test several times (auto-heal off) and report a stability score | +| | `test wait` | Block on one or more `runId`s until terminal | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` / `agent status` | Add, list, or health-check coding-agent skills (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). diff --git a/package.json b/package.json index 341e713..e78ce17 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.2.0", + "version": "0.3.0", "description": "Official TestSprite command-line interface", "type": "module", "main": "dist/index.js", diff --git a/skills/testsprite-onboard.skill.md b/skills/testsprite-onboard.skill.md index cc9342c..df04c18 100644 --- a/skills/testsprite-onboard.skill.md +++ b/skills/testsprite-onboard.skill.md @@ -91,6 +91,27 @@ Each file is a COMPLETE plan and must include `projectId` (from step 2), `type: **Backend** — one `.py` file per endpoint, using `requests` with concrete assertions on status code and response body. +**Backend auth — read the injected `__AUTH_HEADERS__`, NEVER hardcode any credential.** This +covers **every** secret the API needs — Bearer/JWT tokens **and** API keys (`sk-…`, +`x-api-key`), basic-auth blobs, cookies. TestSprite prepends a managed credential block +(`__AUTH_CREDENTIAL__` / `__AUTH_TYPE__` / `__AUTH_HEADERS__`) to every backend test from the +project's Authentication settings, and `__AUTH_HEADERS__` already holds the right header(s) for +the configured type (Bearer → `{"Authorization": "Bearer …"}`; API key → `{"X-API-Key": "…"}`; +basic → `{"Authorization": "Basic …"}`). Spread it into your request headers — never paste a +literal `Bearer …` / `sk-…` / key value into the script: + +```python +r = requests.get(f"{TARGET_URL}/orders", headers={**__AUTH_HEADERS__}) +``` + +Configure the credential once on the project (ask the user for the value — never invent it or +reuse a key you happened to see): a static credential with +`testsprite project credential --type "Bearer token"|"API key"|"basic token" --credential `, +or an auto-refreshing login with `testsprite project auto-auth …` so scheduled/repeat +runs keep working after the token expires. A hardcoded token expires within hours and a hardcoded +key can't be rotated centrally — `test create` emits a `[warn]` on an inlined credential; treat it +as a must-fix. + **Assertion rule (this is the whole game for FE):** every `assertion` step must name a **concrete, observable** outcome — an element, text, URL, count, or status. Never write `"verify it works"`, `"check the page loads"`, or other narrative that an AI judge can diff --git a/skills/testsprite-verify.skill.md b/skills/testsprite-verify.skill.md index 650a88b..fe7c661 100644 --- a/skills/testsprite-verify.skill.md +++ b/skills/testsprite-verify.skill.md @@ -165,6 +165,39 @@ only the Python **standard library + `requests` + `pytest` + `numpy` + `scipy`** - Get values from the API's responses (and captured variables), not by importing and calling the app's internals. +**Authentication — read the injected credential, NEVER hardcode any credential.** This +applies to **every** secret the API needs — Bearer/JWT tokens **and** API keys, basic-auth +blobs, session cookies. Do not paste a literal `Bearer …`, `sk-…`, `x-api-key` value, or any +other credential into the test. Before your script runs, TestSprite prepends a managed +credential block built from the project's Authentication settings, and `__AUTH_HEADERS__` +already contains the right header(s) for the configured auth type: + +```python +# Auto-injected credentials — do not modify +__AUTH_CREDENTIAL__ = "..." +__AUTH_TYPE__ = "Bearer token" # or "API key" / "basic token" / "public" +__AUTH_HEADERS__ = {"Authorization": "Bearer ..."} # API key → {"X-API-Key": "..."}; basic → {"Authorization": "Basic ..."} +``` + +Spread `__AUTH_HEADERS__` into every authenticated request — it adapts to whatever auth type +the project is configured for, so the same line works for Bearer, API-key, or basic auth: + +```python +r = requests.get(f"{TARGET_URL}/profile", headers={**__AUTH_HEADERS__}) +``` + +Configure the credential **once on the project** (ask the user for the value — never invent +or reuse a key you happened to see), and the block stays correct + refreshable: + +- **Bearer / API key / basic (static):** + `testsprite project credential --type "Bearer token"|"API key"|"basic token" --credential ` +- **Auto-refreshing login (recurring token):** `testsprite project auto-auth …` + +A hardcoded token expires within hours (and a hardcoded key can't be rotated centrally), so +the test breaks on later runs; the managed block is rewritten with a fresh value each run. +`test create` emits a `[warn]` when it detects an inlined credential literal — treat that as +a must-fix, not a nuisance. + **Backend tests that share state declare dependencies at create time.** For a one-off verification, prefer a single self-contained script (log in inside the same file). But when the coverage set splits naturally into producer → consumer diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 4f5fdd3..24d8ec6 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -661,29 +661,39 @@ describe('runInstall — multi-target', () => { // --------------------------------------------------------------------------- describe('runInstall — empty target', () => { - it('non-TTY with no target throws exit 5', async () => { - const { fs: agentFs } = makeMemFs(); + it('non-TTY with no target defaults to claude and installs the skill file', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: [], + force: false, + }, + { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, + ); + + const claudeAbs = path.resolve(CWD, TARGETS.claude.path); + expect(store.has(claudeAbs)).toBe(true); + expect(capture.stderr.join('\n')).toContain('defaulting to claude'); + }); + + it('non-TTY default writes the canonical claude content', async () => { + const { store, fs: agentFs } = makeMemFs(); const { deps } = makeCapture(); - let thrown: unknown; - try { - await runInstall( - { - profile: 'default', - output: 'text', - debug: false, - dryRun: false, - target: [], - force: false, - }, - { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, - ); - } catch (err) { - thrown = err; - } + await runInstall( + { profile: 'default', output: 'text', debug: false, dryRun: false, target: [], force: false }, + { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, + ); - expect(thrown).toBeInstanceOf(ApiError); - expect((thrown as ApiError).exitCode).toBe(5); + const { path: relPath, content } = renderForTarget('claude', 'testsprite-verify'); + const abs = path.resolve(CWD, relPath); + expect(store.get(abs)).toBe(content); }); it('TTY with injected prompt returning "claude" installs claude', async () => { diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 0a40c4b..7d7e7c8 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -371,18 +371,19 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr if (rawTargets.length === 0) { const isTTY = deps.isTTY ?? Boolean(process.stdin.isTTY); if (!isTTY) { - throw localValidationError( - 'target', - `required; pass --target=claude (comma-separated or repeated for several). Supported: ${Object.keys(TARGETS).join(', ')}`, + stderrFn( + '[info] --target not specified; defaulting to claude. Pass --target= to select a different agent.', ); + resolvedTargetStrings = ['claude']; + } else { + const promptFn = deps.prompt ?? ((q: string) => promptText(q)); + const answer = (await promptFn('Targets to install (comma-separated) [claude]: ')).trim(); + const defaulted = answer || 'claude'; + resolvedTargetStrings = defaulted + .split(',') + .map(s => s.trim()) + .filter(Boolean); } - const promptFn = deps.prompt ?? ((q: string) => promptText(q)); - const answer = (await promptFn('Targets to install (comma-separated) [claude]: ')).trim(); - const defaulted = answer || 'claude'; - resolvedTargetStrings = defaulted - .split(',') - .map(s => s.trim()) - .filter(Boolean); } else { resolvedTargetStrings = rawTargets; } diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index 75fdff0..617fd6f 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -199,8 +199,14 @@ describe('runInit — happy path (interactive)', () => { const stdout = captured.stdout.join('\n'); expect(stdout).toContain('TestSprite initialized.'); expect(stdout).toContain('profile:'); + // Next steps leads with creating a project; no command that fails without --project. expect(stdout).toContain('Next steps:'); - expect(stdout).toContain('testsprite test list'); + expect(stdout).toContain('testsprite project create --type frontend'); + expect(stdout).toContain('testsprite test run --all --project '); + expect(stdout).toContain('the testsprite-onboard skill is installed'); + // No "current project" wording, no bare test list. + expect(stdout).not.toContain('current project'); + expect(stdout).not.toContain('testsprite test list'); }); it('json mode: emits structured InitSummary object', async () => { @@ -306,6 +312,15 @@ describe('runInit — --no-agent', () => { const stdout = captured.stdout.join('\n'); expect(stdout).toContain('skipped (--no-agent)'); + // --no-agent points at manual test creation; must not claim the skill is installed. + expect(stdout).toContain('Next steps:'); + expect(stdout).toContain('testsprite project create --type frontend'); + expect(stdout).toContain('testsprite test create --project '); + expect(stdout).toContain('testsprite test run --all --project '); + expect(stdout).not.toContain('skill is installed'); + // No "current project" wording, no bare test list. + expect(stdout).not.toContain('current project'); + expect(stdout).not.toContain('testsprite test list'); }); it('text mode with agent: summary contains skills line with both default skills', async () => { diff --git a/src/commands/init.ts b/src/commands/init.ts index aff0644..c2d5678 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -407,12 +407,33 @@ function renderInitText(data: unknown): string { } lines.push(''); lines.push('Next steps:'); - lines.push(' testsprite test list # list tests in the current project'); - lines.push(' testsprite agent list # check installed agent targets'); + lines.push(' # 1. Create your first project (frontend example) — prints a projectId'); + lines.push( + ' testsprite project create --type frontend --name "My App" --url https://your-app.com', + ); + lines.push(''); if (s.agent) { lines.push( - ' testsprite agent install --target= # re-install or install additional targets', + ' # 2. Generate tests: ask your coding agent (the testsprite-onboard skill is installed),', + ); + lines.push(' # or create one yourself, then run them (use the projectId from step 1):'); + lines.push(' testsprite test run --all --project '); + lines.push(''); + lines.push(' # Manage installed agent skills'); + lines.push(' testsprite agent list'); + lines.push( + ' testsprite agent install --target= # re-install or install additional targets', + ); + } else { + lines.push(' # 2. Create a test, then run it (use the projectId from step 1):'); + lines.push(' testsprite test create --project ...'); + lines.push(' testsprite test run --all --project '); + lines.push( + ' # Tip: `testsprite agent install` sets up the onboarding skill for your coding agent', ); + lines.push(''); + lines.push(' # Manage installed agent skills'); + lines.push(' testsprite agent list'); } return lines.join('\n'); diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index f63928c..daea851 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -8,7 +8,9 @@ import { type CliProject, type CliUpdateProjectResponse, createProjectCommand, + runAutoAuth, runCreate, + runCredential, runGet, runList, runUpdate, @@ -72,10 +74,10 @@ describe('createProjectCommand', () => { errorSpy.mockRestore(); }); - it('exposes list, get, create and update subcommands', () => { + it('exposes list, get, create, update, credential and auto-auth subcommands', () => { const project = createProjectCommand(); const names = project.commands.map(c => c.name()).sort(); - expect(names).toEqual(['create', 'get', 'list', 'update']); + expect(names).toEqual(['auto-auth', 'create', 'credential', 'get', 'list', 'update']); }); it('list exposes the pagination flags from the design contract', () => { @@ -327,6 +329,21 @@ describe('runList', () => { }); }); +describe('DEV-244 — project update no longer accepts the dead --description flag', () => { + it('rejects --description on `project update` as an unknown option', async () => { + const project = createProjectCommand(); + const update = project.commands.find(c => c.name() === 'update')!; + project.exitOverride(); + update.exitOverride(); + + await expect( + project.parseAsync(['update', 'proj_x', '--description', 'should not exist'], { + from: 'user', + }), + ).rejects.toThrow(/unknown option.*--description/i); + }); +}); + describe('createProjectCommand --page-size option parser', () => { it('rejects non-numeric --page-size values via commander', async () => { const project = createProjectCommand(); @@ -864,7 +881,6 @@ describe('runUpdate', () => { debug: false, projectId: 'proj_text', name: 'New Name', - description: 'New desc', }, { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, ); @@ -929,3 +945,249 @@ describe('runUpdate', () => { expect(result.updatedFields).toBeUndefined(); }); }); + +describe('runCredential', () => { + interface Captured { + url: string; + method: string; + body: unknown; + headers: Headers; + } + function captureFetch(captured: Captured[], body: unknown) { + return makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : undefined, + headers: new Headers(init.headers as Record), + }); + return { status: 200, body }; + }); + } + + it('PUTs /projects/:id/credential with authType + credential + idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured, { + projectId: 'p1', + authType: 'Bearer token', + rewroteCount: 2, + }); + const res = await runCredential( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + authType: 'Bearer token', + credential: 'tok-123', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + expect(res.rewroteCount).toBe(2); + const put = captured.find(c => c.method === 'PUT')!; + expect(put.url).toContain('/projects/p1/credential'); + expect(put.body).toEqual({ authType: 'Bearer token', credential: 'tok-123' }); + expect(put.headers.get('idempotency-key')).toMatch(/^cli-proj-cred-[0-9a-f-]{36}$/); + }); + + it('public clears the credential (no credential in body, none required)', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured, { + projectId: 'p1', + authType: 'public', + rewroteCount: 0, + }); + await runCredential( + { profile: 'default', output: 'json', debug: false, projectId: 'p1', authType: 'public' }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + const put = captured.find(c => c.method === 'PUT')!; + expect(put.body).toEqual({ authType: 'public' }); + }); + + it('non-public without --credential → VALIDATION_ERROR (exit 5), no fetch', async () => { + const { credentialsPath } = makeCreds(); + let fetched = false; + const fetchImpl = makeFetch(() => { + fetched = true; + return { body: {} }; + }); + await expect( + runCredential( + { profile: 'default', output: 'json', debug: false, projectId: 'p1', authType: 'API key' }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetched).toBe(false); + }); + + it('rejects an unknown --type locally (no fetch)', async () => { + const { credentialsPath } = makeCreds(); + let fetched = false; + const fetchImpl = makeFetch(() => { + fetched = true; + return { body: {} }; + }); + await expect( + runCredential( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + authType: 'jwt', + credential: 'x', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetched).toBe(false); + }); +}); + +describe('runAutoAuth', () => { + interface Captured { + url: string; + method: string; + body: Record; + headers: Headers; + } + function captureFetch(captured: Captured[]) { + return makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : {}, + headers: new Headers(init.headers as Record), + }); + return { + status: 200, + body: { projectId: 'p1', enabled: true, method: 'aws_cognito_refresh', inject: 'bearer' }, + }; + }); + } + + it('PUTs /projects/:id/auto-auth with the config body + idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured); + await runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + method: 'aws_cognito_refresh', + inject: 'bearer', + region: 'us-east-1', + clientId: 'abc', + refreshToken: 'rt-xyz', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + const put = captured.find(c => c.method === 'PUT')!; + expect(put.url).toContain('/projects/p1/auto-auth'); + expect(put.body).toEqual({ + enabled: true, + method: 'aws_cognito_refresh', + inject: 'bearer', + region: 'us-east-1', + clientId: 'abc', + refreshToken: 'rt-xyz', + }); + expect(put.headers.get('idempotency-key')).toMatch(/^cli-proj-autoauth-[0-9a-f-]{36}$/); + }); + + it('--disable sends enabled:false', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured); + await runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + disable: true, + method: 'password', + inject: 'bearer', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + expect(captured.find(c => c.method === 'PUT')!.body.enabled).toBe(false); + }); + + it('reads a secret from --refresh-token-file', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-rt-')); + const rtFile = join(dir, 'rt.txt'); + writeFileSync(rtFile, ' rt-from-file\n'); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured); + await runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + method: 'refresh_token', + inject: 'bearer', + tokenEndpoint: 'https://idp.example.com/token', + refreshTokenFile: rtFile, + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + expect(captured.find(c => c.method === 'PUT')!.body.refreshToken).toBe('rt-from-file'); + }); + + it('rejects an unknown --method / --inject locally (no fetch)', async () => { + const { credentialsPath } = makeCreds(); + let fetched = false; + const fetchImpl = makeFetch(() => { + fetched = true; + return { body: {} }; + }); + await expect( + runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + method: 'magic', + inject: 'bearer', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetched).toBe(false); + }); +}); + +describe('dogfood 2026-06-30 — whitespace-only --name is rejected (parity with `test create`)', () => { + const noNetwork = () => { + throw new Error('network should not be hit'); + }; + + it('runCreate rejects a whitespace-only --name (exit 5, no network)', async () => { + const { credentialsPath } = makeCreds(); + await expect( + runCreate( + { profile: 'default', output: 'json', debug: false, type: 'backend', name: ' ' }, + { credentialsPath, fetchImpl: makeFetch(noNetwork), stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('runUpdate rejects a whitespace-only --name (exit 5, no network)', async () => { + const { credentialsPath } = makeCreds(); + await expect( + runUpdate( + { profile: 'default', output: 'json', debug: false, projectId: 'p1', name: '\t \n' }, + { credentialsPath, fetchImpl: makeFetch(noNetwork), stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); diff --git a/src/commands/project.ts b/src/commands/project.ts index ef4d5e8..96f2fc8 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -142,20 +142,16 @@ export async function runCreate( // (exit 10 UNAVAILABLE) — fail fast with a clear exit 5 instead. assertIdempotencyKey(opts.idempotencyKey); - // Reject empty / whitespace-only names so a junk record never reaches the - // backend — matches the `requireString` whitespace guard `test create` uses - // (dogfood P1 fix #1). Without this, `--name " "` passes the action - // handler's `if (!name)` check (a non-empty string is truthy) and is sent - // verbatim, creating a blank-named project. - if (opts.name !== undefined && opts.name.trim().length === 0) { - throw localValidationError('--name must not be empty or whitespace-only'); + // P1-3: client-side length checks matching server limits. + // Whitespace-only / empty rejection (parity with `test create`'s requireString; + // a truthy `--name " "` otherwise creates a blank-named project on the backend). + if (opts.name === undefined || opts.name.trim().length === 0) { + throw localValidationError('--name is required and must not be empty or whitespace-only'); } if (opts.password !== undefined && opts.password.trim().length === 0) { throw localValidationError('--password must not be empty or whitespace-only'); } - - // P1-3: client-side length checks matching server limits. - if (opts.name !== undefined && opts.name.length > 200) { + if (opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } if (opts.description !== undefined && opts.description.length > 2000) { @@ -248,7 +244,6 @@ interface UpdateOptions extends CommonOptions { username?: string; password?: string; passwordFile?: string; - description?: string; instruction?: string; idempotencyKey?: string; } @@ -264,6 +259,8 @@ export async function runUpdate( assertIdempotencyKey(opts.idempotencyKey); // P1-3: client-side length checks matching server limits. + // Reject a whitespace-only `--name` on update too (parity with create); name + // stays optional here, so only validate when the flag is supplied. if (opts.name !== undefined && opts.name.trim().length === 0) { throw localValidationError('--name must not be empty or whitespace-only'); } @@ -273,10 +270,6 @@ export async function runUpdate( if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } - if (opts.description !== undefined && opts.description.length > 2000) { - throw localValidationError('--description must be at most 2000 characters'); - } - // P2-7: guard --url against localhost/RFC1918/non-http(s). if (opts.targetUrl !== undefined) { assertNotLocal(opts.targetUrl); @@ -288,7 +281,6 @@ export async function runUpdate( targetUrl: opts.targetUrl !== undefined, username: opts.username !== undefined, password: passwordSupplied, - description: opts.description !== undefined, instruction: opts.instruction !== undefined, }; const presentFieldNames = Object.entries(mutableFields) @@ -296,7 +288,7 @@ export async function runUpdate( .map(([field]) => field); if (presentFieldNames.length === 0) { throw localValidationError( - 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, --description, or --instruction.', + 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, or --instruction.', ); } @@ -336,7 +328,6 @@ export async function runUpdate( targetUrl: opts.targetUrl, username: opts.username, password, - description: opts.description, instruction: opts.instruction, }; const body = Object.fromEntries( @@ -355,6 +346,228 @@ export async function runUpdate( return updated; } +// --------------------------------------------------------------------------- +// project credential — set the static backend credential +// --------------------------------------------------------------------------- + +const CLI_AUTH_TYPES = ['public', 'Bearer token', 'API key', 'basic token'] as const; + +export interface CliProjectCredentialResponse { + projectId: string; + authType: string; + rewroteCount: number; +} + +interface CredentialOptions extends CommonOptions { + projectId: string; + authType: string; + credential?: string; + credentialFile?: string; + idempotencyKey?: string; +} + +export async function runCredential( + opts: CredentialOptions, + deps: ProjectDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + assertIdempotencyKey(opts.idempotencyKey); + + if (!(CLI_AUTH_TYPES as readonly string[]).includes(opts.authType)) { + throw localValidationError(`--type must be one of: ${CLI_AUTH_TYPES.join(', ')}`); + } + + // Resolve the credential value (flag or file). Required for every type + // except `public` (which clears it). + let credential = opts.credential; + if (credential === undefined && opts.credentialFile !== undefined) { + credential = readFileSync(opts.credentialFile, 'utf8').trim(); + } + if (opts.authType !== 'public' && (credential === undefined || credential === '')) { + throw localValidationError( + '--credential (or --credential-file) is required unless --type is "public"', + ); + } + + const body: Record = { authType: opts.authType }; + if (opts.authType !== 'public' && credential !== undefined) body.credential = credential; + + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-cred-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + stderr(`idempotency-key: ${idempotencyKey}`); + } + + if (opts.dryRun) { + const sample: CliProjectCredentialResponse = { + projectId: opts.projectId, + authType: opts.authType, + rewroteCount: 0, + }; + out.print(sample, data => renderCredentialText(data as CliProjectCredentialResponse)); + return sample; + } + + const client = makeClient(opts, deps); + const res = await client.put( + `/projects/${encodeURIComponent(opts.projectId)}/credential`, + { body, headers: { 'idempotency-key': idempotencyKey } }, + ); + out.print(res, data => renderCredentialText(data as CliProjectCredentialResponse)); + return res; +} + +function renderCredentialText(r: CliProjectCredentialResponse): string { + return [ + `projectId ${r.projectId}`, + `authType ${r.authType}`, + `rewroteCount ${r.rewroteCount}`, + ].join('\n'); +} + +// --------------------------------------------------------------------------- +// project auto-auth — configure the recurring-token (auto-refresh) login +// --------------------------------------------------------------------------- + +const AUTO_AUTH_METHODS = ['password', 'refresh_token', 'aws_cognito_refresh'] as const; +const AUTO_AUTH_INJECTS = ['bearer', 'header', 'cookie'] as const; + +export interface CliProjectAutoAuthResponse { + projectId: string; + enabled: boolean; + method: string; + inject: string; + /** + * Present when the server's trial refresh failed: `enabled` is then `false` + * and this carries the reason (e.g. a bad refresh token). The config is still + * stored, but auto-auth won't run until the login succeeds. + */ + lastRefreshError?: string; +} + +interface AutoAuthOptions extends CommonOptions { + projectId: string; + disable?: boolean; + method: string; + inject: string; + injectKey?: string; + // password method + loginUrl?: string; + loginMethod?: string; + loginContentType?: string; + loginBodyTemplate?: string; + username?: string; + password?: string; + passwordFile?: string; + tokenPath?: string; + // refresh_token method + tokenEndpoint?: string; + clientId?: string; + clientSecret?: string; + clientSecretFile?: string; + refreshToken?: string; + refreshTokenFile?: string; + scope?: string; + // aws_cognito_refresh method + region?: string; + idempotencyKey?: string; +} + +export async function runAutoAuth( + opts: AutoAuthOptions, + deps: ProjectDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + assertIdempotencyKey(opts.idempotencyKey); + + if (!(AUTO_AUTH_METHODS as readonly string[]).includes(opts.method)) { + throw localValidationError(`--method must be one of: ${AUTO_AUTH_METHODS.join(', ')}`); + } + if (!(AUTO_AUTH_INJECTS as readonly string[]).includes(opts.inject)) { + throw localValidationError(`--inject must be one of: ${AUTO_AUTH_INJECTS.join(', ')}`); + } + + // Resolve secrets from --*-file variants so they stay out of shell history. + const password = + opts.password ?? + (opts.passwordFile !== undefined ? readFileSync(opts.passwordFile, 'utf8').trim() : undefined); + const clientSecret = + opts.clientSecret ?? + (opts.clientSecretFile !== undefined + ? readFileSync(opts.clientSecretFile, 'utf8').trim() + : undefined); + const refreshToken = + opts.refreshToken ?? + (opts.refreshTokenFile !== undefined + ? readFileSync(opts.refreshTokenFile, 'utf8').trim() + : undefined); + + const enabled = opts.disable !== true; + const body: Record = { enabled, method: opts.method, inject: opts.inject }; + const maybe = (k: string, v: string | undefined): void => { + if (v !== undefined) body[k] = v; + }; + maybe('injectKey', opts.injectKey); + maybe('loginUrl', opts.loginUrl); + maybe('loginMethod', opts.loginMethod); + maybe('loginContentType', opts.loginContentType); + maybe('loginBodyTemplate', opts.loginBodyTemplate); + maybe('username', opts.username); + maybe('password', password); + maybe('tokenPath', opts.tokenPath); + maybe('tokenEndpoint', opts.tokenEndpoint); + maybe('clientId', opts.clientId); + maybe('clientSecret', clientSecret); + maybe('refreshToken', refreshToken); + maybe('scope', opts.scope); + maybe('region', opts.region); + + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-autoauth-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + stderr(`idempotency-key: ${idempotencyKey}`); + } + + if (opts.dryRun) { + const sample: CliProjectAutoAuthResponse = { + projectId: opts.projectId, + enabled, + method: opts.method, + inject: opts.inject, + }; + out.print(sample, data => renderAutoAuthText(data as CliProjectAutoAuthResponse)); + return sample; + } + + const client = makeClient(opts, deps); + const res = await client.put( + `/projects/${encodeURIComponent(opts.projectId)}/auto-auth`, + { body, headers: { 'idempotency-key': idempotencyKey } }, + ); + out.print(res, data => renderAutoAuthText(data as CliProjectAutoAuthResponse)); + return res; +} + +function renderAutoAuthText(r: CliProjectAutoAuthResponse): string { + const lines = [ + `projectId ${r.projectId}`, + `enabled ${r.enabled}`, + `method ${r.method}`, + `inject ${r.inject}`, + ]; + if (r.lastRefreshError) { + lines.push(`lastRefreshError ${r.lastRefreshError}`); + } + // A disabled result after a write means the trial login failed — call it out + // so the user doesn't assume auto-auth is live. + if (!r.enabled) { + lines.push( + 'note auto-auth was stored but is DISABLED — the trial login failed. Fix the credentials (e.g. a valid refresh token) and re-run.', + ); + } + return lines.join('\n'); +} + export function createProjectCommand(deps: ProjectDeps = {}): Command { const project = new Command('project').description('Manage TestSprite projects'); @@ -448,7 +661,6 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { .option('--username ', 'new auth username') .option('--password ', 'new auth password') .option('--password-file ', 'read new password from file') - .option('--description ', 'new description') .option('--instruction ', 'new FE plan-gen instruction hint') .option( '--idempotency-key ', @@ -465,7 +677,6 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { username: cmdOpts.username, password: cmdOpts.password, passwordFile: cmdOpts.passwordFile, - description: cmdOpts.description, instruction: cmdOpts.instruction, idempotencyKey: cmdOpts.idempotencyKey, }, @@ -473,6 +684,99 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { ); }); + project + .command('credential ') + .description( + 'Set the static backend credential injected into every backend test\n' + + '(Bearer token / API key / Basic token / public). Free tier.', + ) + .requiredOption('--type ', 'public | "Bearer token" | "API key" | "basic token"') + .option('--credential ', 'credential value (required unless --type public)') + .option('--credential-file ', 'read the credential value from a file') + .option( + '--idempotency-key ', + 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (projectId: string, cmdOpts: CredentialFlagOpts, command: Command) => { + await runCredential( + { + ...resolveCommonOptions(command), + projectId, + authType: cmdOpts.type, + credential: cmdOpts.credential, + credentialFile: cmdOpts.credentialFile, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + + project + .command('auto-auth ') + .description( + 'Configure the recurring-token (auto-refresh login) for backend tests (Pro).\n' + + 'A fresh token is fetched on each run and injected into every backend test.', + ) + .requiredOption('--method ', 'password | refresh_token | aws_cognito_refresh') + .requiredOption('--inject ', 'bearer | header | cookie') + .option('--disable', 'turn auto-auth off (keeps stored config)') + .option('--inject-key ', 'header/cookie name when --inject is header/cookie') + // password method + .option('--login-url ', 'login endpoint (method=password)') + .option('--login-method ', 'POST | PUT (method=password)') + .option('--login-content-type ', 'application/json | application/x-www-form-urlencoded') + .option('--login-body-template ', 'login body template with {{username}}/{{password}}') + .option('--username ', 'login username (method=password)') + .option('--password ', 'login password (method=password)') + .option('--password-file ', 'read login password from a file') + .option('--token-path ', 'JSONPath to the token in the login response') + // refresh_token method + .option('--token-endpoint ', 'OAuth token endpoint (method=refresh_token)') + .option('--client-id ', 'OAuth client id') + .option('--client-secret ', 'OAuth client secret') + .option('--client-secret-file ', 'read OAuth client secret from a file') + .option('--refresh-token ', 'OAuth/Cognito refresh token') + .option('--refresh-token-file ', 'read the refresh token from a file') + .option('--scope ', 'OAuth scope') + // aws_cognito_refresh method + .option('--region ', "AWS region (method=aws_cognito_refresh, e.g. 'us-east-1')") + .option( + '--idempotency-key ', + 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (projectId: string, cmdOpts: AutoAuthFlagOpts, command: Command) => { + await runAutoAuth( + { + ...resolveCommonOptions(command), + projectId, + disable: cmdOpts.disable, + method: cmdOpts.method, + inject: cmdOpts.inject, + injectKey: cmdOpts.injectKey, + loginUrl: cmdOpts.loginUrl, + loginMethod: cmdOpts.loginMethod, + loginContentType: cmdOpts.loginContentType, + loginBodyTemplate: cmdOpts.loginBodyTemplate, + username: cmdOpts.username, + password: cmdOpts.password, + passwordFile: cmdOpts.passwordFile, + tokenPath: cmdOpts.tokenPath, + tokenEndpoint: cmdOpts.tokenEndpoint, + clientId: cmdOpts.clientId, + clientSecret: cmdOpts.clientSecret, + clientSecretFile: cmdOpts.clientSecretFile, + refreshToken: cmdOpts.refreshToken, + refreshTokenFile: cmdOpts.refreshTokenFile, + scope: cmdOpts.scope, + region: cmdOpts.region, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + return project; } @@ -500,11 +804,41 @@ interface UpdateFlagOpts { username?: string; password?: string; passwordFile?: string; - description?: string; instruction?: string; idempotencyKey?: string; } +interface CredentialFlagOpts { + type: string; + credential?: string; + credentialFile?: string; + idempotencyKey?: string; +} + +interface AutoAuthFlagOpts { + disable?: boolean; + method: string; + inject: string; + injectKey?: string; + loginUrl?: string; + loginMethod?: string; + loginContentType?: string; + loginBodyTemplate?: string; + username?: string; + password?: string; + passwordFile?: string; + tokenPath?: string; + tokenEndpoint?: string; + clientId?: string; + clientSecret?: string; + clientSecretFile?: string; + refreshToken?: string; + refreshTokenFile?: string; + scope?: string; + region?: string; + idempotencyKey?: string; +} + function parseFlag(raw: string | undefined, flagName: string): number | undefined { if (raw === undefined) return undefined; const n = Number(raw); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 6ae831c..f7c7602 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -4631,6 +4631,48 @@ describe('runCreate', () => { expect(sent.headers.get('x-api-key')).toBe('sk-user-test'); }); + it('emits backend warnings[] to stderr without polluting stdout JSON', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('BEARER = "eyJhbGciOi.eyJzdWIiOiJ4In0.sig"\n'); + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'GET') return { status: 200, body: { items: [] } }; + return { + status: 200, + body: { + ...SAMPLE_RESPONSE, + type: 'backend', + warnings: [ + 'This test appears to hardcode an auth credential — read auth from __AUTH_HEADERS__.', + ], + }, + }; + }); + const out: string[] = []; + const err: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'backend', + name: 'hardcoded be', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => err.push(line), + }, + ); + // Warning lands on stderr, prefixed `[warn]`. + expect(err.some(l => l.includes('[warn]') && l.includes('__AUTH_HEADERS__'))).toBe(true); + // stdout stays the parseable wire object — no warning noise. + expect(out.join('\n')).not.toContain('[warn]'); + }); + it('respects a caller-supplied --idempotency-key (for safe retries)', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('code body'); @@ -5674,6 +5716,36 @@ describe('runCreate — M4 BE dependency authoring flags', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); + it('[FE guard] --produces rejection is well-formed: bare field, no ----produces, singular verb (dogfood 2026-06-30)', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// fe code'); + const err = (await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_fe', + type: 'frontend', + name: 'fe test', + codeFile, + produces: ['some_var'], + }, + { + credentialsPath, + fetchImpl: () => Promise.resolve(new Response('{}')), + stdout: () => undefined, + stderr: () => undefined, + }, + ).catch((e: unknown) => e)) as ApiError; + expect(err).toBeInstanceOf(ApiError); + // details.field is the BARE flag name (was '--produces' → double-dashed subject). + expect(err.details).toMatchObject({ field: 'produces' }); + expect(err.nextAction).toContain('--produces'); + expect(err.nextAction).not.toContain('----'); // regression: '----produces' + expect(err.nextAction).toContain('is a backend-only flag'); // singular for one flag + expect(err.nextAction).not.toContain('backend..'); // regression: double period + }); + it('[FE guard] throws VALIDATION_ERROR exit 5 when --type frontend + --needs', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('// fe code'); diff --git a/src/commands/test.ts b/src/commands/test.ts index 1c6edd8..0602afc 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -543,6 +543,12 @@ export interface CliCreateTestResponse { type: 'frontend' | 'backend'; codeVersion: string; createdAt: string; + /** + * Non-fatal advisories from the backend (e.g. the BE auth guardrail + * flagging a hardcoded credential). Rendered on stderr; the create + * still succeeded. + */ + warnings?: string[]; } export const CLI_CREATE_PRIORITIES = ['p0', 'p1', 'p2', 'p3'] as const; @@ -752,14 +758,19 @@ export async function runCreate( // save a round-trip. if (opts.type === 'frontend') { const depFlags: string[] = []; - if (opts.produces !== undefined && opts.produces.length > 0) depFlags.push('--produces'); - if (opts.needs !== undefined && opts.needs.length > 0) depFlags.push('--needs'); - if (opts.category !== undefined) depFlags.push('--category'); + if (opts.produces !== undefined && opts.produces.length > 0) depFlags.push('produces'); + if (opts.needs !== undefined && opts.needs.length > 0) depFlags.push('needs'); + if (opts.category !== undefined) depFlags.push('category'); if (depFlags.length > 0) { + // Pass the BARE flag name to localValidationError — its kind:'flag' branch + // adds the `--` prefix, so '--produces' would render as '----produces'. + const flagList = depFlags.map(f => `--${f}`); + const verb = depFlags.length === 1 ? 'is a backend-only flag' : 'are backend-only flags'; + // No trailing period: localValidationError appends one after the reason. throw localValidationError( depFlags[0]!, - `${depFlags.join(', ')} are backend-only flags; frontend plans have no wave model. ` + - `Remove ${depFlags.join('/')} or use --type backend.`, + `${flagList.join(', ')} ${verb}; frontend plans have no wave model. ` + + `Remove ${flagList.join('/')} or use --type backend`, ); } } @@ -836,6 +847,11 @@ export async function runCreate( headers: { 'idempotency-key': idempotencyKey }, }); + // Surface backend advisories (e.g. a hardcoded-credential warning for BE + // tests) on stderr so they reach the agent without polluting stdout JSON. + // Emitted before the --run early return so they always show. + emitResponseWarnings(response.warnings, deps); + // --run chain (M3.3 piece-3). Per codex round-1 P1: suppress the // create's own print when chaining; `runTestRun` emits a single // merged envelope `{ ...createResponse, run: }` so @@ -998,6 +1014,16 @@ function renderCreateText(response: CliCreateTestResponse): string { ].join('\n'); } +/** + * Emit backend `warnings[]` advisories to stderr (one `[warn]` line each), + * keeping stdout — JSON or text — uncluttered. No-op when absent/empty. + */ +function emitResponseWarnings(warnings: string[] | undefined, deps: TestDeps): void { + if (!warnings || warnings.length === 0) return; + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + for (const w of warnings) stderrFn(`[warn] ${w}`); +} + /** * §6.X / M3.2 piece-6 — response from `PUT /tests/{id}/plan-steps`. * `planStepsHash` is a sha256 over the canonicalized new array so @@ -3400,6 +3426,12 @@ export interface CliPutTestCodeResponse { testId: string; codeVersion: string; updatedAt: string; + /** + * Non-fatal advisories (e.g. the BE auth guardrail flagging a hardcoded + * credential in the replaced code). Rendered on stderr; the update still + * succeeded. + */ + warnings?: string[]; } type CodePutLanguage = CliTestCode['language']; @@ -3577,6 +3609,7 @@ export async function runCodePut( }, }, ); + emitResponseWarnings(response.warnings, deps); out.print(response, data => renderCodePutText(data as CliPutTestCodeResponse)); return response; } catch (err) { @@ -5803,7 +5836,7 @@ export async function runTestWait( // --------------------------------------------------------------------------- interface RunTestRunAllOptions extends CommonOptions { - /** projectId to run all BE tests in. */ + /** projectId to run all tests in. */ projectId: string; /** --filter : only run tests whose name contains this substring (case-insensitive). */ nameFilter?: string; @@ -5940,7 +5973,7 @@ export async function runTestRunAll( stderrFn(`idempotency-key: ${idempotencyKey}`); } - // Resolve testIds: fetch all BE tests in the project, apply --filter. + // Resolve testIds: fetch all tests in the project, apply --filter. let testIds: string[] | undefined; if (opts.nameFilter !== undefined && opts.nameFilter !== '') { // We need to resolve the full test set to apply the name filter. @@ -5978,7 +6011,8 @@ export async function runTestRunAll( `Resolved ${testIds.length} test${testIds.length !== 1 ? 's' : ''} in project ${opts.projectId} for batch run.`, ); } - // When no --filter, omit testIds → server runs ALL BE tests in the project. + // When no --filter, omit testIds → server runs ALL tests in the project + // (BE tests on the legacy V2 wave engine; FE + BE on the V3 unified engine). const batchResp = await client.triggerBatchRunFresh( { @@ -8369,7 +8403,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { .command('run [test-id]') .description( 'Trigger a test run. With --wait, polls until terminal status.\n' + - 'Use --all --project for a wave-ordered batch run of all BE tests (M4).\n' + + 'Use --all --project for a wave-ordered batch run of all tests in a project (M4).\n' + '\nExit codes:\n' + ' 0 passed (or queued without --wait)\n' + ' 1 failed / blocked / cancelled\n' + @@ -8397,7 +8431,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option( '--all', - 'run all BE tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', + 'run all tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', false, ) .option( @@ -8424,11 +8458,14 @@ export function createTestCommand(deps: TestDeps = {}): Command { .addHelpText( 'after', '\nDependency-aware fresh run (M4):\n' + - ' testsprite test run --all --project run all BE tests in wave order\n' + + ' testsprite test run --all --project run all project tests in wave order\n' + ' testsprite test run --all --project --filter name-glob subset\n' + ' testsprite test run --all --project --wait --report junit --report-file ./results.xml\n' + '\nBE tests can declare --produces/--needs at create time to drive wave ordering\n' + - '(see `testsprite test create --help` for details).', + '(see `testsprite test create --help` for details).\n' + + '\nFrontend tests: the current unified engine runs FE tests too (they are billed\n' + + 'like any run). On the legacy backend-only engine FE tests cannot run — they are\n' + + "reported under skippedFrontend with an advisory; run those with 'test run '.", ) .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (testIdArg: string | undefined, cmdOpts: RunFlagOpts, command: Command) => { @@ -8444,7 +8481,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { if (testIdArg === undefined && !isAll) { throw localValidationError( 'test-id', - 'provide a , or use --all --project to run all BE tests in a project', + 'provide a , or use --all --project to run all tests in a project', ); } // --filter is an --all-only narrowing flag (mirrors `test rerun --filter`). @@ -8473,14 +8510,16 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--all requires a project id — pass --project ', ); } - // --target-url has no effect on the --all batch path: it is BE-only - // (FE tests are skipped server-side) and a BE test's base URL is baked - // into its code. Silently dropping it could run the suite against an - // unintended environment in the caller's mind — reject loudly instead. + // --target-url has no effect on the --all batch path: a BE test's base + // URL is baked into its code, and the unified engine resolves each + // project's configured environment server-side (per-run URL overrides + // are not applied to batch FE runs either). Silently dropping it could + // run the suite against an unintended environment in the caller's mind + // — reject loudly instead. if (cmdOpts.targetUrl !== undefined && cmdOpts.targetUrl !== '') { throw localValidationError( 'target-url', - '--target-url has no effect with --all (the batch path is the BE-only wave engine; a BE test’s URL is baked into its code). Remove --target-url.', + '--target-url has no effect with --all (the batch path does not apply a per-run URL override — BE test URLs are baked into their code and the unified engine resolves the project environment server-side). Remove --target-url.', ); } await runTestRunAll( @@ -9973,7 +10012,11 @@ export function createTestArtifactCommand(deps: TestDeps): Command { 'Parent must exist. The bundle dir itself is created if absent.', ].join(' '), ) - .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') + .option( + '--failed-only', + 'Trim to the failed step ±1. The bundle is already failure-focused server-side, ' + + 'so this is usually a no-op; use `test steps ` for the full run trail.', + ) .addHelpText('after', GLOBAL_OPTS_HINT) .action( async (runId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { @@ -10003,7 +10046,11 @@ function createTestFailureCommand(deps: TestDeps): Command { '--out ', 'Directory to write the §7 disk layout into (default: print wire envelope to stdout)', ) - .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') + .option( + '--failed-only', + 'Trim to the failed step ±1. The bundle is already failure-focused server-side, ' + + 'so this is usually a no-op; use `test steps ` for the full run trail.', + ) .addHelpText('after', GLOBAL_OPTS_HINT) .action( async (testId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { diff --git a/src/commands/usage.ts b/src/commands/usage.ts index a97dbba..1980d26 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -154,8 +154,12 @@ function renderUsage(u: UsageResponse, portalBase?: string): string { } if (u.creditsPerRun !== undefined) { lines.push(`cost per frontend run: ${u.creditsPerRun} credit(s)`); + // Backend runs DO consume credits (confirmed by design 2026-06-30 / DEV-289). + // The API exposes no backend-specific per-run cost field, and it differs from + // the frontend rate, so state that it bills without asserting a possibly-wrong + // number — check your balance before/after, or see the billing page. lines.push( - `cost per backend run: 0 credit(s) (backend tests bill at code-generation, not at run time)`, + `cost per backend run: also consumes credits (exact amount not reported by the API)`, ); } diff --git a/src/index.ts b/src/index.ts index 806f6e4..bf9a9aa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -52,7 +52,7 @@ program .option('--debug', 'Print HTTP method/path, request id, latency, retry decisions to stderr') .option( '--dry-run', - 'Skip the network, credentials, and filesystem; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key.', + 'Skip the network and credentials; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key. Note: file inputs you pass (--plan-from/--plans/--steps) are still read and validated locally; only --code-file uses a placeholder.', ) .option( '--request-timeout ', diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index 51bf9b5..2ac9158 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -34,7 +34,9 @@ import { * The description value is a single line (no folded/literal block scalars). */ function parseFrontmatterDescription(content: string): string | undefined { - const lines = content.split('\n'); + // Tolerate CRLF so a Windows checkout (autocrlf) doesn't leave a trailing + // \r on the description and break the byte-identical comparisons. + const lines = content.split(/\r?\n/); let inFrontmatter = false; for (const line of lines) { if (line.trim() === '---') { diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index aec451a..ff20e8c 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -10,7 +10,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { isAbsolute, join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; import { applyFailedOnly, @@ -593,8 +593,8 @@ describe('resolveBundleDir', () => { it('resolves a relative path against cwd', () => { const out = resolveBundleDir('./tmp/x'); - expect(out.endsWith('/tmp/x')).toBe(true); - expect(out.startsWith('/')).toBe(true); + expect(out).toBe(resolve(process.cwd(), 'tmp', 'x')); + expect(isAbsolute(out)).toBe(true); }); it('strips a trailing slash', () => { diff --git a/src/lib/credentials.test.ts b/src/lib/credentials.test.ts index 896d057..d50ad52 100644 --- a/src/lib/credentials.test.ts +++ b/src/lib/credentials.test.ts @@ -1,5 +1,5 @@ import { mkdtempSync, statSync, readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { @@ -139,8 +139,11 @@ describe('writeProfile', () => { it('creates the file with mode 0600 and writes the profile', () => { writeProfile(DEFAULT_PROFILE, { apiKey: 'sk-new' }, { path: credentialsPath }); expect(existsSync(credentialsPath)).toBe(true); - const mode = statSync(credentialsPath).mode & 0o777; - expect(mode).toBe(0o600); + // POSIX file modes don't exist on Windows (stat reports 0666). + if (process.platform !== 'win32') { + const mode = statSync(credentialsPath).mode & 0o777; + expect(mode).toBe(0o600); + } expect(readProfile(DEFAULT_PROFILE, { path: credentialsPath })).toEqual({ apiKey: 'sk-new' }); }); @@ -188,7 +191,8 @@ describe('ensureRestrictiveMode', () => { expect(() => ensureRestrictiveMode(credentialsPath)).not.toThrow(); }); - it('downgrades over-permissive modes', () => { + // POSIX-only premise: Windows has no 0644/0600 distinction to downgrade. + it.skipIf(process.platform === 'win32')('downgrades over-permissive modes', () => { mkdirSync(tmpRoot, { recursive: true }); writeFileSync(credentialsPath, 'data', { mode: 0o644 }); ensureRestrictiveMode(credentialsPath); @@ -199,7 +203,7 @@ describe('ensureRestrictiveMode', () => { describe('defaultCredentialsPath', () => { it('points at ~/.testsprite/credentials', () => { - expect(defaultCredentialsPath().endsWith('/.testsprite/credentials')).toBe(true); + expect(defaultCredentialsPath()).toBe(join(homedir(), '.testsprite', 'credentials')); }); }); diff --git a/src/lib/http.ts b/src/lib/http.ts index af1c25a..7a9051a 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -528,9 +528,9 @@ export class HttpClient { } // Edge proxies / load balancers return 408/502/504 without our error - // envelope on transient outages. Per the CLI error spec §7 these are - // transport-level retries, not facade errors — fold them in here so - // we get the bounded backoff budget instead of a single INTERNAL bail. + // envelope on transient outages. These are transport-level retries, + // not facade errors — fold them in here so we get the bounded backoff + // budget instead of a single INTERNAL bail. if (rawBody === null && isTransportEdgeStatus(response.status)) { this.debug({ kind: 'error', diff --git a/src/lib/skill-nudge.test.ts b/src/lib/skill-nudge.test.ts index c15b09d..2b26c0e 100644 --- a/src/lib/skill-nudge.test.ts +++ b/src/lib/skill-nudge.test.ts @@ -13,24 +13,30 @@ import { // isVerifySkillInstalled // --------------------------------------------------------------------------- +// The implementation joins paths with the native separator; normalize so the +// fakes below match on Windows (backslashes) as well as POSIX. +const toPosix = (p: string) => p.replaceAll('\\', '/'); + describe('isVerifySkillInstalled', () => { it('true when the claude own-file SKILL.md exists', () => { - const existsSync = (p: string) => p.endsWith('.claude/skills/testsprite-verify/SKILL.md'); + const existsSync = (p: string) => + toPosix(p).endsWith('.claude/skills/testsprite-verify/SKILL.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the cursor .mdc landing file', () => { - const existsSync = (p: string) => p.endsWith('.cursor/rules/testsprite-verify.mdc'); + const existsSync = (p: string) => toPosix(p).endsWith('.cursor/rules/testsprite-verify.mdc'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the cline landing file', () => { - const existsSync = (p: string) => p.endsWith('.clinerules/testsprite-verify.md'); + const existsSync = (p: string) => toPosix(p).endsWith('.clinerules/testsprite-verify.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the antigravity landing file', () => { - const existsSync = (p: string) => p.endsWith('.agents/skills/testsprite-verify/SKILL.md'); + const existsSync = (p: string) => + toPosix(p).endsWith('.agents/skills/testsprite-verify/SKILL.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); @@ -73,7 +79,7 @@ describe('isVerifySkillInstalled', () => { return false; }, }); - expect(seen.every(p => p.startsWith('/some/proj'))).toBe(true); + expect(seen.every(p => toPosix(p).startsWith('/some/proj'))).toBe(true); // One probe per target landing path. expect(seen).toHaveLength(Object.keys(TARGETS).length); }); @@ -198,6 +204,6 @@ describe('maybeEmitSkillNudge', () => { }); maybeEmitSkillNudge(ctx); expect(probed.length).toBeGreaterThan(0); - expect(probed.every(p => p.startsWith('/work/here'))).toBe(true); + expect(probed.every(p => toPosix(p).startsWith('/work/here'))).toBe(true); }); }); diff --git a/src/version.ts b/src/version.ts index efa916d..f60fdfd 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,3 +1,3 @@ // AUTO-GENERATED by scripts/generate-version.mjs — do not edit by hand. // Run `npm run build` (or `npm run generate:version`) to regenerate. -export const VERSION = '0.2.0'; +export const VERSION = '0.3.0'; diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index fcc3d85..62efe5d 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -133,20 +133,29 @@ exports[`--help snapshots > project 1`] = ` Manage TestSprite projects Options: - -h, --help display help for command + -h, --help display help for command Commands: - list [options] List projects visible to the API key + list [options] List projects visible to the API key Exit codes: 0 success 3 auth error 5 validation error (e.g., bad --page-size) 10 transport/network failure (UNAVAILABLE) — retry the command - get Get a project by id - create [options] Create a new project - update [options] Update project metadata - help [command] display help for command + get Get a project by id + create [options] Create a new project + update [options] Update project metadata + credential [options] Set the static backend credential injected + into every backend test + (Bearer token / API key / Basic token / + public). Free tier. + auto-auth [options] Configure the recurring-token + (auto-refresh login) for backend tests + (Pro). + A fresh token is fetched on each run and + injected into every backend test. + help [command] display help for command " `; @@ -242,7 +251,7 @@ Commands: Note: a 404 "not found" response is counted as skipped in the summary, not an error. run [options] [test-id] Trigger a test run. With --wait, polls until terminal status. - Use --all --project for a wave-ordered batch run of all BE tests (M4). + Use --all --project for a wave-ordered batch run of all tests in a project (M4). Exit codes: 0 passed (or queued without --wait) @@ -362,7 +371,9 @@ Write a self-contained failure-context bundle for a test's latest failing run Options: --out Directory to write the §7 disk layout into (default: print wire envelope to stdout) - --failed-only Keep only the failed step plus its immediate neighbors (±1) + --failed-only Trim to the failed step ±1. The bundle is already + failure-focused server-side, so this is usually a no-op; use + \`test steps \` for the full run trail. -h, --help display help for command Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): @@ -548,7 +559,7 @@ exports[`--help snapshots > test run 1`] = ` "Usage: testsprite test run [options] [test-id] Trigger a test run. With --wait, polls until terminal status. -Use --all --project for a wave-ordered batch run of all BE tests (M4). +Use --all --project for a wave-ordered batch run of all tests in a project (M4). Exit codes: 0 passed (or queued without --wait) @@ -572,9 +583,9 @@ Options: 600) --idempotency-key opaque key for safe retries (1–256 chars). Printed to stderr at --debug if auto-generated. - --all run all BE tests in the project (wave-ordered - fresh run; requires --project). Mutually - exclusive with . (default: false) + --all run all tests in the project (wave-ordered fresh + run; requires --project). Mutually exclusive with + . (default: false) --project project id (required with --all; returned by \`testsprite project list\`) --filter with --all: only run tests whose name contains @@ -589,13 +600,17 @@ Options: -h, --help display help for command Dependency-aware fresh run (M4): - testsprite test run --all --project run all BE tests in wave order + testsprite test run --all --project run all project tests in wave order testsprite test run --all --project --filter name-glob subset testsprite test run --all --project --wait --report junit --report-file ./results.xml BE tests can declare --produces/--needs at create time to drive wave ordering (see \`testsprite test create --help\` for details). +Frontend tests: the current unified engine runs FE tests too (they are billed +like any run). On the legacy backend-only engine FE tests cannot run — they are +reported under skippedFrontend with an advisory; run those with 'test run '. + Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " @@ -640,10 +655,13 @@ Options: without the full trace. --debug Print HTTP method/path, request id, latency, retry decisions to stderr - --dry-run Skip the network, credentials, and filesystem; - emit a canned sample matching the OpenAPI - contract. Useful for learning the CLI surface - without an API key. + --dry-run Skip the network and credentials; emit a canned + sample matching the OpenAPI contract. Useful for + learning the CLI surface without an API key. + Note: file inputs you pass + (--plan-from/--plans/--steps) are still read and + validated locally; only --code-file uses a + placeholder. --request-timeout Client-side per-request timeout in seconds (default: 120). Aborts any single fetch that does not complete within this deadline. Override diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index b0537b1..959f212 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -8,7 +8,7 @@ */ import { execFileSync, spawn } from 'node:child_process'; -import { existsSync, mkdtempSync, statSync } from 'node:fs'; +import { existsSync, mkdtempSync, rmSync, statSync } from 'node:fs'; import type { IncomingMessage, Server, ServerResponse } from 'node:http'; import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; @@ -364,7 +364,10 @@ function runCli(args: string[], envOverrides: Record = {}): Prom cwd: REPO_ROOT, env: { ...process.env, + // os.homedir() reads HOME on POSIX but USERPROFILE on Windows — + // set both so the child never sees the real ~/.testsprite. HOME: tmpHome, + USERPROFILE: tmpHome, TESTSPRITE_API_KEY: undefined, TESTSPRITE_API_URL: undefined, ...envOverrides, @@ -897,7 +900,10 @@ describe('setup --from-env subprocess', () => { expect(result.exitCode).toBe(0); const credentialsPath = join(tmpHome, '.testsprite', 'credentials'); expect(existsSync(credentialsPath)).toBe(true); - expect(statSync(credentialsPath).mode & 0o777).toBe(0o600); + // POSIX file modes don't exist on Windows (stat reports 0666). + if (process.platform !== 'win32') { + expect(statSync(credentialsPath).mode & 0o777).toBe(0o600); + } }, 30_000); it('exits 5 with VALIDATION_ERROR when --from-env is set without TESTSPRITE_API_KEY', async () => { @@ -1044,7 +1050,7 @@ describe('--dry-run subprocess smoke', () => { // skipped the prompt. const credPath = join(tmpHome, '.testsprite', 'credentials'); // Make sure any previous test didn't leave one behind. - if (existsSync(credPath)) execFileSync('rm', [credPath]); + rmSync(credPath, { force: true }); const result = await runCli(['setup', '--dry-run', '--no-agent', '--output', 'json']); expect(result.exitCode).toBe(0); expect(existsSync(credPath)).toBe(false); diff --git a/test/helpers/hermetic-env.ts b/test/helpers/hermetic-env.ts new file mode 100644 index 0000000..c798bb3 --- /dev/null +++ b/test/helpers/hermetic-env.ts @@ -0,0 +1,41 @@ +/** + * Unit-test env hermeticity (vitest `setupFiles`, runs before each test file). + * + * Two leaks this closes, both of which made results depend on the + * developer's machine: + * + * 1. Real `TESTSPRITE_*` env vars. `loadConfig` gives `TESTSPRITE_API_KEY` + * precedence over the credentials file, so a key exported in the + * developer's shell silently overrode test fixtures. + * 2. The real home directory. `os.homedir()` reads `HOME` on POSIX but + * `USERPROFILE` on Windows, so the documented `HOME=$(mktemp -d)` + * recipe never isolated Windows runs. Both vars are redirected to a + * throwaway dir so no test can read or write `~/.testsprite`. + * + * Tests that need these vars set them explicitly (on `process.env` or via + * injected `env` deps) after this runs. + */ +import { existsSync, mkdirSync, mkdtempSync } from 'node:fs'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const realHome = homedir(); +const hermeticHome = mkdtempSync(join(tmpdir(), 'testsprite-unit-home-')); +if (process.platform === 'win32') { + // Node version shims (Volta) resolve LocalAppData under USERPROFILE and + // abort if it's missing, which would break the `npm run build` beforeAll + // in the subprocess/snapshot suites. + mkdirSync(join(hermeticHome, 'AppData', 'Local'), { recursive: true }); +} +// Same shim concern on macOS/Linux: Volta derives ~/.volta from HOME unless +// VOLTA_HOME is set. Pin it to the real install before redirecting HOME. +const realVoltaHome = join(realHome, '.volta'); +if (!process.env.VOLTA_HOME && existsSync(realVoltaHome)) { + process.env.VOLTA_HOME = realVoltaHome; +} +process.env.HOME = hermeticHome; +process.env.USERPROFILE = hermeticHome; + +for (const key of Object.keys(process.env)) { + if (key.startsWith('TESTSPRITE_')) delete process.env[key]; +} diff --git a/vitest.config.ts b/vitest.config.ts index add8f0e..bd9b2ad 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,6 +4,9 @@ export default defineConfig({ test: { include: ['src/**/*.{test,spec}.ts', 'test/**/*.{test,spec}.ts'], exclude: ['test/dev-e2e/**', 'test/e2e/**', 'node_modules/**', 'dist/**'], + // Strip real TESTSPRITE_* env vars and redirect the home dir so results + // never depend on the developer's shell or ~/.testsprite (see the file). + setupFiles: ['./test/helpers/hermetic-env.ts'], // Subprocess/snapshot suites each run `npm run build` in beforeAll; parallel // file workers can race on dist/ and produce a stale binary (exit 1 vs 5 flakes). fileParallelism: false,