From db78dd4c04908db75d37e815897fed911f047913 Mon Sep 17 00:00:00 2001 From: galuis116 Date: Mon, 13 Jul 2026 05:47:48 -0400 Subject: [PATCH] feat(miner): add a committed benchmark suite for discovery ranking and the local-store path No profiling or baseline data existed for the miner package's two purely local, synchronous hot paths, so a future change to the ranking pass or the SQLite store layer had no repeatable number to compare against. Adds packages/gittensory-miner/scripts/benchmark.mjs (npm run benchmark:miner), which times rankCandidateIssues() over 500 deterministic synthetic candidates and 500 enqueue/dequeue cycles against an in-memory portfolio-queue store, reporting the median of 5 runs. Documented in BENCHMARKS.md with a baseline sample run. --- package.json | 1 + packages/gittensory-miner/BENCHMARKS.md | 44 ++++++ packages/gittensory-miner/README.md | 3 + packages/gittensory-miner/package.json | 1 + .../gittensory-miner/scripts/benchmark.d.mts | 38 +++++ .../gittensory-miner/scripts/benchmark.mjs | 134 ++++++++++++++++++ test/unit/miner-benchmark-script.test.ts | 84 +++++++++++ 7 files changed, 305 insertions(+) create mode 100644 packages/gittensory-miner/BENCHMARKS.md create mode 100644 packages/gittensory-miner/scripts/benchmark.d.mts create mode 100644 packages/gittensory-miner/scripts/benchmark.mjs create mode 100644 test/unit/miner-benchmark-script.test.ts diff --git a/package.json b/package.json index 9dd4c65a0..0b6fe62cc 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "selfhost:env-reference:check": "node scripts/gen-selfhost-env-reference.mjs --check", "miner:env-reference": "node packages/gittensory-miner/scripts/generate-env-reference.mjs", "miner:env-reference:check": "node packages/gittensory-miner/scripts/generate-env-reference.mjs --check", + "benchmark:miner": "node packages/gittensory-miner/scripts/benchmark.mjs", "command-reference": "node scripts/gen-command-reference.mjs", "command-reference:check": "node scripts/gen-command-reference.mjs --check", "selfhost:validate-observability": "node scripts/validate-observability-configs.mjs", diff --git a/packages/gittensory-miner/BENCHMARKS.md b/packages/gittensory-miner/BENCHMARKS.md new file mode 100644 index 000000000..ffffee46b --- /dev/null +++ b/packages/gittensory-miner/BENCHMARKS.md @@ -0,0 +1,44 @@ +# gittensory-miner benchmarks + +A small, committed micro-benchmark for the two purely local, synchronous hot paths that have no other signal +for a future regression: the discovery fan-out ranking pass and the local-store (SQLite) read/write path. +Neither makes a network call or touches a real GitHub repo — both run against deterministic synthetic input, +so the numbers are comparable across runs on the same machine. + +## Running it + +```sh +npm run benchmark:miner +# or, from a workspace checkout: +npm --workspace @loopover/miner run benchmark +``` + +This prints a short text report to stdout and exits `0`. It does not fail the build or a CI job on its own — +it is a signal to read, not a hard gate (there is no fixed pass/fail threshold, since wall-clock timing on +shared CI runners is too noisy to gate on reliably). Run it locally before/after a change to `opportunity-ranker.js`, +`opportunity-fanout.js`, or the SQLite store layer (`local-store.js`, `portfolio-queue.js`, and friends) to see +whether the change moved the needle. + +## What it measures + +- **`discovery-fanout-ranking`** — `rankCandidateIssues()` (`lib/opportunity-ranker.js`) over 500 synthetic + candidates, the pure ranking pass discovery runs once per repo per cycle over every open candidate. +- **`local-store-read-write`** — 500 `enqueue()` calls followed by 500 `dequeueNext()` calls against a fresh + in-memory (`:memory:`) `portfolio-queue.js` store, the same prepared-statement read/write path every real + enqueue/claim exercises on disk, minus filesystem I/O — isolating the query-plan/schema cost this package + actually controls. + +Each benchmark repeats its work 5 times and reports the **median** wall time, to smooth over GC pauses and +other one-off scheduling noise rather than reporting a single potentially-unlucky sample. + +## Baseline (informational only, machine-dependent) + +Captured on a Linux x86_64 dev container, Node.js 22.21.0. Absolute numbers vary by hardware — use this as a +rough sense of scale, not a target: + +``` +gittensory-miner benchmark + +discovery-fanout-ranking: median ~42ms over 5 runs, ~11,800 ops/sec (n=500) +local-store-read-write: median ~38ms over 5 runs, ~26,400 ops/sec (n=1000) +``` diff --git a/packages/gittensory-miner/README.md b/packages/gittensory-miner/README.md index 48851763f..8e61dc484 100644 --- a/packages/gittensory-miner/README.md +++ b/packages/gittensory-miner/README.md @@ -19,6 +19,9 @@ Regenerate that file with `npm run miner:env-reference` from the repo root after Config precedence (`.gittensory-miner.yml` vs operator env vs CLI flags) is documented in [`docs/config-precedence.md`](docs/config-precedence.md). +A committed micro-benchmark for the discovery-ranking and local-store read/write paths lives at +[`BENCHMARKS.md`](BENCHMARKS.md) — run it with `npm run benchmark:miner` from the repo root. + Real miner commands land in follow-up issues. The package also includes the first metadata-only discovery primitive: `fetchCandidateIssues` lists open issue diff --git a/packages/gittensory-miner/package.json b/packages/gittensory-miner/package.json index 820afc036..f49d79878 100644 --- a/packages/gittensory-miner/package.json +++ b/packages/gittensory-miner/package.json @@ -37,6 +37,7 @@ "expected-engine.version" ], "scripts": { + "benchmark": "node scripts/benchmark.mjs", "build": "node --check bin/gittensory-miner.js && node --check bin/gittensory-miner-mcp.js && node --check lib/ams-policy.js && node --check lib/attempt-cli.js && node --check lib/attempt-input-builder.js && node --check lib/attempt-log.js && node --check lib/attempt-runner.js && node --check lib/attempt-worktree.js && node --check lib/calibration-run.js && node --check lib/calibration-types.js && node --check lib/calibration.js && node --check lib/ci-poller.js && node --check lib/claim-adjudication.js && node --check lib/claim-conflict-resolver.js && node --check lib/claim-ledger-cli.js && node --check lib/claim-ledger-expiry.js && node --check lib/claim-ledger.js && node --check lib/cli.js && node --check lib/coding-agent-construction.js && node --check lib/coding-agent-house-rules.js && node --check lib/coding-task-spec.js && node --check lib/deny-check.js && node --check lib/deny-hook-synthesis.js && node --check lib/deny-hooks.js && node --check lib/deployment-docs-audit.js && node --check lib/discover-cli.js && node --check lib/event-ledger-cli.js && node --check lib/event-ledger.js && node --check lib/execute-local-write.js && node --check lib/feasibility-cli.js && node --check lib/governor-action-mode.js && node --check lib/governor-chokepoint-persisted.js && node --check lib/governor-chokepoint.js && node --check lib/governor-kill-switch.js && node --check lib/governor-ledger-cli.js && node --check lib/governor-ledger.js && node --check lib/governor-metrics-cli.js && node --check lib/governor-open-pr.js && node --check lib/governor-pause-cli.js && node --check lib/governor-run-halt.js && node --check lib/governor-state.js && node --check lib/governor-write-rate-limit.js && node --check lib/harness-submission-trigger.js && node --check lib/laptop-init.js && node --check lib/live-issue-snapshot.js && node --check lib/local-store.js && node --check lib/logger.js && node --check lib/loop-cli.js && node --check lib/loop-closure.js && node --check lib/loop-reentry.js && node --check lib/manage-poll.js && node --check lib/manage-status.js && node --check lib/metrics-cli.js && node --check lib/miner-goal-spec.js && node --check lib/opportunity-fanout.js && node --check lib/opportunity-ranker.js && node --check lib/orb-export.js && node --check lib/plan-store-cli.js && node --check lib/plan-store.js && node --check lib/policy-doc-cache.js && node --check lib/policy-verdict-cache.js && node --check lib/portfolio-dashboard.js && node --check lib/portfolio-discovery.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-queue-manager.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-expiry.js && node --check lib/pr-disposition-poller.js && node --check lib/pr-number-parse.js && node --check lib/pr-outcome.js && node --check lib/prediction-ledger.js && node --check lib/pretooluse-hook.js && node --check lib/purge-cli.js && node --check lib/rejection-signal.js && node --check lib/rejection-state-machine.js && node --check lib/rejection-templates.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-snapshot.js && node --check lib/replay-task-generation.js && node --check lib/repo-clone.js && node --check lib/run-state-cli.js && node --check lib/run-state.js && node --check lib/self-review-context.js && node --check lib/slop-assessment.js && node --check lib/stack-detection.js && node --check lib/status.js && node --check lib/submission-freshness-check.js && node --check lib/update-check.js && node --check lib/version.js && node --check lib/worktree-allocator.js" }, "dependencies": { diff --git a/packages/gittensory-miner/scripts/benchmark.d.mts b/packages/gittensory-miner/scripts/benchmark.d.mts new file mode 100644 index 000000000..a932d0a81 --- /dev/null +++ b/packages/gittensory-miner/scripts/benchmark.d.mts @@ -0,0 +1,38 @@ +export type BenchmarkCandidate = { + repoFullName: string; + issueNumber: number; + title: string; + labels: string[]; + commentsCount: number; + createdAt: string; + updatedAt: string; + htmlUrl: string; + aiPolicyAllowed: boolean; + aiPolicySource: "AI-USAGE.md" | "none"; +}; + +export type BenchmarkOptions = { + candidateCount?: number; + operationCount?: number; + iterations?: number; +}; + +export type BenchmarkResult = { + name: string; + unitCount: number; + iterations: number; + medianMs: number; + opsPerSecond: number; +}; + +export declare const DEFAULT_CANDIDATE_COUNT: number; +export declare const DEFAULT_QUEUE_OPERATION_COUNT: number; +export declare const DEFAULT_ITERATIONS: number; + +export declare function buildSyntheticCandidates(count: number): BenchmarkCandidate[]; + +export declare function runRankingBenchmark(options?: BenchmarkOptions): BenchmarkResult; + +export declare function runLocalStoreBenchmark(options?: BenchmarkOptions): BenchmarkResult; + +export declare function formatBenchmarkReport(results: readonly BenchmarkResult[]): string; diff --git a/packages/gittensory-miner/scripts/benchmark.mjs b/packages/gittensory-miner/scripts/benchmark.mjs new file mode 100644 index 000000000..ffaddecd3 --- /dev/null +++ b/packages/gittensory-miner/scripts/benchmark.mjs @@ -0,0 +1,134 @@ +#!/usr/bin/env node +import { performance } from "node:perf_hooks"; +import { rankCandidateIssues } from "../lib/opportunity-ranker.js"; +import { initPortfolioQueueStore } from "../lib/portfolio-queue.js"; + +// Committed micro-benchmark for the two hot local paths that have no other way to notice a regression: the +// discovery fan-out ranking pass (opportunity-ranker.js, run once per repo per discovery cycle over every open +// candidate) and the local-store read/write path (portfolio-queue.js, run on every enqueue/claim). Neither is +// covered by the request-latency instrumentation the coding-agent driver already has, since both are purely +// synchronous/local. See BENCHMARKS.md (#4845) for how to run this and read the numbers. + +export const DEFAULT_CANDIDATE_COUNT = 500; +export const DEFAULT_QUEUE_OPERATION_COUNT = 500; +export const DEFAULT_ITERATIONS = 5; + +const LABEL_POOL = ["good first issue", "help wanted", "bug", "gittensor:feature", "visual"]; +const SYNTHETIC_EPOCH_MS = Date.UTC(2024, 0, 1); + +/** + * Deterministic synthetic candidate generator: every field is derived from `i` alone, never `Math.random()` or + * `Date.now()`, so `buildSyntheticCandidates(n)` returns byte-identical input on every call/machine/run -- the + * benchmark's numbers are comparable across runs precisely because its input never varies. + */ +export function buildSyntheticCandidates(count) { + const candidates = []; + for (let i = 0; i < count; i += 1) { + const timestamp = new Date(SYNTHETIC_EPOCH_MS + i * 3_600_000).toISOString(); + candidates.push({ + repoFullName: `bench-owner/bench-repo-${i % 7}`, + issueNumber: i + 1, + title: `Synthetic benchmark issue #${i + 1}`, + labels: [LABEL_POOL[i % LABEL_POOL.length]], + commentsCount: i % 11, + createdAt: timestamp, + updatedAt: timestamp, + htmlUrl: `https://github.com/bench-owner/bench-repo-${i % 7}/issues/${i + 1}`, + aiPolicyAllowed: i % 5 !== 0, + aiPolicySource: i % 5 === 0 ? "AI-USAGE.md" : "none", + }); + } + return candidates; +} + +function timeMs(fn) { + const start = performance.now(); + fn(); + return performance.now() - start; +} + +function median(values) { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; +} + +/** Rank the same synthetic candidate set `iterations` times and report the median wall time (#4845). */ +export function runRankingBenchmark(options = {}) { + const candidateCount = options.candidateCount ?? DEFAULT_CANDIDATE_COUNT; + const iterations = options.iterations ?? DEFAULT_ITERATIONS; + const candidates = buildSyntheticCandidates(candidateCount); + const samples = []; + for (let i = 0; i < iterations; i += 1) { + samples.push(timeMs(() => rankCandidateIssues(candidates, { nowMs: SYNTHETIC_EPOCH_MS }))); + } + const medianMs = median(samples); + return { + name: "discovery-fanout-ranking", + unitCount: candidateCount, + iterations, + medianMs, + opsPerSecond: candidateCount / (medianMs / 1000), + }; +} + +/** + * Enqueue then dequeue `operationCount` items against a fresh in-memory store, `iterations` times, and report + * the median wall time -- the same read/write path every real enqueue/claim exercises against the on-disk file, + * minus filesystem I/O, so the number isolates the query-plan/schema cost this package actually controls. + */ +export function runLocalStoreBenchmark(options = {}) { + const operationCount = options.operationCount ?? DEFAULT_QUEUE_OPERATION_COUNT; + const iterations = options.iterations ?? DEFAULT_ITERATIONS; + const samples = []; + for (let i = 0; i < iterations; i += 1) { + const store = initPortfolioQueueStore(":memory:"); + try { + samples.push( + timeMs(() => { + for (let n = 0; n < operationCount; n += 1) { + store.enqueue({ + repoFullName: `bench-owner/bench-repo-${n % 7}`, + identifier: `issue-${n}`, + priority: n % 100, + }); + } + for (let n = 0; n < operationCount; n += 1) { + store.dequeueNext(); + } + }), + ); + } finally { + store.close(); + } + } + const medianMs = median(samples); + return { + name: "local-store-read-write", + unitCount: operationCount * 2, + iterations, + medianMs, + opsPerSecond: (operationCount * 2) / (medianMs / 1000), + }; +} + +/** Render benchmark results as a stable, greppable text report (no locale-dependent number formatting). */ +export function formatBenchmarkReport(results) { + const lines = ["gittensory-miner benchmark", ""]; + for (const result of results) { + lines.push( + `${result.name}: median ${result.medianMs.toFixed(2)}ms over ${result.iterations} runs, ` + + `${Math.round(result.opsPerSecond)} ops/sec (n=${result.unitCount})`, + ); + } + return lines.join("\n"); +} + +function main() { + const results = [runRankingBenchmark(), runLocalStoreBenchmark()]; + console.log(formatBenchmarkReport(results)); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} diff --git a/test/unit/miner-benchmark-script.test.ts b/test/unit/miner-benchmark-script.test.ts new file mode 100644 index 000000000..d6f8d35ee --- /dev/null +++ b/test/unit/miner-benchmark-script.test.ts @@ -0,0 +1,84 @@ +import { spawnSync } from "node:child_process"; +import { describe, expect, it } from "vitest"; +import { + DEFAULT_CANDIDATE_COUNT, + DEFAULT_QUEUE_OPERATION_COUNT, + buildSyntheticCandidates, + formatBenchmarkReport, + runLocalStoreBenchmark, + runRankingBenchmark, +} from "../../packages/gittensory-miner/scripts/benchmark.mjs"; + +describe("gittensory-miner benchmark script (#4845)", () => { + it("REGRESSION: synthetic candidate generation is byte-for-byte deterministic across calls", () => { + expect(buildSyntheticCandidates(50)).toEqual(buildSyntheticCandidates(50)); + }); + + it("generates exactly the requested candidate count with valid ranker input shape", () => { + const candidates = buildSyntheticCandidates(12); + expect(candidates).toHaveLength(12); + for (const candidate of candidates) { + expect(candidate.repoFullName).toMatch(/^bench-owner\/bench-repo-\d$/); + expect(Number.isInteger(candidate.issueNumber)).toBe(true); + expect(candidate.issueNumber).toBeGreaterThan(0); + expect(typeof candidate.title).toBe("string"); + expect(Array.isArray(candidate.labels)).toBe(true); + } + }); + + it("runs the discovery fan-out ranking benchmark and reports a positive, finite result", () => { + const result = runRankingBenchmark({ candidateCount: 20, iterations: 2 }); + expect(result.name).toBe("discovery-fanout-ranking"); + expect(result.unitCount).toBe(20); + expect(result.iterations).toBe(2); + expect(Number.isFinite(result.medianMs)).toBe(true); + expect(result.medianMs).toBeGreaterThanOrEqual(0); + expect(Number.isFinite(result.opsPerSecond)).toBe(true); + expect(result.opsPerSecond).toBeGreaterThan(0); + }); + + it("runs the local-store read/write benchmark and reports a positive, finite result", () => { + const result = runLocalStoreBenchmark({ operationCount: 20, iterations: 2 }); + expect(result.name).toBe("local-store-read-write"); + expect(result.unitCount).toBe(40); + expect(result.iterations).toBe(2); + expect(Number.isFinite(result.medianMs)).toBe(true); + expect(result.medianMs).toBeGreaterThanOrEqual(0); + expect(Number.isFinite(result.opsPerSecond)).toBe(true); + expect(result.opsPerSecond).toBeGreaterThan(0); + }); + + it("falls back to the documented default counts when no options are passed", () => { + const ranking = runRankingBenchmark(); + const localStore = runLocalStoreBenchmark(); + expect(ranking.unitCount).toBe(DEFAULT_CANDIDATE_COUNT); + expect(localStore.unitCount).toBe(DEFAULT_QUEUE_OPERATION_COUNT * 2); + }); + + it("renders a deterministic report with no locale-dependent number formatting", () => { + expect( + formatBenchmarkReport([ + { name: "discovery-fanout-ranking", unitCount: 500, iterations: 5, medianMs: 12.345, opsPerSecond: 40501.6 }, + { name: "local-store-read-write", unitCount: 1000, iterations: 5, medianMs: 8, opsPerSecond: 125000 }, + ]), + ).toBe( + [ + "gittensory-miner benchmark", + "", + "discovery-fanout-ranking: median 12.35ms over 5 runs, 40502 ops/sec (n=500)", + "local-store-read-write: median 8.00ms over 5 runs, 125000 ops/sec (n=1000)", + ].join("\n"), + ); + }); + + it("runs end-to-end as a CLI script and prints both benchmark lines", () => { + const result = spawnSync(process.execPath, ["packages/gittensory-miner/scripts/benchmark.mjs"], { + cwd: process.cwd(), + encoding: "utf8", + }); + expect(result.status).toBe(0); + expect(result.stdout).toContain("gittensory-miner benchmark"); + expect(result.stdout).toContain("discovery-fanout-ranking:"); + expect(result.stdout).toContain("local-store-read-write:"); + }); +});