tangle-network · drewstone · Jul 8, 2026 · Jul 8, 2026 · Jul 8, 2026 · Jul 8, 2026
diff --git a/bench/.gitignore b/bench/.gitignore
@@ -1,4 +1,4 @@
-.venv/
+.venv
 .venv-*/
 node_modules/
 *.log

diff --git a/bench/package.json b/bench/package.json
@@ -2,7 +2,7 @@
   "name": "@tangle-network/agent-bench",
   "version": "0.1.0",
   "type": "module",
-  "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
+  "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, \u2026) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
   "main": "src/index.ts",
   "types": "src/index.ts",
   "exports": {
@@ -15,10 +15,12 @@
     "gate-cli": "tsx src/gate-cli.mts",
     "run-benchmarks": "tsx src/run-benchmarks-cli.mts",
     "gate-report": "tsx src/corpus-report.mts corpus/finsearch.jsonl",
-    "terminal-compare": "tsx src/terminal-compare.ts"
+    "terminal-compare": "tsx src/terminal-compare.ts",
+    "self-improve": "tsx src/self-improve-swe.mts"
   },
   "dependencies": {
     "@tangle-network/agent-eval": "^0.106.3",
+    "@tangle-network/agent-knowledge": "^1.11.1",
     "@tangle-network/agent-runtime": "^0.89.0",
     "@tangle-network/sandbox": "^0.9.7"
   },

diff --git a/bench/pnpm-lock.yaml b/bench/pnpm-lock.yaml
diff --git a/bench/src/benchmarks/humaneval.ts b/bench/src/benchmarks/humaneval.ts
@@ -17,7 +17,7 @@
  */
 
 import { execFile } from 'node:child_process'
-import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { gunzipSync } from 'node:zlib'
@@ -41,9 +41,17 @@ export interface HumanEvalTask {
  *  selects a deeper slice (the later tasks are harder) so the worker has a
  *  correctable middle band rather than a saturated easy prefix. */
 export async function loadHumanEval(limit: number, offset = 0): Promise<HumanEvalTask[]> {
-  const res = await fetch(humanevalUrl)
-  if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`)
-  const gz = Buffer.from(await res.arrayBuffer())
+  // Prefer a locally-cached .jsonl.gz (HUMANEVAL_GZ) — the GitHub raw URL rate-limits
+  // (429) under repeated runs. Fall back to the network fetch when unset.
+  const localGz = process.env.HUMANEVAL_GZ
+  let gz: Buffer
+  if (localGz) {
+    gz = readFileSync(localGz)
+  } else {
+    const res = await fetch(humanevalUrl)
+    if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`)
+    gz = Buffer.from(await res.arrayBuffer())
+  }
   const text = gunzipSync(gz).toString('utf8')
   const tasks: HumanEvalTask[] = []
   for (const line of text.split('\n')) {

diff --git a/bench/src/benchmarks/swe-bench.ts b/bench/src/benchmarks/swe-bench.ts
@@ -58,12 +58,38 @@ export const swePatchOutput: OutputAdapter<string> = {
 }
 
 const DATASET = 'princeton-nlp/SWE-bench_Verified'
+const TEST_FILE_EXCLUDES = [
+  "':(exclude,glob)**/tests/**'",
+  "':(exclude,glob)**/test/**'",
+  "':(exclude,glob)test_*.py'",
+  "':(exclude,glob)**/test_*.py'",
+  "':(exclude,glob)*_test.py'",
+  "':(exclude,glob)**/*_test.py'",
+  "':(exclude,glob)conftest.py'",
+  "':(exclude,glob)**/conftest.py'",
+].join(' ')
 
 interface SweReport {
   resolved_instances?: number
   resolved_ids?: string[]
 }
 
+function shellQuote(value: string): string {
+  return `'${value.replace(/'/g, `'\\''`)}'`
+}
+
+function sweMetadata(task: BenchTask): { repo: string; base: string } {
+  const repo = String(task.metadata?.repo ?? '')
+  const base = String(task.metadata?.base_commit ?? '')
+  if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) {
+    throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`)
+  }
+  if (!/^[0-9a-f]{7,40}$/i.test(base)) {
+    throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`)
+  }
+  return { repo, base }
+}
+
 export function createSweBenchAdapter(): BenchmarkAdapter {
   return {
     name: 'swe-bench-verified',
@@ -79,15 +105,14 @@ export function createSweBenchAdapter(): BenchmarkAdapter {
     // agent only edits (the harness owns the checkout — a stochastic model can't be
     // trusted to clone to an exact path). `--quiet` keeps the exec output small.
     boxSetup(task) {
-      const repo = String(task.metadata?.repo ?? '')
-      const base = String(task.metadata?.base_commit ?? '')
+      const { repo, base } = sweMetadata(task)
       return {
-        command: `rm -rf ${SWE_REPO_DIR} && git clone --quiet https://github.com/${repo} ${SWE_REPO_DIR} && git -C ${SWE_REPO_DIR} checkout --quiet ${base}`,
+        command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`,
       }
     },
     boxExtract() {
       return {
-        command: `git -C ${SWE_REPO_DIR} add -A && git -C ${SWE_REPO_DIR} diff --cached -- . ':(exclude)*/test*'`,
+        command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`,
       }
     },
 
@@ -175,7 +200,12 @@ print(json.dumps(out))
           '--run_id', runId,
           '--instance_ids', task.id,
           '--max_workers', '1',
-          '--cache_level', 'env',
+          // 'env' (the default) REMOVES the per-instance image after each run — measured: it
+          // silently pruned the 23-image fingerprint cache between sessions, and it forces a
+          // re-pull between back-to-back judges of the same instance (the stream judges F then L).
+          // Drivers that manage image rotation themselves set SWEBENCH_CACHE_LEVEL=instance and
+          // delete images explicitly.
+          '--cache_level', process.env.SWEBENCH_CACHE_LEVEL ?? 'env',
         ],
         async parseReport(dir) {
           // Report file: agent-runtime-bench.<run_id>.json

diff --git a/bench/src/hev-eval.mts b/bench/src/hev-eval.mts
@@ -0,0 +1,69 @@
+/**
+ * Minimal HumanEval evaluator: given an INSTRUCTION (env) + a fixed task set, run the
+ * worker model on each task and print the pass rate + the per-task result. Used to
+ * measure a baseline instruction vs a proposer-supplied instruction on the SAME
+ * held-out set (the proposer proposes; this grades — kept separate for honesty).
+ *
+ *   INSTRUCTION="..." IDS=HumanEval/55,... WORKER_MODEL=... ROUTER_BASE=... TANGLE_API_KEY=... \
+ *     HUMANEVAL_GZ=/abs/HumanEval.jsonl.gz tsx src/hev-eval.mts
+ */
+import { readFileSync } from 'node:fs'
+import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval'
+
+const SEED_INSTRUCTION =
+  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'
+
+async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise<string> {
+  const res = await fetch(`${base}/chat/completions`, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
+    body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }),
+  })
+  if (!res.ok) return ''
+  const d = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
+  return d.choices?.[0]?.message?.content ?? ''
+}
+
+async function main(): Promise<void> {
+  const key = process.env.TANGLE_API_KEY
+  if (!key) throw new Error('TANGLE_API_KEY required')
+  const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1'
+  const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite'
+  const instruction = process.env.INSTRUCTION_FILE
+    ? readFileSync(process.env.INSTRUCTION_FILE, 'utf8')
+    : (process.env.INSTRUCTION ?? SEED_INSTRUCTION)
+  const maxTokens = Number(process.env.MAX_TOKENS ?? 2500)
+  const conc = Number(process.env.CONC ?? 6)
+  const offset = Number(process.env.OFFSET ?? 55)
+  const n = Number(process.env.N ?? 40)
+  const idsEnv = (process.env.IDS ?? '').split(',').map((s) => s.trim()).filter(Boolean)
+
+  const all = await loadHumanEval(164, 0)
+  const byId = new Map(all.map((t) => [t.taskId, t]))
+  const tasks: HumanEvalTask[] = idsEnv.length
+    ? idsEnv.map((id) => byId.get(id)).filter((t): t is HumanEvalTask => !!t)
+    : all.slice(offset, offset + n)
+
+  console.log(`eval model=${model} n=${tasks.length} instr_len=${instruction.length}`)
+  let pass = 0
+  const fails: string[] = []
+  // simple concurrency pool
+  let i = 0
+  async function worker(): Promise<void> {
+    while (i < tasks.length) {
+      const t = tasks[i++]
+      const reply = await complete(base, key, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens)
+      const { pass: p } = await runChecker(t, extractCode(reply))
+      if (p === 1) pass += 1
+      else fails.push(t.taskId)
+    }
+  }
+  await Promise.all(Array.from({ length: conc }, () => worker()))
+  console.log(`PASS ${pass}/${tasks.length} = ${((100 * pass) / tasks.length).toFixed(1)}%`)
+  console.log(`FAILED: ${fails.sort().join(', ')}`)
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})