Update benchmark behavior

src/merge-results.ts

@@ -101,9 +101,15 @@ async function main() {
   for (const file of jsonFiles) {
     const raw: ResultFile = JSON.parse(fs.readFileSync(file, 'utf-8'));
     const fromSingleProvider = raw.results.length === 1;
+    const dirName = path.basename(path.dirname(file));
+    const isSandboxDir = dirName === 'sequential_tti' || dirName === 'staggered_tti' || dirName === 'burst_tti';
+
+    if (!isSandboxDir) {
+      continue;
+    }
+
     for (const result of raw.results) {
       // Determine mode from the directory name (e.g. sequential_tti, burst_tti)
-      const dirName = path.basename(path.dirname(file));
       let mode = normalizeMode(result.mode || 'sequential');
       // Infer from directory name if available
       if (dirName.includes('sequential')) mode = 'sequential';

src/sandbox/benchmark.ts

@@ -1,6 +1,7 @@
 import type { ProviderConfig, BenchmarkResult, TimingResult } from './types.js';
 import { computeStats } from '../util/stats.js';
 import { withTimeout } from '../util/timeout.js';
+import { randomUUID } from 'node:crypto';
 
 export async function runBenchmark(config: ProviderConfig): Promise<BenchmarkResult> {
   const { name, iterations = 100, timeout = 120_000, requiredEnvVars, sandboxOptions, destroyTimeoutMs } = config;
@@ -19,14 +20,25 @@ export async function runBenchmark(config: ProviderConfig): Promise<BenchmarkRes
 
   const compute = config.createCompute();
   const results: TimingResult[] = [];
+  const runNonce = randomUUID();
+  const reuseDetector = {
+    runNonce,
+    seenSignals: new Map<string, Set<string>>(),
+  };
 
   console.log(`\n--- Benchmarking: ${name} (${iterations} iterations) ---`);
 
   for (let i = 0; i < iterations; i++) {
     console.log(`  Iteration ${i + 1}/${iterations}...`);
 
     try {
-      const iterationResult = await runIteration(compute, timeout, sandboxOptions, destroyTimeoutMs);
+      const iterationResult = await runIteration(
+        compute,
+        timeout,
+        sandboxOptions,
+        destroyTimeoutMs,
+        reuseDetector,
+      );
       results.push(iterationResult);
       console.log(`    TTI: ${(iterationResult.ttiMs / 1000).toFixed(2)}s`);
     } catch (err) {
@@ -49,14 +61,123 @@ export async function runBenchmark(config: ProviderConfig): Promise<BenchmarkRes
   };
 }
 
-export async function runIteration(compute: any, timeout: number, sandboxOptions?: Record<string, any>, destroyTimeoutMs: number = 15_000): Promise<TimingResult> {
+type ReuseDetector = {
+  runNonce: string;
+  seenSignals: Map<string, Set<string>>;
+};
+
+const STRONG_SIGNAL_KEYS = ['ns_mnt', 'ns_pid', 'ns_uts', 'cgroup_hash', 'boot_id', 'pid1'] as const;
+
+function parseKeyValueOutput(stdout: string): Record<string, string> {
+  const parsed: Record<string, string> = {};
+  for (const line of stdout.split('\n')) {
+    const index = line.indexOf('=');
+    if (index <= 0) continue;
+    const key = line.slice(0, index).trim();
+    const value = line.slice(index + 1).trim();
+    if (!key) continue;
+    parsed[key] = value;
+  }
+  return parsed;
+}
+
+function countStrongSignalMatches(identity: Record<string, string>, detector: ReuseDetector): number {
+  let matches = 0;
+
+  for (const key of STRONG_SIGNAL_KEYS) {
+    const value = identity[key];
+    if (!value || value === 'unknown') continue;
+    const seen = detector.seenSignals.get(key);
+    if (seen?.has(value)) matches++;
+  }
+
+  return matches;
+}
+
+function rememberSignals(identity: Record<string, string>, detector: ReuseDetector): void {
+  for (const key of STRONG_SIGNAL_KEYS) {
+    const value = identity[key];
+    if (!value || value === 'unknown') continue;
+    if (!detector.seenSignals.has(key)) detector.seenSignals.set(key, new Set<string>());
+    detector.seenSignals.get(key)!.add(value);
+  }
+}
+
+export async function runIteration(
+  compute: any,
+  timeout: number,
+  sandboxOptions?: Record<string, any>,
+  destroyTimeoutMs: number = 15_000,
+  reuseDetector?: ReuseDetector,
+): Promise<TimingResult> {
   let sandbox: any = null;
 
   try {
     const start = performance.now();
 
     sandbox = await withTimeout(compute.sandbox.create(sandboxOptions), timeout, 'Sandbox creation timed out');
 
+    const markerA = '/tmp/.bench_ephemeral_check';
+    const markerB = '/var/tmp/.bench_ephemeral_check';
+    const probeToken = reuseDetector
+      ? `${reuseDetector.runNonce}:${Date.now().toString(36)}:${Math.random().toString(36).slice(2, 10)}`
+      : `${Date.now().toString(36)}:${Math.random().toString(36).slice(2, 10)}`;
+
+    const identityProbeCommand = [
+      `marker_a='${markerA}'`,
+      `marker_b='${markerB}'`,
+      "marker_path=''",
+      "for p in \"$marker_a\" \"$marker_b\"; do if [ -f \"$p\" ]; then marker_path=$p; break; fi; done",
+      "marker_value='unknown'",
+      "if [ -n \"$marker_path\" ]; then marker_value=$(tr -d '\\n' < \"$marker_path\" 2>/dev/null || true); fi",
+      "ns_mnt=$(readlink /proc/self/ns/mnt 2>/dev/null || printf unknown)",
+      "ns_pid=$(readlink /proc/self/ns/pid 2>/dev/null || printf unknown)",
+      "ns_uts=$(readlink /proc/self/ns/uts 2>/dev/null || printf unknown)",
+      "cgroup_hash=$(cat /proc/self/cgroup 2>/dev/null | sha256sum 2>/dev/null | cut -d\" \" -f1)",
+      "if [ -z \"$cgroup_hash\" ]; then cgroup_hash=unknown; fi",
+      "boot_id=$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || printf unknown)",
+      "pid1=$(tr -d '\\0' < /proc/1/cmdline 2>/dev/null || printf unknown)",
+      "uptime=$(cut -d' ' -f1 /proc/uptime 2>/dev/null || printf unknown)",
+      "printf 'marker_path=%s\\n' \"$marker_path\"",
+      "printf 'marker_value=%s\\n' \"$marker_value\"",
+      "printf 'ns_mnt=%s\\n' \"$ns_mnt\"",
+      "printf 'ns_pid=%s\\n' \"$ns_pid\"",
+      "printf 'ns_uts=%s\\n' \"$ns_uts\"",
+      "printf 'cgroup_hash=%s\\n' \"$cgroup_hash\"",
+      "printf 'boot_id=%s\\n' \"$boot_id\"",
+      "printf 'pid1=%s\\n' \"$pid1\"",
+      "printf 'uptime=%s\\n' \"$uptime\"",
+      `printf '%s' '${probeToken}' > ${markerA}`,
+      `printf '%s' '${probeToken}' > ${markerB}`,
+    ].join('; ');
+
+    const identityResult = await withTimeout(
+      sandbox.runCommand(identityProbeCommand),
+      30_000,
+      'Sandbox identity check timed out'
+    ) as { exitCode: number; stdout?: string; stderr?: string };
+
+    if (identityResult.exitCode !== 0) {
+      throw new Error(`Sandbox identity check failed with exit code ${identityResult.exitCode}: ${identityResult.stderr || 'Unknown error'}`);
+    }
+
+    const identity = parseKeyValueOutput(identityResult.stdout || '');
+
+    if (reuseDetector) {
+      if (identity.marker_path) {
+        throw new Error(`Sandbox/container reuse detected: persistent marker at ${identity.marker_path}`);
+      }
+
+      const strongMatches = countStrongSignalMatches(identity, reuseDetector);
+      if (strongMatches >= 3) {
+        if (process.env.BENCH_REUSE_DEBUG === '1') {
+          console.warn(`    [reuse-check] Sandbox/container reuse suspected: ${strongMatches} strong runtime signals repeated`);
+        }
+      }
+
+      rememberSignals(identity, reuseDetector);
+    }
+
     const result = await withTimeout(
       sandbox.runCommand('node -v'),
       30_000,
@@ -88,4 +209,3 @@ export async function runIteration(compute: any, timeout: number, sandboxOptions
     }
   }
 }
-

src/sandbox/concurrent.ts

@@ -1,6 +1,7 @@
 import type { ProviderConfig, TimingResult, ConcurrentBenchmarkResult } from './types.js';
 import { runIteration } from './benchmark.js';
 import { computeStats } from '../util/stats.js';
+import { randomUUID } from 'node:crypto';
 
 interface ConcurrentConfig extends ProviderConfig {
   concurrency: number;
@@ -26,14 +27,17 @@ export async function runConcurrentBenchmark(config: ConcurrentConfig): Promise<
   }
 
   const compute = config.createCompute();
-
   console.log(`\n--- Concurrent Benchmark: ${name} (${concurrency} sandboxes) ---`);
 
   const wallStart = performance.now();
+  const reuseDetector = {
+    runNonce: randomUUID(),
+    seenSignals: new Map<string, Set<string>>(),
+  };
 
   // Fire all sandbox creations simultaneously — no awaiting between launches
   const promises = Array.from({ length: concurrency }, (_, i) =>
-    runIteration(compute, timeout, sandboxOptions, destroyTimeoutMs)
+    runIteration(compute, timeout, sandboxOptions, destroyTimeoutMs, reuseDetector)
       .then(result => {
         console.log(`  Sandbox ${i + 1}/${concurrency}: TTI ${(result.ttiMs / 1000).toFixed(2)}s`);
         return result;

src/sandbox/staggered.ts

@@ -1,6 +1,7 @@
 import type { ProviderConfig, TimingResult, StaggeredBenchmarkResult } from './types.js';
 import { runIteration } from './benchmark.js';
 import { computeStats } from '../util/stats.js';
+import { randomUUID } from 'node:crypto';
 
 interface StaggeredConfig extends ProviderConfig {
   concurrency: number;
@@ -29,17 +30,20 @@ export async function runStaggeredBenchmark(config: StaggeredConfig): Promise<St
   }
 
   const compute = config.createCompute();
-
   console.log(`\n--- Staggered Benchmark: ${name} (${concurrency} sandboxes, ${staggerDelayMs}ms apart) ---`);
 
   const wallStart = performance.now();
+  const reuseDetector = {
+    runNonce: randomUUID(),
+    seenSignals: new Map<string, Set<string>>(),
+  };
   const promises: Promise<TimingResult>[] = [];
   const rampProfile: { launchedAt: number; readyAt: number; ttiMs: number }[] = [];
 
   for (let i = 0; i < concurrency; i++) {
     const launchedAt = performance.now() - wallStart;
 
-    const p = runIteration(compute, timeout, sandboxOptions, destroyTimeoutMs)
+    const p = runIteration(compute, timeout, sandboxOptions, destroyTimeoutMs, reuseDetector)
       .then(result => {
         const readyAt = performance.now() - wallStart;
         rampProfile.push({ launchedAt, readyAt, ttiMs: result.ttiMs });