diff --git a/src/merge-results.ts b/src/merge-results.ts index a75d651..eac75c2 100644 --- a/src/merge-results.ts +++ b/src/merge-results.ts @@ -101,9 +101,15 @@ async function main() { for (const file of jsonFiles) { const raw: ResultFile = JSON.parse(fs.readFileSync(file, 'utf-8')); const fromSingleProvider = raw.results.length === 1; + const dirName = path.basename(path.dirname(file)); + const isSandboxDir = dirName === 'sequential_tti' || dirName === 'staggered_tti' || dirName === 'burst_tti'; + + if (!isSandboxDir) { + continue; + } + for (const result of raw.results) { // Determine mode from the directory name (e.g. sequential_tti, burst_tti) - const dirName = path.basename(path.dirname(file)); let mode = normalizeMode(result.mode || 'sequential'); // Infer from directory name if available if (dirName.includes('sequential')) mode = 'sequential'; diff --git a/src/sandbox/benchmark.ts b/src/sandbox/benchmark.ts index fcb73bf..7cfcdce 100644 --- a/src/sandbox/benchmark.ts +++ b/src/sandbox/benchmark.ts @@ -1,6 +1,7 @@ import type { ProviderConfig, BenchmarkResult, TimingResult } from './types.js'; import { computeStats } from '../util/stats.js'; import { withTimeout } from '../util/timeout.js'; +import { randomUUID } from 'node:crypto'; export async function runBenchmark(config: ProviderConfig): Promise { const { name, iterations = 100, timeout = 120_000, requiredEnvVars, sandboxOptions, destroyTimeoutMs } = config; @@ -19,6 +20,11 @@ export async function runBenchmark(config: ProviderConfig): Promise>(), + }; console.log(`\n--- Benchmarking: ${name} (${iterations} iterations) ---`); @@ -26,7 +32,13 @@ export async function runBenchmark(config: ProviderConfig): Promise, destroyTimeoutMs: number = 15_000): Promise { +type ReuseDetector = { + runNonce: string; + seenSignals: Map>; +}; + +const STRONG_SIGNAL_KEYS = ['ns_mnt', 'ns_pid', 'ns_uts', 'cgroup_hash', 'boot_id', 'pid1'] as const; + +function parseKeyValueOutput(stdout: string): Record { + const parsed: Record = {}; + for (const line of stdout.split('\n')) { + const index = line.indexOf('='); + if (index <= 0) continue; + const key = line.slice(0, index).trim(); + const value = line.slice(index + 1).trim(); + if (!key) continue; + parsed[key] = value; + } + return parsed; +} + +function countStrongSignalMatches(identity: Record, detector: ReuseDetector): number { + let matches = 0; + + for (const key of STRONG_SIGNAL_KEYS) { + const value = identity[key]; + if (!value || value === 'unknown') continue; + const seen = detector.seenSignals.get(key); + if (seen?.has(value)) matches++; + } + + return matches; +} + +function rememberSignals(identity: Record, detector: ReuseDetector): void { + for (const key of STRONG_SIGNAL_KEYS) { + const value = identity[key]; + if (!value || value === 'unknown') continue; + if (!detector.seenSignals.has(key)) detector.seenSignals.set(key, new Set()); + detector.seenSignals.get(key)!.add(value); + } +} + +export async function runIteration( + compute: any, + timeout: number, + sandboxOptions?: Record, + destroyTimeoutMs: number = 15_000, + reuseDetector?: ReuseDetector, +): Promise { let sandbox: any = null; try { @@ -57,6 +117,67 @@ export async function runIteration(compute: any, timeout: number, sandboxOptions sandbox = await withTimeout(compute.sandbox.create(sandboxOptions), timeout, 'Sandbox creation timed out'); + const markerA = '/tmp/.bench_ephemeral_check'; + const markerB = '/var/tmp/.bench_ephemeral_check'; + const probeToken = reuseDetector + ? `${reuseDetector.runNonce}:${Date.now().toString(36)}:${Math.random().toString(36).slice(2, 10)}` + : `${Date.now().toString(36)}:${Math.random().toString(36).slice(2, 10)}`; + + const identityProbeCommand = [ + `marker_a='${markerA}'`, + `marker_b='${markerB}'`, + "marker_path=''", + "for p in \"$marker_a\" \"$marker_b\"; do if [ -f \"$p\" ]; then marker_path=$p; break; fi; done", + "marker_value='unknown'", + "if [ -n \"$marker_path\" ]; then marker_value=$(tr -d '\\n' < \"$marker_path\" 2>/dev/null || true); fi", + "ns_mnt=$(readlink /proc/self/ns/mnt 2>/dev/null || printf unknown)", + "ns_pid=$(readlink /proc/self/ns/pid 2>/dev/null || printf unknown)", + "ns_uts=$(readlink /proc/self/ns/uts 2>/dev/null || printf unknown)", + "cgroup_hash=$(cat /proc/self/cgroup 2>/dev/null | sha256sum 2>/dev/null | cut -d\" \" -f1)", + "if [ -z \"$cgroup_hash\" ]; then cgroup_hash=unknown; fi", + "boot_id=$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || printf unknown)", + "pid1=$(tr -d '\\0' < /proc/1/cmdline 2>/dev/null || printf unknown)", + "uptime=$(cut -d' ' -f1 /proc/uptime 2>/dev/null || printf unknown)", + "printf 'marker_path=%s\\n' \"$marker_path\"", + "printf 'marker_value=%s\\n' \"$marker_value\"", + "printf 'ns_mnt=%s\\n' \"$ns_mnt\"", + "printf 'ns_pid=%s\\n' \"$ns_pid\"", + "printf 'ns_uts=%s\\n' \"$ns_uts\"", + "printf 'cgroup_hash=%s\\n' \"$cgroup_hash\"", + "printf 'boot_id=%s\\n' \"$boot_id\"", + "printf 'pid1=%s\\n' \"$pid1\"", + "printf 'uptime=%s\\n' \"$uptime\"", + `printf '%s' '${probeToken}' > ${markerA}`, + `printf '%s' '${probeToken}' > ${markerB}`, + ].join('; '); + + const identityResult = await withTimeout( + sandbox.runCommand(identityProbeCommand), + 30_000, + 'Sandbox identity check timed out' + ) as { exitCode: number; stdout?: string; stderr?: string }; + + if (identityResult.exitCode !== 0) { + throw new Error(`Sandbox identity check failed with exit code ${identityResult.exitCode}: ${identityResult.stderr || 'Unknown error'}`); + } + + const identity = parseKeyValueOutput(identityResult.stdout || ''); + + if (reuseDetector) { + if (identity.marker_path) { + throw new Error(`Sandbox/container reuse detected: persistent marker at ${identity.marker_path}`); + } + + const strongMatches = countStrongSignalMatches(identity, reuseDetector); + if (strongMatches >= 3) { + if (process.env.BENCH_REUSE_DEBUG === '1') { + console.warn(` [reuse-check] Sandbox/container reuse suspected: ${strongMatches} strong runtime signals repeated`); + } + } + + rememberSignals(identity, reuseDetector); + } + const result = await withTimeout( sandbox.runCommand('node -v'), 30_000, @@ -88,4 +209,3 @@ export async function runIteration(compute: any, timeout: number, sandboxOptions } } } - diff --git a/src/sandbox/concurrent.ts b/src/sandbox/concurrent.ts index 16b2650..03ee9ce 100644 --- a/src/sandbox/concurrent.ts +++ b/src/sandbox/concurrent.ts @@ -1,6 +1,7 @@ import type { ProviderConfig, TimingResult, ConcurrentBenchmarkResult } from './types.js'; import { runIteration } from './benchmark.js'; import { computeStats } from '../util/stats.js'; +import { randomUUID } from 'node:crypto'; interface ConcurrentConfig extends ProviderConfig { concurrency: number; @@ -26,14 +27,17 @@ export async function runConcurrentBenchmark(config: ConcurrentConfig): Promise< } const compute = config.createCompute(); - console.log(`\n--- Concurrent Benchmark: ${name} (${concurrency} sandboxes) ---`); const wallStart = performance.now(); + const reuseDetector = { + runNonce: randomUUID(), + seenSignals: new Map>(), + }; // Fire all sandbox creations simultaneously — no awaiting between launches const promises = Array.from({ length: concurrency }, (_, i) => - runIteration(compute, timeout, sandboxOptions, destroyTimeoutMs) + runIteration(compute, timeout, sandboxOptions, destroyTimeoutMs, reuseDetector) .then(result => { console.log(` Sandbox ${i + 1}/${concurrency}: TTI ${(result.ttiMs / 1000).toFixed(2)}s`); return result; diff --git a/src/sandbox/staggered.ts b/src/sandbox/staggered.ts index 78e63af..c958472 100644 --- a/src/sandbox/staggered.ts +++ b/src/sandbox/staggered.ts @@ -1,6 +1,7 @@ import type { ProviderConfig, TimingResult, StaggeredBenchmarkResult } from './types.js'; import { runIteration } from './benchmark.js'; import { computeStats } from '../util/stats.js'; +import { randomUUID } from 'node:crypto'; interface StaggeredConfig extends ProviderConfig { concurrency: number; @@ -29,17 +30,20 @@ export async function runStaggeredBenchmark(config: StaggeredConfig): Promise>(), + }; const promises: Promise[] = []; const rampProfile: { launchedAt: number; readyAt: number; ttiMs: number }[] = []; for (let i = 0; i < concurrency; i++) { const launchedAt = performance.now() - wallStart; - const p = runIteration(compute, timeout, sandboxOptions, destroyTimeoutMs) + const p = runIteration(compute, timeout, sandboxOptions, destroyTimeoutMs, reuseDetector) .then(result => { const readyAt = performance.now() - wallStart; rampProfile.push({ launchedAt, readyAt, ttiMs: result.ttiMs });