diff --git a/.github/workflows/linux-kernel-builder-benchmarks.yml b/.github/workflows/linux-kernel-builder-benchmarks.yml new file mode 100644 index 0000000..492ba32 --- /dev/null +++ b/.github/workflows/linux-kernel-builder-benchmarks.yml @@ -0,0 +1,180 @@ +name: Linux Kernel Builder Benchmark + +on: + push: + paths: + - 'src/linux-kernel-builder/**' + - 'src/util/**' + - 'src/run.ts' + - 'package.json' + - '.github/workflows/linux-kernel-builder-benchmarks.yml' + pull_request: + paths: + - 'src/linux-kernel-builder/**' + - 'src/util/**' + - 'src/run.ts' + - 'package.json' + - '.github/workflows/linux-kernel-builder-benchmarks.yml' + workflow_dispatch: + inputs: + iterations: + description: 'Iterations per provider' + required: false + default: '1' + +concurrency: + group: linux-kernel-builder-benchmarks + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + +jobs: + bench: + name: LKB ${{ matrix.provider }} + runs-on: namespace-profile-default + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + provider: + - blaxel + - daytona + - e2b + - hopx + - namespace + - runloop + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: 'npm' + - name: Install dependencies + run: npm ci + - name: Run Linux Kernel Builder benchmark (LKB) + env: + BL_API_KEY: ${{ secrets.BL_API_KEY }} + BL_WORKSPACE: ${{ secrets.BL_WORKSPACE }} + DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} + E2B_API_KEY: ${{ secrets.E2B_API_KEY }} + HOPX_API_KEY: ${{ secrets.HOPX_API_KEY }} + NSC_TOKEN: ${{ secrets.NSC_TOKEN }} + RUNLOOP_API_KEY: ${{ secrets.RUNLOOP_API_KEY }} + run: | + npm run bench -- \ + --mode linux-kernel-builder \ + --provider ${{ matrix.provider }} \ + --iterations ${{ github.event.inputs.iterations || '1' }} + - name: Upload Linux Kernel Builder artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: linux-kernel-builder-${{ matrix.provider }} + path: results/linux_kernel_builder/ + if-no-files-found: ignore + retention-days: 7 + + collect: + name: Collect LKB Results + runs-on: namespace-profile-default + needs: bench + if: always() && github.event_name == 'pull_request' + steps: + - uses: actions/checkout@v4 + - name: Download all LKB artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts/ + pattern: linux-kernel-builder-* + - name: Post results to PR + continue-on-error: true + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + function walk(dir, out) { + if (!fs.existsSync(dir)) return; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full, out); + else if (entry.name === 'latest.json') out.push(full); + } + } + + const latestFiles = []; + walk('artifacts', latestFiles); + + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const marker = '## Linux Kernel Builder Benchmark Results'; + + let body = `${marker}\n\n`; + + if (latestFiles.length === 0) { + body += '> No Linux Kernel Builder results were generated.\n\n'; + } else { + const byProvider = new Map(); + + for (const file of latestFiles) { + const raw = JSON.parse(fs.readFileSync(file, 'utf-8')); + for (const r of raw.results || []) { + byProvider.set(r.provider, r); + } + } + + const results = Array.from(byProvider.values()) + .sort((a, b) => (b.compositeScore || 0) - (a.compositeScore || 0)); + + body += '| # | Provider | Score | Build Median | Build P95 | Build P99 | Status | Sample Failure |\n'; + body += '|---|----------|-------|--------------|-----------|-----------|--------|----------------|\n'; + + results.forEach((r, i) => { + if (r.skipped) { + const reason = (r.skipReason || '').replace(/\n/g, ' ').slice(0, 120); + body += `| ${i + 1} | ${r.provider} | -- | -- | -- | -- | SKIPPED | ${reason || '--'} |\n`; + return; + } + + const score = r.compositeScore !== undefined ? r.compositeScore.toFixed(1) : '--'; + const ok = (r.iterations || []).filter(it => !it.error).length; + const total = (r.iterations || []).length; + const hasSuccess = ok > 0; + const med = hasSuccess && r.summary?.buildMs ? `${(r.summary.buildMs.median / 1000).toFixed(2)}s` : '--'; + const p95 = hasSuccess && r.summary?.buildMs ? `${(r.summary.buildMs.p95 / 1000).toFixed(2)}s` : '--'; + const p99 = hasSuccess && r.summary?.buildMs ? `${(r.summary.buildMs.p99 / 1000).toFixed(2)}s` : '--'; + const firstError = (r.iterations || []).find(it => it.error)?.error || ''; + const compactError = firstError.replace(/\n/g, ' ').slice(0, 120); + body += `| ${i + 1} | ${r.provider} | ${score} | ${med} | ${p95} | ${p99} | ${ok}/${total} | ${compactError || '--'} |\n`; + }); + + body += '\n'; + } + + body += `---\n*[View full run](${runUrl}) ยท JSON artifacts available in the run artifacts*`; + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const existing = comments.find(c => c.body.startsWith(marker)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } diff --git a/package.json b/package.json index 932c58d..bcb0622 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,13 @@ "bench:vercel": "tsx src/run.ts --provider vercel", "bench:just-bash": "tsx src/run.ts --provider just-bash", "bench:sprites": "tsx src/run.ts --provider sprites", + "bench:linux-kernel-builder": "tsx src/run.ts --mode linux-kernel-builder", + "bench:linux-kernel-builder:blaxel": "tsx src/run.ts --mode linux-kernel-builder --provider blaxel", + "bench:linux-kernel-builder:daytona": "tsx src/run.ts --mode linux-kernel-builder --provider daytona", + "bench:linux-kernel-builder:e2b": "tsx src/run.ts --mode linux-kernel-builder --provider e2b", + "bench:linux-kernel-builder:hopx": "tsx src/run.ts --mode linux-kernel-builder --provider hopx", + "bench:linux-kernel-builder:namespace": "tsx src/run.ts --mode linux-kernel-builder --provider namespace", + "bench:linux-kernel-builder:runloop": "tsx src/run.ts --mode linux-kernel-builder --provider runloop", "bench:browser": "tsx src/run.ts --mode browser", "bench:browser:browserbase": "tsx src/run.ts --mode browser --provider browserbase", "bench:storage": "tsx src/run.ts --mode storage", diff --git a/src/linux-kernel-builder/benchmark.ts b/src/linux-kernel-builder/benchmark.ts new file mode 100644 index 0000000..3d2ea17 --- /dev/null +++ b/src/linux-kernel-builder/benchmark.ts @@ -0,0 +1,203 @@ +import { computeStats } from '../util/stats.js'; +import { withTimeout } from '../util/timeout.js'; +import type { LinuxKernelBuilderBenchmarkResult, LinuxKernelBuilderProviderConfig, LinuxKernelBuilderTimingResult } from './types.js'; +import { KERNEL_VERSION, WORKLOAD, WORKLOAD_ACRONYM, WORKLOAD_LABEL } from './types.js'; + +const KERNEL_TARBALL = `linux-${KERNEL_VERSION}.tar.xz`; +const KERNEL_URL = `https://cdn.kernel.org/pub/linux/kernel/v6.x/${KERNEL_TARBALL}`; +const KERNEL_SHA256 = '643142c1b5991560dd12f950825cc19e4497b95b82641918ecff1177f4130c1d'; + +function parseResultLine(output: string): { buildMs: number; cpuCount?: number; memTotalKb?: number } | null { + const markerLine = output + .split('\n') + .find(line => line.startsWith('__LKB_RESULT__')); + + if (!markerLine) return null; + + const build = markerLine.match(/buildMs=(\d+)/); + const cpu = markerLine.match(/cpu=(\d+)/); + const mem = markerLine.match(/memKb=(\d+)/); + + if (!build) return null; + + return { + buildMs: Number(build[1]), + cpuCount: cpu ? Number(cpu[1]) : undefined, + memTotalKb: mem ? Number(mem[1]) : undefined, + }; +} + +function getKernelBuildScript(): string { + return [ + 'set -euo pipefail', + 'WORKDIR=/tmp/lkb-linux', + 'rm -rf "$WORKDIR"', + 'mkdir -p "$WORKDIR"', + 'cd "$WORKDIR"', + 'if command -v curl >/dev/null 2>&1; then', + ` curl -fsSL --retry 3 "${KERNEL_URL}" -o "${KERNEL_TARBALL}"`, + 'elif command -v wget >/dev/null 2>&1; then', + ` wget -qO "${KERNEL_TARBALL}" "${KERNEL_URL}"`, + 'else', + ' echo "Missing downloader: curl or wget"', + ' exit 2', + 'fi', + `echo "${KERNEL_SHA256} ${KERNEL_TARBALL}" | sha256sum -c -`, + `tar -xf "${KERNEL_TARBALL}"`, + `cd "linux-${KERNEL_VERSION}"`, + 'for cmd in make gcc ld bison flex bc; do', + ' command -v "$cmd" >/dev/null 2>&1 || { echo "Missing dependency: $cmd"; exit 3; }', + 'done', + 'make defconfig >/dev/null', + 'START_NS=$(date +%s%N)', + 'make -j"$(nproc)" bzImage >/tmp/lkb-build.log 2>&1', + 'END_NS=$(date +%s%N)', + 'BUILD_MS=$(( (END_NS - START_NS) / 1000000 ))', + 'CPU_COUNT=$(nproc)', + "MEM_TOTAL_KB=$(awk '/MemTotal:/ { print $2 }' /proc/meminfo)", + 'echo "__LKB_RESULT__ buildMs=${BUILD_MS} cpu=${CPU_COUNT} memKb=${MEM_TOTAL_KB}"', + ].join('\n'); +} + +async function runLinuxKernelBuilderIteration( + compute: any, + timeout: number, + sandboxOptions?: Record, + destroyTimeoutMs: number = 15_000, +): Promise { + let sandbox: any = null; + + try { + sandbox = await withTimeout( + compute.sandbox.create(sandboxOptions), + timeout, + 'Sandbox creation timed out', + ); + + const iterationStart = performance.now(); + const command = [ + "cat <<'__LKB_SCRIPT__' >/tmp/lkb.sh", + getKernelBuildScript(), + '__LKB_SCRIPT__', + 'bash /tmp/lkb.sh', + ].join('\n'); + + const result = await withTimeout( + sandbox.runCommand(command), + timeout, + 'Kernel build command timed out', + ) as { exitCode: number; stdout?: string; stderr?: string }; + + if (result.exitCode !== 0) { + const details = [result.stderr, result.stdout] + .filter(Boolean) + .join('\n') + .trim(); + throw new Error(`Command failed with exit code ${result.exitCode}${details ? `: ${details}` : ''}`); + } + + const totalMs = performance.now() - iterationStart; + const parsed = parseResultLine(`${result.stdout || ''}\n${result.stderr || ''}`); + if (!parsed) { + throw new Error('Unable to parse build metrics from command output'); + } + + return { + totalMs, + buildMs: parsed.buildMs, + cpuCount: parsed.cpuCount, + memTotalKb: parsed.memTotalKb, + }; + } finally { + if (sandbox) { + let timer: ReturnType | undefined; + try { + await Promise.race([ + sandbox.destroy(), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('Destroy timeout')), destroyTimeoutMs); + }), + ]); + } catch (err) { + console.warn(` [cleanup] destroy failed: ${err instanceof Error ? err.message : String(err)}`); + } finally { + if (timer) clearTimeout(timer); + } + } + } +} + +export async function runLinuxKernelBuilderBenchmark(config: LinuxKernelBuilderProviderConfig): Promise { + const { + name, + iterations = 5, + timeout = 2_700_000, + requiredEnvVars, + sandboxOptions, + destroyTimeoutMs, + } = config; + + const missingVars = requiredEnvVars.filter(v => !process.env[v]); + if (missingVars.length > 0) { + return { + provider: name, + mode: 'linux-kernel-builder', + workload: WORKLOAD, + workloadAcronym: WORKLOAD_ACRONYM, + kernelVersion: KERNEL_VERSION, + iterations: [], + summary: { + buildMs: { median: 0, p95: 0, p99: 0 }, + totalMs: { median: 0, p95: 0, p99: 0 }, + }, + skipped: true, + skipReason: `Missing: ${missingVars.join(', ')}`, + }; + } + + const compute = config.createCompute(); + const results: LinuxKernelBuilderTimingResult[] = []; + + console.log(`\n--- ${WORKLOAD_LABEL} (${WORKLOAD_ACRONYM}): ${name} (${iterations} iterations, linux-${KERNEL_VERSION}) ---`); + + for (let i = 0; i < iterations; i++) { + console.log(` Iteration ${i + 1}/${iterations}...`); + + try { + const iterationResult = await runLinuxKernelBuilderIteration( + compute, + timeout, + sandboxOptions, + destroyTimeoutMs, + ); + results.push(iterationResult); + console.log( + ` Build: ${(iterationResult.buildMs / 1000).toFixed(2)}s | Total: ${(iterationResult.totalMs / 1000).toFixed(2)}s` + + `${iterationResult.cpuCount ? ` | CPU: ${iterationResult.cpuCount}` : ''}`, + ); + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + console.log(` FAILED: ${error}`); + results.push({ totalMs: 0, buildMs: 0, error }); + } + } + + const successful = results.filter(r => !r.error); + + return { + provider: name, + mode: 'linux-kernel-builder', + workload: WORKLOAD, + workloadAcronym: WORKLOAD_ACRONYM, + kernelVersion: KERNEL_VERSION, + iterations: results, + summary: { + buildMs: successful.length > 0 + ? computeStats(successful.map(r => r.buildMs)) + : { median: 0, p95: 0, p99: 0 }, + totalMs: successful.length > 0 + ? computeStats(successful.map(r => r.totalMs)) + : { median: 0, p95: 0, p99: 0 }, + }, + }; +} diff --git a/src/linux-kernel-builder/providers.ts b/src/linux-kernel-builder/providers.ts new file mode 100644 index 0000000..3a6e610 --- /dev/null +++ b/src/linux-kernel-builder/providers.ts @@ -0,0 +1,48 @@ +import { e2b } from '@computesdk/e2b'; +import { daytona } from '@computesdk/daytona'; +import { blaxel } from '@computesdk/blaxel'; +import { hopx } from '@computesdk/hopx'; +import { namespace } from '@computesdk/namespace'; +import { runloop } from '@computesdk/runloop'; +import type { LinuxKernelBuilderProviderConfig } from './types.js'; + +/** + * Providers that participate in sustained compute benchmark runs. + * + * Keep this list intentionally narrow to providers that position around + * runner-style or general-purpose compute performance. + */ +export const linuxKernelBuilderProviders: LinuxKernelBuilderProviderConfig[] = [ + { + name: 'blaxel', + requiredEnvVars: ['BL_API_KEY', 'BL_WORKSPACE'], + createCompute: () => blaxel({ apiKey: process.env.BL_API_KEY!, workspace: process.env.BL_WORKSPACE!, region: 'us-was-1' }), + }, + { + name: 'daytona', + requiredEnvVars: ['DAYTONA_API_KEY'], + createCompute: () => daytona({ apiKey: process.env.DAYTONA_API_KEY! }), + sandboxOptions: { autoStopInterval: 15, autoDeleteInterval: 0 }, + }, + { + name: 'e2b', + requiredEnvVars: ['E2B_API_KEY'], + createCompute: () => e2b({ apiKey: process.env.E2B_API_KEY! }), + }, + { + name: 'hopx', + requiredEnvVars: ['HOPX_API_KEY'], + createCompute: () => hopx({ apiKey: process.env.HOPX_API_KEY! }), + }, + { + name: 'namespace', + requiredEnvVars: ['NSC_TOKEN'], + createCompute: () => namespace({ token: process.env.NSC_TOKEN! }), + sandboxOptions: { image: 'node:22' }, + }, + { + name: 'runloop', + requiredEnvVars: ['RUNLOOP_API_KEY'], + createCompute: () => runloop({ apiKey: process.env.RUNLOOP_API_KEY! }), + }, +]; diff --git a/src/linux-kernel-builder/scoring.ts b/src/linux-kernel-builder/scoring.ts new file mode 100644 index 0000000..4145000 --- /dev/null +++ b/src/linux-kernel-builder/scoring.ts @@ -0,0 +1,67 @@ +import type { LinuxKernelBuilderBenchmarkResult, LinuxKernelBuilderStats } from './types.js'; + +export interface LinuxKernelBuilderScoringWeights { + median: number; + p95: number; + p99: number; +} + +export const DEFAULT_LKB_WEIGHTS: LinuxKernelBuilderScoringWeights = { + median: 0.60, + p95: 0.25, + p99: 0.15, +}; + +/** + * Ceiling for build-time metric scoring. + * 30 minutes at or above scores 0. + */ +const BUILD_CEILING_MS = 30 * 60 * 1000; + +function scoreMetric(valueMs: number): number { + return Math.max(0, 100 * (1 - valueMs / BUILD_CEILING_MS)); +} + +export function computeLinuxKernelBuilderSuccessRate(result: LinuxKernelBuilderBenchmarkResult): number { + if (result.skipped || result.iterations.length === 0) return 0; + const successful = result.iterations.filter(i => !i.error).length; + return successful / result.iterations.length; +} + +function computeBuildScore( + stats: LinuxKernelBuilderStats, + weights: LinuxKernelBuilderScoringWeights = DEFAULT_LKB_WEIGHTS, +): number { + return ( + weights.median * scoreMetric(stats.median) + + weights.p95 * scoreMetric(stats.p95) + + weights.p99 * scoreMetric(stats.p99) + ); +} + +export function computeLinuxKernelBuilderCompositeScores( + results: LinuxKernelBuilderBenchmarkResult[], + weights: LinuxKernelBuilderScoringWeights = DEFAULT_LKB_WEIGHTS, +): void { + for (const result of results) { + const successRate = computeLinuxKernelBuilderSuccessRate(result); + result.successRate = successRate; + + if (result.skipped || successRate === 0) { + result.compositeScore = 0; + continue; + } + + const buildScore = computeBuildScore(result.summary.buildMs, weights); + result.compositeScore = Math.round(buildScore * successRate * 100) / 100; + } +} + +export function sortLinuxKernelBuilderByCompositeScore(results: LinuxKernelBuilderBenchmarkResult[]): LinuxKernelBuilderBenchmarkResult[] { + return [...results].sort((a, b) => { + if (a.skipped && !b.skipped) return 1; + if (!a.skipped && b.skipped) return -1; + if (a.skipped && b.skipped) return 0; + return (b.compositeScore ?? 0) - (a.compositeScore ?? 0); + }); +} diff --git a/src/linux-kernel-builder/table.ts b/src/linux-kernel-builder/table.ts new file mode 100644 index 0000000..61e931b --- /dev/null +++ b/src/linux-kernel-builder/table.ts @@ -0,0 +1,123 @@ +import type { LinuxKernelBuilderBenchmarkResult } from './types.js'; +import { sortLinuxKernelBuilderByCompositeScore } from './scoring.js'; +import { WORKLOAD, WORKLOAD_ACRONYM, WORKLOAD_LABEL } from './types.js'; + +function pad(str: string, width: number): string { + return str.padEnd(width); +} + +function formatSeconds(ms: number): string { + return (ms / 1000).toFixed(2); +} + +function round(n: number): number { + return Math.round(n * 100) / 100; +} + +function roundStats(s: { median: number; p95: number; p99: number }) { + return { median: round(s.median), p95: round(s.p95), p99: round(s.p99) }; +} + +export function printLinuxKernelBuilderResultsTable(results: LinuxKernelBuilderBenchmarkResult[]): void { + const header = [ + pad('Provider', 12), + pad('Score', 8), + pad('Build Med', 12), + pad('Build P95', 12), + pad('Build P99', 12), + pad('Total Med', 12), + pad('Status', 10), + ].join(' | '); + + const separator = [12, 8, 12, 12, 12, 12, 10] + .map(w => '-'.repeat(w)) + .join('-+-'); + + console.log('\n' + '='.repeat(separator.length)); + console.log(` COMPUTE PERFORMANCE RESULTS - ${WORKLOAD_ACRONYM} (${WORKLOAD_LABEL})`); + console.log('='.repeat(separator.length)); + console.log(header); + console.log(separator); + + const sorted = sortLinuxKernelBuilderByCompositeScore(results); + + for (const result of sorted) { + const successful = result.iterations.filter(i => !i.error).length; + const total = result.iterations.length; + + if (result.skipped) { + console.log([ + pad(result.provider, 12), + pad('--', 8), + pad('--', 12), + pad('--', 12), + pad('--', 12), + pad('--', 12), + pad('SKIPPED', 10), + ].join(' | ')); + continue; + } + + const score = result.compositeScore !== undefined ? result.compositeScore.toFixed(1) : '--'; + const allFailed = successful === 0; + + console.log([ + pad(result.provider, 12), + pad(score, 8), + pad(allFailed ? '--' : formatSeconds(result.summary.buildMs.median), 12), + pad(allFailed ? '--' : formatSeconds(result.summary.buildMs.p95), 12), + pad(allFailed ? '--' : formatSeconds(result.summary.buildMs.p99), 12), + pad(allFailed ? '--' : formatSeconds(result.summary.totalMs.median), 12), + pad(`${successful}/${total} OK`, 10), + ].join(' | ')); + } + + console.log('='.repeat(separator.length)); +} + +export async function writeLinuxKernelBuilderResultsJson(results: LinuxKernelBuilderBenchmarkResult[], outPath: string): Promise { + const fs = await import('fs'); + const os = await import('os'); + + const cleanResults = results.map(r => ({ + provider: r.provider, + mode: r.mode, + workload: r.workload, + workloadAcronym: r.workloadAcronym, + kernelVersion: r.kernelVersion, + iterations: r.iterations.map(i => ({ + totalMs: round(i.totalMs), + buildMs: round(i.buildMs), + ...(i.cpuCount !== undefined ? { cpuCount: i.cpuCount } : {}), + ...(i.memTotalKb !== undefined ? { memTotalKb: i.memTotalKb } : {}), + ...(i.error ? { error: i.error } : {}), + })), + summary: { + buildMs: roundStats(r.summary.buildMs), + totalMs: roundStats(r.summary.totalMs), + }, + ...(r.compositeScore !== undefined ? { compositeScore: round(r.compositeScore) } : {}), + ...(r.successRate !== undefined ? { successRate: round(r.successRate) } : {}), + ...(r.skipped ? { skipped: r.skipped, skipReason: r.skipReason } : {}), + })); + + const output = { + version: '1.0', + timestamp: new Date().toISOString(), + environment: { + node: process.version, + platform: os.platform(), + arch: os.arch(), + }, + config: { + iterations: results[0]?.iterations.length || 0, + mode: 'linux-kernel-builder', + workload: WORKLOAD, + workloadAcronym: WORKLOAD_ACRONYM, + }, + results: cleanResults, + }; + + fs.writeFileSync(outPath, JSON.stringify(output, null, 2)); + console.log(`Results written to ${outPath}`); +} diff --git a/src/linux-kernel-builder/types.ts b/src/linux-kernel-builder/types.ts new file mode 100644 index 0000000..ea863a3 --- /dev/null +++ b/src/linux-kernel-builder/types.ts @@ -0,0 +1,59 @@ +export interface LinuxKernelBuilderProviderConfig { + /** Provider name */ + name: string; + /** Number of iterations (default: 5) */ + iterations?: number; + /** Timeout per iteration in ms (default: 2700000) */ + timeout?: number; + /** Environment variables that must all be set to run this benchmark */ + requiredEnvVars: string[]; + /** Creates a compute instance */ + createCompute: () => any; + /** Options passed to sandbox.create() */ + sandboxOptions?: Record; + /** Timeout for sandbox.destroy() in ms (default: 15000) */ + destroyTimeoutMs?: number; +} + +export interface LinuxKernelBuilderTimingResult { + /** End-to-end workload time inside sandbox command execution */ + totalMs: number; + /** Linux kernel compile time captured in-sandbox */ + buildMs: number; + /** Number of CPUs reported by sandbox */ + cpuCount?: number; + /** Total memory in KB reported by sandbox */ + memTotalKb?: number; + /** Error message if this iteration failed */ + error?: string; +} + +export interface LinuxKernelBuilderStats { + median: number; + p95: number; + p99: number; +} + +export interface LinuxKernelBuilderBenchmarkResult { + provider: string; + mode: 'linux-kernel-builder'; + workload: 'linux-kernel-builder'; + workloadAcronym: 'LKB'; + kernelVersion: string; + iterations: LinuxKernelBuilderTimingResult[]; + summary: { + buildMs: LinuxKernelBuilderStats; + totalMs: LinuxKernelBuilderStats; + }; + /** Composite weighted score (0-100, higher = better). Computed post-benchmark. */ + compositeScore?: number; + /** Success rate as a fraction (0 to 1). Computed post-benchmark. */ + successRate?: number; + skipped?: boolean; + skipReason?: string; +} + +export const KERNEL_VERSION = '6.12.24'; +export const WORKLOAD = 'linux-kernel-builder'; +export const WORKLOAD_ACRONYM = 'LKB'; +export const WORKLOAD_LABEL = 'Linux Kernel Builder'; diff --git a/src/run.ts b/src/run.ts index 160b810..7ec94ee 100644 --- a/src/run.ts +++ b/src/run.ts @@ -7,16 +7,21 @@ import { runConcurrentBenchmark } from './sandbox/concurrent.js'; import { runStaggeredBenchmark } from './sandbox/staggered.js'; import { runStorageBenchmark, writeStorageResultsJson } from './storage/benchmark.js'; import { runBrowserBenchmark, writeBrowserResultsJson } from './browser/benchmark.js'; +import { runLinuxKernelBuilderBenchmark } from './linux-kernel-builder/benchmark.js'; import { printResultsTable, writeResultsJson } from './sandbox/table.js'; +import { printLinuxKernelBuilderResultsTable, writeLinuxKernelBuilderResultsJson } from './linux-kernel-builder/table.js'; import { providers } from './sandbox/providers.js'; import { storageProviders } from './storage/providers.js'; import { browserProviders } from './browser/providers.js'; +import { linuxKernelBuilderProviders } from './linux-kernel-builder/providers.js'; import { computeCompositeScores } from './sandbox/scoring.js'; import { computeStorageCompositeScores } from './storage/scoring.js'; import { computeBrowserCompositeScores } from './browser/scoring.js'; +import { computeLinuxKernelBuilderCompositeScores } from './linux-kernel-builder/scoring.js'; import type { BenchmarkResult, BenchmarkMode } from './sandbox/types.js'; import type { StorageBenchmarkResult } from './storage/types.js'; import type { BrowserBenchmarkResult } from './browser/types.js'; +import type { LinuxKernelBuilderBenchmarkResult } from './linux-kernel-builder/types.js'; // Load .env from the benchmarking root const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -37,26 +42,57 @@ function getArgValue(args: string[], flag: string): string | undefined { } /** Resolve which modes to run */ -function getModesToRun(): BenchmarkMode[] | ['storage'] | ['browser'] { +function getModesToRun(): BenchmarkMode[] | ['storage'] | ['browser'] | ['linux-kernel-builder'] { if (!rawMode) return ['sequential', 'staggered', 'burst']; if (rawMode === 'storage') return ['storage']; if (rawMode === 'browser') return ['browser']; + if (rawMode === 'linux-kernel-builder') return ['linux-kernel-builder']; const m = rawMode === 'concurrent' ? 'burst' : rawMode as BenchmarkMode; return [m]; } /** Map mode to results subdirectory name */ -function modeToDir(m: BenchmarkMode | 'storage'): string { +function modeToDir(m: BenchmarkMode | 'storage' | 'linux-kernel-builder'): string { switch (m) { case 'sequential': return 'sequential_tti'; case 'staggered': return 'staggered_tti'; case 'burst': case 'concurrent': return 'burst_tti'; case 'storage': return 'storage'; + case 'linux-kernel-builder': return 'linux_kernel_builder'; default: return `${m}_tti`; } } +async function runLinuxKernelBuilder(toRun: typeof linuxKernelBuilderProviders): Promise { + console.log('\n' + '='.repeat(70)); + console.log(' MODE: LINUX-KERNEL-BUILDER'); + console.log(` Iterations per provider: ${iterations}`); + console.log(' Workload: LKB (Linux Kernel Builder) - defconfig + bzImage'); + console.log('='.repeat(70)); + + const results: LinuxKernelBuilderBenchmarkResult[] = []; + + for (const providerConfig of toRun) { + const result = await runLinuxKernelBuilderBenchmark({ ...providerConfig, iterations }); + results.push(result); + } + + computeLinuxKernelBuilderCompositeScores(results); + printLinuxKernelBuilderResultsTable(results); + + const timestamp = new Date().toISOString().slice(0, 10); + const resultsDir = path.resolve(__dirname, `../results/${modeToDir('linux-kernel-builder')}`); + fs.mkdirSync(resultsDir, { recursive: true }); + + const outPath = path.join(resultsDir, `${timestamp}.json`); + await writeLinuxKernelBuilderResultsJson(results, outPath); + + const latestPath = path.join(resultsDir, 'latest.json'); + fs.copyFileSync(outPath, latestPath); + console.log(`Copied latest: ${latestPath}`); +} + async function runMode(mode: BenchmarkMode, toRun: typeof providers): Promise { console.log('\n' + '='.repeat(70)); console.log(` MODE: ${mode.toUpperCase()}`); @@ -244,6 +280,29 @@ async function main() { return; } + if (modes[0] === 'linux-kernel-builder') { + console.log('ComputeSDK Linux Kernel Builder Benchmarks'); + console.log(`Date: ${new Date().toISOString()}\n`); + + const toRun = providerFilter + ? linuxKernelBuilderProviders.filter(p => p.name === providerFilter) + : linuxKernelBuilderProviders; + + if (toRun.length === 0) { + if (providerFilter) { + console.error(`Unknown linux-kernel-builder provider: ${providerFilter}`); + console.error(`Available: ${linuxKernelBuilderProviders.map(p => p.name).join(', ')}`); + } else { + console.error('No linux-kernel-builder providers configured. Add entries to src/linux-kernel-builder/providers.ts.'); + } + process.exit(1); + } + + await runLinuxKernelBuilder(toRun); + console.log('\nAll linux-kernel-builder tests complete.'); + return; + } + // Handle storage mode separately if (modes[0] === 'storage') { console.log('ComputeSDK Storage Provider Benchmarks');