benchmark.yml

name: Benchmark

on:
  push:
    branches: [dev, stage, master]
  pull_request:
    branches: [master, stage, dev]

permissions:
  contents: read
  pull-requests: write

jobs:
  benchmark:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-go@v5
        with:
          go-version: "1.22"

      - name: Set up Python 3.13
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"

      - name: Install Python dependencies
        run: |
          pip install --upgrade pip
          pip install -e ".[dev,benchmark]"

      - name: Build Go Fiber benchmark server
        working-directory: benchmarks/fiber
        run: go mod tidy && go build -o fiberbench .

      - name: Enforce benchmark floors (ASGI + routing)
        run: python benchmarks/check_regressions.py

      - name: Export HTTP / ASGI / routing JSON for PR comment
        run: python benchmarks/export_pr_benchmarks.py

      - name: Fail if Fiber HTTP server did not start
        run: |
          if [[ -f fiber_warn.txt ]]; then
            echo "::error::Fiber benchmark server failed to start"
            cat fiber_warn.txt
            exit 1
          fi

      - name: Comment benchmark results on PR
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const bench = JSON.parse(fs.readFileSync('bench_results.json', 'utf8'));
            let asgi = {};
            try {
              asgi = JSON.parse(fs.readFileSync('asgi_micro.json', 'utf8'));
            } catch (e) { /* optional file */ }
            const routing = JSON.parse(fs.readFileSync('routing_results.json', 'utf8'));

            const baseline = {
              asgi_speedup: { health: 6.85, users_get: 8.73, users_post: 7.15 },
              routing_speedup: 7.6,
            };

            function delta(current, base) {
              if (base === undefined || base === 0) return '—';
              const pct = ((current - base) / base * 100).toFixed(1);
              if (pct > 2) return `+${pct}%`;
              if (pct < -5) return `${pct}%`;
              return `${pct}%`;
            }

            function pctChange(current, base) {
              if (base === undefined || base === 0 || current === undefined || current === null) return null;
              return ((current - base) / base) * 100;
            }

            function reqCell(n) {
              if (n === undefined || n === null) return '—';
              return `${Math.round(n).toLocaleString()}/s`;
            }

            function fiberCell(n) {
              if (n === undefined || n === null) return '—';
              return `**${Math.round(n).toLocaleString()}/s**`;
            }

            const rows = ['health', 'users_get', 'users_post'];
            const labels = { health: 'GET /health', users_get: 'GET /users/{id}', users_post: 'POST /users' };

            let httpTable = '| Endpoint | FasterAPI | FastAPI | Fiber (Go) | F / Fast |\n|---|---|---|---|---|\n';
            for (const k of rows) {
              const b = bench[k];
              const sp = b.speedup !== undefined ? `${b.speedup.toFixed(2)}x` : '—';
              httpTable += `| \`${labels[k]}\` | **${reqCell(b.fasterapi)}** | ${reqCell(b.fastapi)} | ${fiberCell(b.fiber)} | **${sp}** |\n`;
            }

            let asgiBlock = '';
            if (asgi.health) {
              asgiBlock = `
            #### Direct ASGI (no HTTP; ${(50000).toLocaleString()} iterations)

            | Endpoint | FasterAPI | FastAPI | Speedup | vs README ASGI ratio |
            |---|---|---|---|---|
            | \`GET /health\` | **${Math.round(asgi.health.fasterapi).toLocaleString()}/s** | ${Math.round(asgi.health.fastapi).toLocaleString()}/s | **${asgi.health.speedup.toFixed(2)}x** | ${delta(asgi.health.speedup, baseline.asgi_speedup.health)} |
            | \`GET /users/{id}\` | **${Math.round(asgi.users_get.fasterapi).toLocaleString()}/s** | ${Math.round(asgi.users_get.fastapi).toLocaleString()}/s | **${asgi.users_get.speedup.toFixed(2)}x** | ${delta(asgi.users_get.speedup, baseline.asgi_speedup.users_get)} |
            | \`POST /users\` | **${Math.round(asgi.users_post.fasterapi).toLocaleString()}/s** | ${Math.round(asgi.users_post.fastapi).toLocaleString()}/s | **${asgi.users_post.speedup.toFixed(2)}x** | ${delta(asgi.users_post.speedup, baseline.asgi_speedup.users_post)} |
            `;
            }

            const checks = [
              { name: 'ASGI GET /health speedup', change: pctChange(asgi?.health?.speedup, baseline.asgi_speedup.health) },
              { name: 'ASGI GET /users/{id} speedup', change: pctChange(asgi?.users_get?.speedup, baseline.asgi_speedup.users_get) },
              { name: 'ASGI POST /users speedup', change: pctChange(asgi?.users_post?.speedup, baseline.asgi_speedup.users_post) },
              { name: 'Routing radix speedup', change: pctChange(routing?.speedup, baseline.routing_speedup) },
            ];

            const regressions = checks.filter(c => c.change !== null && c.change <= -5);
            const improvements = checks.filter(c => c.change !== null && c.change >= 2);

            let statusLine = '⚪ Benchmark status: stable (within tolerance).';
            if (regressions.length > 0) {
              statusLine = '🔴 Benchmark status: regression detected (>= 5% slower). **Do not merge this PR until fixed.**';
            } else if (improvements.length > 0) {
              statusLine = '🟢 Benchmark status: improvement detected.';
            }

            const regressionList = regressions.length
              ? `\n\n### Regression details\n${regressions
                  .map(r => `- ${r.name}: ${r.change.toFixed(1)}% vs README baseline`)
                  .join('\n')}`
              : '';

            const body = `## Benchmark results

            > Ubuntu runner, Python 3.13. **HTTP table** uses the same httpx load against uvicorn (Python) and Fiber (Go). **Direct ASGI** (below) is Python-only and excludes network I/O.

            ${statusLine}${regressionList}

            ### HTTP throughput (FasterAPI vs FastAPI vs Fiber)

            ${httpTable}

            ${asgiBlock}

            ### Routing (radix vs regex, ${(500000 * 3).toLocaleString()} lookups)

            | Router | Ops/s | Speedup | vs README |
            |---|---|---|---|
            | **Radix** | **${Math.round(routing.radix).toLocaleString()}** | **${routing.speedup.toFixed(1)}x** | ${delta(routing.speedup, baseline.routing_speedup)} |
            | Regex | ${Math.round(routing.regex).toLocaleString()} | 1.0x | |

            <details>
            <summary>How to read this</summary>

            - **F / Fast** = FasterAPI req/s ÷ FastAPI req/s on the same HTTP harness (higher is better).
            - **Fiber** uses the Go app in \`benchmarks/fiber\` (same routes). Go is often several times faster than Python here; the important guard for regressions is **\`check_regressions.py\`** (ASGI + routing floors), which must pass in this workflow.
            - **vs README** compares combined speedups to documented reference numbers (local machine); CI absolute req/s differs by hardware.

            </details>`;

            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });
            const existing = comments.find(c => c.body.includes('## Benchmark results'));

            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existing.id,
                body: body,
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body: body,
              });
            }

            if (regressions.length > 0) {
              core.setFailed(`Benchmark regression >=5% detected in ${regressions.length} metric(s).`);
            }