Add Phase 1 benchmark runner and CI gates

.github/workflows/ci.yml

@@ -32,14 +32,17 @@ jobs:
           pip install -e ".[dev]"
 
       - name: Lint
-        run: python -m ruff check src tests
+        run: python -m ruff check src tests scripts
 
       - name: Type check
         run: python -m mypy src --ignore-missing-imports
 
       - name: Run tests
         run: python -m pytest -q --timeout=60
 
+      - name: Eval thresholds gate
+        run: python scripts/eval.py --skip-pytest --thresholds benchmarks/thresholds/golden_minimum.json --output-dir data/eval-ci
+
   web-build:
     name: Web UI build
     runs-on: ubuntu-latest
@@ -79,7 +82,6 @@ jobs:
   docker:
     runs-on: ubuntu-latest
     needs: [lint-and-test, web-build]
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     steps:
       - uses: actions/checkout@v4
 

README.md

@@ -202,4 +202,4 @@ MIT. See `LICENSE`.
 
 ## Not Financial Advice
 
-Jetbot produces structured extraction and analytical signals. It does not provide investment advice or recommend trades.
+Jetbot produces structured extraction and analytical signals. It does not provide investment advice or recommend trades.

benchmarks/README.md

@@ -24,4 +24,19 @@ Run the current golden evaluation gate with:
 python scripts/eval.py --thresholds benchmarks/thresholds/golden_minimum.json
 ```
 
-Real PDF benchmark manifests should point to local-only files through relative paths such as `raw/company-2025-10k.pdf`. Those raw files are intentionally ignored by git.
+Run the sample local benchmark manifest with:
+
+```bash
+python scripts/eval.py --benchmark-manifest benchmarks/sample_manifest.json --skip-pytest --output-dir data/eval-dev
+```
+
+Run the first local private filing batches after placing raw PDFs under ignored paths:
+
+```bash
+python scripts/eval.py --benchmark-manifest benchmarks/local_manifests/us_filings_private_batch.json --skip-pytest --output-dir data/eval-us-private
+python scripts/eval.py --benchmark-manifest benchmarks/local_manifests/stress_private_batch.json --skip-pytest --output-dir data/eval-stress-private
+```
+
+Real PDF benchmark manifests should point to local-only files through relative paths such as `raw/company-2025-10k.pdf`. Those raw files are intentionally ignored by git.
+The current eval runner supports committed `synthetic` fixtures and local `pdf` sources from a manifest.
+The committed manifests under `benchmarks/local_manifests/` are safe to commit because they only store anonymized metadata and expected facts; the raw PDFs still live under ignored paths such as `benchmarks/private/us_filings/` and `benchmarks/private/stress/`.

benchmarks/fixtures/synthetic_income_001.json

@@ -0,0 +1,16 @@
+{
+  "pages": [
+    {
+      "page_number": 1,
+      "text": "Consolidated Balance Sheet\nCash and equivalents 50\nTotal Assets 90\nTotal Liabilities 40\nTotal Equity 50\n"
+    },
+    {
+      "page_number": 2,
+      "text": "Consolidated Income Statement\nRevenue 100\nCost of goods sold 60\nGross Profit 40\nOperating expenses 20\nOperating income 20\nNet Income 10\n"
+    },
+    {
+      "page_number": 3,
+      "text": "Notes to Financial Statements\nAccounting Policy: Revenue is recognized over time.\n"
+    }
+  ]
+}

benchmarks/local_manifests/stress_private_batch.json

@@ -0,0 +1,83 @@
+{
+  "schema_version": 1,
+  "benchmark_id": "local-private-stress-v1",
+  "name": "Local Private Stress Batch",
+  "description": "Local-only stress cases for scanned pages, rotated layouts, and complex tables. Raw PDFs remain under ignored paths and are not committed.",
+  "data_policy": {
+    "raw_files_committed": false,
+    "label_policy": "anonymized",
+    "notes": "Place local stress PDFs under benchmarks/private/stress/ before running. Keep detailed operator notes outside git when they contain vendor-specific annotations."
+  },
+  "cases": [
+    {
+      "case_id": "stress-scanned-001",
+      "company": "Stress Scan Co A",
+      "ticker": "SSC",
+      "filing_type": "10-K",
+      "period_end": "2025-12-31",
+      "source": {
+        "type": "pdf",
+        "path": "../private/stress/stress-scanned-001.pdf",
+        "license": "local-private"
+      },
+      "expected_facts": [
+        {"statement_type": "income", "concept": "revenue", "value": 420.0, "currency": "USD", "evidence": [{"page": 18}]},
+        {"statement_type": "income", "concept": "net_income", "value": 22.0, "currency": "USD", "evidence": [{"page": 18}]},
+        {"statement_type": "balance", "concept": "total_assets", "value": 870.0, "currency": "USD", "evidence": [{"page": 15}]},
+        {"statement_type": "balance", "concept": "cash_and_equivalents", "value": 55.0, "currency": "USD", "evidence": [{"page": 15}]},
+        {"statement_type": "cashflow", "concept": "operating_cash_flow", "value": 38.0, "currency": "USD", "evidence": [{"page": 19}]},
+        {"statement_type": "cashflow", "concept": "capex", "value": -27.0, "currency": "USD", "evidence": [{"page": 19}]}
+      ],
+      "expected_notes": ["audit_opinion"],
+      "expected_risk_categories": ["cash_vs_profit"]
+    },
+    {
+      "case_id": "stress-rotated-001",
+      "company": "Stress Rotation Co B",
+      "ticker": "SRB",
+      "filing_type": "10-Q",
+      "period_end": "2026-03-31",
+      "source": {
+        "type": "pdf",
+        "path": "../private/stress/stress-rotated-001.pdf",
+        "license": "local-private"
+      },
+      "expected_facts": [
+        {"statement_type": "income", "concept": "revenue", "value": 310.0, "currency": "USD", "evidence": [{"page": 7}]},
+        {"statement_type": "income", "concept": "gross_profit", "value": 96.0, "currency": "USD", "evidence": [{"page": 7}]},
+        {"statement_type": "income", "concept": "operating_income", "value": 34.0, "currency": "USD", "evidence": [{"page": 7}]},
+        {"statement_type": "balance", "concept": "total_assets", "value": 640.0, "currency": "USD", "evidence": [{"page": 5}]},
+        {"statement_type": "balance", "concept": "total_liabilities", "value": 350.0, "currency": "USD", "evidence": [{"page": 5}]},
+        {"statement_type": "balance", "concept": "total_equity", "value": 290.0, "currency": "USD", "evidence": [{"page": 5}]},
+        {"statement_type": "cashflow", "concept": "operating_cash_flow", "value": 29.0, "currency": "USD", "evidence": [{"page": 8}]}
+      ],
+      "expected_notes": ["other"],
+      "expected_risk_categories": []
+    },
+    {
+      "case_id": "stress-complex-table-001",
+      "company": "Stress Layout Co C",
+      "ticker": "SLC",
+      "filing_type": "10-K",
+      "period_end": "2025-12-31",
+      "source": {
+        "type": "pdf",
+        "path": "../private/stress/stress-complex-table-001.pdf",
+        "license": "local-private"
+      },
+      "expected_facts": [
+        {"statement_type": "income", "concept": "revenue", "value": 760.0, "currency": "USD", "evidence": [{"page": 28}]},
+        {"statement_type": "income", "concept": "gross_profit", "value": 250.0, "currency": "USD", "evidence": [{"page": 28}]},
+        {"statement_type": "income", "concept": "net_income", "value": 84.0, "currency": "USD", "evidence": [{"page": 28}]},
+        {"statement_type": "balance", "concept": "total_assets", "value": 1510.0, "currency": "USD", "evidence": [{"page": 25}]},
+        {"statement_type": "balance", "concept": "total_liabilities", "value": 690.0, "currency": "USD", "evidence": [{"page": 25}]},
+        {"statement_type": "balance", "concept": "total_equity", "value": 820.0, "currency": "USD", "evidence": [{"page": 25}]},
+        {"statement_type": "balance", "concept": "cash_and_equivalents", "value": 118.0, "currency": "USD", "evidence": [{"page": 25}]},
+        {"statement_type": "cashflow", "concept": "operating_cash_flow", "value": 132.0, "currency": "USD", "evidence": [{"page": 29}]},
+        {"statement_type": "cashflow", "concept": "capex", "value": -44.0, "currency": "USD", "evidence": [{"page": 29}]}
+      ],
+      "expected_notes": ["segment", "related_party"],
+      "expected_risk_categories": ["disclosure_inconsistency"]
+    }
+  ]
+}

benchmarks/local_manifests/us_filings_private_batch.json

@@ -0,0 +1,117 @@
+{
+  "schema_version": 1,
+  "benchmark_id": "local-private-us-filings-v1",
+  "name": "Local Private US Filing Batch",
+  "description": "First local-only batch covering anonymized 10-K and 10-Q filings. Raw PDFs stay under ignored paths while manifests and anonymized expected facts remain committed.",
+  "data_policy": {
+    "raw_files_committed": false,
+    "label_policy": "anonymized",
+    "notes": "Copy private PDFs into benchmarks/private/us_filings/ before running. Keep any non-anonymized analyst labels outside git."
+  },
+  "cases": [
+    {
+      "case_id": "us10k-tech-001",
+      "company": "US Tech Co A",
+      "ticker": "UTA",
+      "filing_type": "10-K",
+      "period_end": "2025-12-31",
+      "source": {
+        "type": "pdf",
+        "path": "../private/us_filings/us10k-tech-001.pdf",
+        "license": "local-private"
+      },
+      "expected_facts": [
+        {"statement_type": "income", "concept": "revenue", "value": 1980.0, "currency": "USD", "evidence": [{"page": 34}]},
+        {"statement_type": "income", "concept": "gross_profit", "value": 820.0, "currency": "USD", "evidence": [{"page": 34}]},
+        {"statement_type": "income", "concept": "operating_income", "value": 330.0, "currency": "USD", "evidence": [{"page": 35}]},
+        {"statement_type": "income", "concept": "net_income", "value": 240.0, "currency": "USD", "evidence": [{"page": 35}]},
+        {"statement_type": "balance", "concept": "total_assets", "value": 2250.0, "currency": "USD", "evidence": [{"page": 32}]},
+        {"statement_type": "balance", "concept": "total_liabilities", "value": 910.0, "currency": "USD", "evidence": [{"page": 32}]},
+        {"statement_type": "balance", "concept": "total_equity", "value": 1340.0, "currency": "USD", "evidence": [{"page": 32}]},
+        {"statement_type": "balance", "concept": "cash_and_equivalents", "value": 280.0, "currency": "USD", "evidence": [{"page": 32}]},
+        {"statement_type": "cashflow", "concept": "operating_cash_flow", "value": 410.0, "currency": "USD", "evidence": [{"page": 36}]},
+        {"statement_type": "cashflow", "concept": "capex", "value": -120.0, "currency": "USD", "evidence": [{"page": 36}]}
+      ],
+      "expected_notes": ["accounting_policy", "segment"],
+      "expected_risk_categories": []
+    },
+    {
+      "case_id": "us10q-industrial-001",
+      "company": "US Industrial Co B",
+      "ticker": "UIB",
+      "filing_type": "10-Q",
+      "period_end": "2026-03-31",
+      "source": {
+        "type": "pdf",
+        "path": "../private/us_filings/us10q-industrial-001.pdf",
+        "license": "local-private"
+      },
+      "expected_facts": [
+        {"statement_type": "income", "concept": "revenue", "value": 620.0, "currency": "USD", "evidence": [{"page": 8}]},
+        {"statement_type": "income", "concept": "gross_profit", "value": 180.0, "currency": "USD", "evidence": [{"page": 8}]},
+        {"statement_type": "income", "concept": "operating_income", "value": 72.0, "currency": "USD", "evidence": [{"page": 8}]},
+        {"statement_type": "income", "concept": "net_income", "value": 41.0, "currency": "USD", "evidence": [{"page": 8}]},
+        {"statement_type": "balance", "concept": "total_assets", "value": 1430.0, "currency": "USD", "evidence": [{"page": 6}]},
+        {"statement_type": "balance", "concept": "total_liabilities", "value": 790.0, "currency": "USD", "evidence": [{"page": 6}]},
+        {"statement_type": "balance", "concept": "total_equity", "value": 640.0, "currency": "USD", "evidence": [{"page": 6}]},
+        {"statement_type": "balance", "concept": "cash_and_equivalents", "value": 92.0, "currency": "USD", "evidence": [{"page": 6}]},
+        {"statement_type": "cashflow", "concept": "operating_cash_flow", "value": 88.0, "currency": "USD", "evidence": [{"page": 10}]},
+        {"statement_type": "cashflow", "concept": "capex", "value": -24.0, "currency": "USD", "evidence": [{"page": 10}]}
+      ],
+      "expected_notes": ["other"],
+      "expected_risk_categories": []
+    },
+    {
+      "case_id": "us10k-healthcare-001",
+      "company": "US Healthcare Co C",
+      "ticker": "UHC",
+      "filing_type": "10-K",
+      "period_end": "2025-12-31",
+      "source": {
+        "type": "pdf",
+        "path": "../private/us_filings/us10k-healthcare-001.pdf",
+        "license": "local-private"
+      },
+      "expected_facts": [
+        {"statement_type": "income", "concept": "revenue", "value": 1140.0, "currency": "USD", "evidence": [{"page": 41}]},
+        {"statement_type": "income", "concept": "gross_profit", "value": 530.0, "currency": "USD", "evidence": [{"page": 41}]},
+        {"statement_type": "income", "concept": "operating_income", "value": 148.0, "currency": "USD", "evidence": [{"page": 41}]},
+        {"statement_type": "income", "concept": "net_income", "value": 101.0, "currency": "USD", "evidence": [{"page": 41}]},
+        {"statement_type": "balance", "concept": "total_assets", "value": 2680.0, "currency": "USD", "evidence": [{"page": 39}]},
+        {"statement_type": "balance", "concept": "total_liabilities", "value": 1180.0, "currency": "USD", "evidence": [{"page": 39}]},
+        {"statement_type": "balance", "concept": "total_equity", "value": 1500.0, "currency": "USD", "evidence": [{"page": 39}]},
+        {"statement_type": "balance", "concept": "cash_and_equivalents", "value": 310.0, "currency": "USD", "evidence": [{"page": 39}]},
+        {"statement_type": "cashflow", "concept": "operating_cash_flow", "value": 205.0, "currency": "USD", "evidence": [{"page": 43}]},
+        {"statement_type": "cashflow", "concept": "capex", "value": -58.0, "currency": "USD", "evidence": [{"page": 43}]}
+      ],
+      "expected_notes": ["accounting_policy", "contingency"],
+      "expected_risk_categories": ["audit_governance"]
+    },
+    {
+      "case_id": "us10q-consumer-001",
+      "company": "US Consumer Co D",
+      "ticker": "UCD",
+      "filing_type": "10-Q",
+      "period_end": "2026-06-30",
+      "source": {
+        "type": "pdf",
+        "path": "../private/us_filings/us10q-consumer-001.pdf",
+        "license": "local-private"
+      },
+      "expected_facts": [
+        {"statement_type": "income", "concept": "revenue", "value": 540.0, "currency": "USD", "evidence": [{"page": 11}]},
+        {"statement_type": "income", "concept": "gross_profit", "value": 210.0, "currency": "USD", "evidence": [{"page": 11}]},
+        {"statement_type": "income", "concept": "operating_income", "value": 64.0, "currency": "USD", "evidence": [{"page": 11}]},
+        {"statement_type": "income", "concept": "net_income", "value": 39.0, "currency": "USD", "evidence": [{"page": 11}]},
+        {"statement_type": "balance", "concept": "total_assets", "value": 980.0, "currency": "USD", "evidence": [{"page": 9}]},
+        {"statement_type": "balance", "concept": "total_liabilities", "value": 470.0, "currency": "USD", "evidence": [{"page": 9}]},
+        {"statement_type": "balance", "concept": "total_equity", "value": 510.0, "currency": "USD", "evidence": [{"page": 9}]},
+        {"statement_type": "balance", "concept": "cash_and_equivalents", "value": 76.0, "currency": "USD", "evidence": [{"page": 9}]},
+        {"statement_type": "cashflow", "concept": "operating_cash_flow", "value": 67.0, "currency": "USD", "evidence": [{"page": 13}]},
+        {"statement_type": "cashflow", "concept": "capex", "value": -19.0, "currency": "USD", "evidence": [{"page": 13}]}
+      ],
+      "expected_notes": ["other"],
+      "expected_risk_categories": []
+    }
+  ]
+}

benchmarks/sample_manifest.json

@@ -17,10 +17,46 @@
       "period_end": "2025-12-31",
       "source": {
         "type": "synthetic",
-        "path": "tests/golden/conftest.py",
+        "path": "fixtures/synthetic_income_001.json",
         "license": "synthetic"
       },
       "expected_facts": [
+        {
+          "statement_type": "balance",
+          "concept": "total_assets",
+          "label": "Total Assets",
+          "value": 90.0,
+          "unit": "USD millions",
+          "currency": "USD",
+          "period_end": "2025-12-31",
+          "evidence": [
+            {"page": 1, "quote": "Total Assets 90"}
+          ]
+        },
+        {
+          "statement_type": "balance",
+          "concept": "total_liabilities",
+          "label": "Total Liabilities",
+          "value": 40.0,
+          "unit": "USD millions",
+          "currency": "USD",
+          "period_end": "2025-12-31",
+          "evidence": [
+            {"page": 1, "quote": "Total Liabilities 40"}
+          ]
+        },
+        {
+          "statement_type": "balance",
+          "concept": "total_equity",
+          "label": "Total Equity",
+          "value": 50.0,
+          "unit": "USD millions",
+          "currency": "USD",
+          "period_end": "2025-12-31",
+          "evidence": [
+            {"page": 1, "quote": "Total Equity 50"}
+          ]
+        },
         {
           "statement_type": "income",
           "concept": "revenue",
@@ -32,9 +68,45 @@
           "evidence": [
             {"page": 1, "quote": "Revenue 100"}
           ]
+        },
+        {
+          "statement_type": "income",
+          "concept": "gross_profit",
+          "label": "Gross Profit",
+          "value": 40.0,
+          "unit": "USD millions",
+          "currency": "USD",
+          "period_end": "2025-12-31",
+          "evidence": [
+            {"page": 2, "quote": "Gross Profit 40"}
+          ]
+        },
+        {
+          "statement_type": "income",
+          "concept": "operating_income",
+          "label": "Operating income",
+          "value": 20.0,
+          "unit": "USD millions",
+          "currency": "USD",
+          "period_end": "2025-12-31",
+          "evidence": [
+            {"page": 2, "quote": "Operating income 20"}
+          ]
+        },
+        {
+          "statement_type": "income",
+          "concept": "net_income",
+          "label": "Net Income",
+          "value": 10.0,
+          "unit": "USD millions",
+          "currency": "USD",
+          "period_end": "2025-12-31",
+          "evidence": [
+            {"page": 2, "quote": "Net Income 10"}
+          ]
         }
       ],
-      "expected_notes": ["other"],
+      "expected_notes": ["accounting_policy"],
       "expected_risk_categories": []
     }
   ]