From ac4f5ea87ed9c8865928e947e732272e7e877a7b Mon Sep 17 00:00:00 2001 From: Aviad Date: Fri, 10 Jul 2026 23:34:15 +0300 Subject: [PATCH] feat: jwt-authentication claim + docs-vs-implementation detector (v0.3.0) Second verification slice: - New built-in claim jwt-authentication ("Authentication uses JWT access tokens"), powered by the v0.0.6 JWT purpose classifier. - New contradiction detector: documentation claims JWT while the only JWT usage found is invitation/verification tokens -> CONTRADICTED with both sides. Documentation with no usage at all stays WEAK, never contradicted: absence is not positive conflicting evidence. - Evaluator registry replaces the single hardcoded dispatch, so adding claims is adding definitions. - fix: signal metadata was hardcoded to '{}' at persistence, silently dropping extractor metadata (including JWT purpose) since V0. Concept detection never noticed (in-process); verification is the first DB reader of metadata and exposed it. Regression test added. 126 tests (6 new). No breaking changes. Version 0.3.0. --- LIMITATIONS.md | 6 +- QUICKSTART.md | 4 +- README.md | 4 +- RELEASE_NOTES_v0.3.0.md | 50 +++++++ VERIFICATION.md | 18 +-- pyproject.toml | 4 +- server.json | 4 +- src/devtime/__init__.py | 2 +- src/devtime/intelligence/verification.py | 141 ++++++++++++++++++- src/devtime/scanner/signals.py | 6 +- tests/integration/test_evidence_precision.py | 8 +- tests/integration/test_verification.py | 82 +++++++++++ 12 files changed, 303 insertions(+), 26 deletions(-) create mode 100644 RELEASE_NOTES_v0.3.0.md diff --git a/LIMITATIONS.md b/LIMITATIONS.md index 2f68cde..9eefb2c 100644 --- a/LIMITATIONS.md +++ b/LIMITATIONS.md @@ -70,10 +70,10 @@ do. Read this before trusting any single output. get_context_pack). No write tools, no network listener, no source code returned. The rest of the planned tool surface is not implemented yet. -## 6b. Verification limitations (v0.2, experimental) +## 6b. Verification limitations (experimental) -- One built-in claim (billing-webhook-signature). User-defined claims are not - supported yet, deliberately. +- Two built-in claims (billing-webhook-signature, jwt-authentication). + User-defined claims are not supported yet, deliberately. - Verification is rule-driven over scanner signals; it inherits every scanner coverage limitation listed here. - Statuses mean "per DevTime's evidence rules", not formal proof or a security diff --git a/QUICKSTART.md b/QUICKSTART.md index 66fe9bb..9889ef5 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -27,7 +27,7 @@ python -m venv .venv source .venv/bin/activate # Windows (PowerShell): .venv\Scripts\Activate.ps1 # Windows (Git Bash): source .venv/Scripts/activate pip install -e ".[dev]" -pytest # optional: all tests pass (120 at v0.2.0) +pytest # optional: all tests pass (126 at v0.3.0) ``` ## 3. Create the demo repo @@ -132,7 +132,7 @@ A fresh-clone check was run on the current candidate: - **OS:** Windows 11 (Git Bash) - **Python:** 3.11.9 - **Install:** `pip install -e ".[dev]"` -- **Tests:** all passing (120 at v0.2.0) +- **Tests:** all passing (126 at v0.3.0) - **Demo:** `dtc init` / `dtc scan` / `dtc concepts` / `dtc explain "Billing Webhooks"` all produced the expected output from a clean `git clone`. diff --git a/README.md b/README.md index 89b8540..5a870f9 100644 --- a/README.md +++ b/README.md @@ -278,8 +278,8 @@ Contradictions: ``` Statuses are SUPPORTED, WEAK, CONTRADICTED, or UNKNOWN; contradictions always -show both sides; changed evidence marks a claim STALE. One built-in claim ships -in v0.2. See **[VERIFICATION.md](VERIFICATION.md)**. +show both sides; changed evidence marks a claim STALE. Two built-in claims ship +(billing webhook signatures, JWT authentication). See **[VERIFICATION.md](VERIFICATION.md)**. ## Example output diff --git a/RELEASE_NOTES_v0.3.0.md b/RELEASE_NOTES_v0.3.0.md new file mode 100644 index 0000000..9ed896b --- /dev/null +++ b/RELEASE_NOTES_v0.3.0.md @@ -0,0 +1,50 @@ +# DevTime v0.3.0 - JWT claim and the docs-vs-implementation detector + +Second step of the verification layer. Two changes and one honest bug fix. No +cloud, no telemetry, no AI, no code execution - unchanged. + +## New built-in claim: jwt-authentication + +```bash +dtc verify jwt-authentication +``` + +"Authentication uses JWT access tokens." Uses the JWT purpose classifier +(introduced in v0.0.6) to distinguish access tokens from invitation and +verification tokens. + +## New: documentation-vs-implementation contradiction + +The classic case: documentation says JWT, the code says otherwise. + +``` +Status: CONTRADICTED + claimed: docs/auth.md references JWT in a documentation or decision + context, implying JWT-based authentication. + observed: src/tokens/invite.ts uses JWT for invitation/verification tokens. + Invitation tokens are not access-token authentication, and no + access-token usage was found. +``` + +Honesty rule preserved: documentation with NO usage found at all is WEAK +(missing evidence), never CONTRADICTED - absence is not positive conflicting +evidence. + +## Fixed: signal metadata was silently dropped at persistence + +A latent V0 bug: every signal's metadata was hardcoded to `{}` when written to +the local database. Concept detection never noticed (it runs in-process on live +objects); the verification engine is the first feature that reads metadata back +from the database, and it exposed the bug. Metadata (including the JWT purpose +classification) now persists, with a regression test. + +## Notes + +- Two built-in claims now: billing-webhook-signature, jwt-authentication. +- Still deliberately no user-defined claim files, and no LLM in the truth path. +- 126 passing tests (6 new). +- No breaking changes. + +## Names + +- PyPI distribution: `devtime-ei`. Python import: `devtime`. CLI: `dtc`. diff --git a/VERIFICATION.md b/VERIFICATION.md index 80222e5..f13639f 100644 --- a/VERIFICATION.md +++ b/VERIFICATION.md @@ -1,4 +1,4 @@ -# Claim verification (experimental, v0.2) +# Claim verification (experimental) DevTime is evolving into the verification layer for repository understanding: the layer developers and coding agents use to know whether a statement about a @@ -55,13 +55,15 @@ same time: the last verification supported it, but its evidence changed since. Freshness only tracks files that were evidence for the claim. Unrelated changes never mark a claim stale. -## Built-in claims (v0.2) - -One claim ships in this release, chosen because it is the best-covered domain -in the scanner and it has a real-world contradiction story: +## Built-in claims - **billing-webhook-signature** - "Incoming billing webhooks verify the payment provider's signature." +- **jwt-authentication** (v0.3) - "Authentication uses JWT access tokens." + Includes the documentation-vs-implementation detector: documentation claiming + JWT while the only JWT usage found is invitation/verification tokens is a + both-sided contradiction. Documentation with no usage at all is WEAK, not + contradicted: absence is not positive conflicting evidence. Example contradiction, found in the wild: Cal.com's community edition ships `pages/api/stripe/webhook.ts` as a handler that always returns 404 ("not @@ -84,7 +86,7 @@ sides and exact paths. - Heuristic scanner: evidence comes from static patterns, not execution. - Signature verification is recognized for known provider patterns (e.g. Stripe `constructEvent`); custom schemes may not be detected. -- One built-in claim in v0.2. User-defined claims are deliberately not +- Two built-in claims. User-defined claims are deliberately not supported yet: the claim model must earn trust before it grows a configuration language. - Coverage follows scanner language support; see [LIMITATIONS.md](LIMITATIONS.md). @@ -92,6 +94,6 @@ sides and exact paths. ## Where this is going Next candidates, in order: more built-in claims over well-covered domains -(webhook idempotency, JWT authentication), a documentation-vs-implementation -contradiction detector, and diff-aware claim staleness in `dtc risk`. User-defined +(webhook idempotency), more contradiction detectors, and diff-aware claim +staleness in `dtc risk`. User-defined claims come after built-in claims prove trustworthy on real repositories. diff --git a/pyproject.toml b/pyproject.toml index bcd9422..3b097de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "devtime-ei" -version = "0.2.0" +version = "0.3.0" description = "Local-first Engineering Intelligence for software repositories" readme = "README.md" requires-python = ">=3.11" @@ -50,7 +50,7 @@ dev = [ Homepage = "https://github.com/Shakargy/devtime" Repository = "https://github.com/Shakargy/devtime" Issues = "https://github.com/Shakargy/devtime/issues" -"Release Notes" = "https://github.com/Shakargy/devtime/releases/tag/v0.2.0" +"Release Notes" = "https://github.com/Shakargy/devtime/releases/tag/v0.3.0" Demo = "https://youtu.be/1Hiu3Y9J_SI" [project.scripts] diff --git a/server.json b/server.json index f11b5cf..2343df3 100644 --- a/server.json +++ b/server.json @@ -8,12 +8,12 @@ "source": "github" }, "websiteUrl": "https://github.com/Shakargy/devtime", - "version": "0.2.0", + "version": "0.3.0", "packages": [ { "registryType": "pypi", "identifier": "devtime-ei", - "version": "0.2.0", + "version": "0.3.0", "transport": { "type": "stdio" } diff --git a/src/devtime/__init__.py b/src/devtime/__init__.py index f94e50f..eee5005 100644 --- a/src/devtime/__init__.py +++ b/src/devtime/__init__.py @@ -1,6 +1,6 @@ """DevTime - local-first Engineering Intelligence for repository memory.""" -__version__ = "0.2.0" +__version__ = "0.3.0" # Version metadata (Builder Edition, Chapter 20). EVIDENCE_MODEL = "2026.06.1" diff --git a/src/devtime/intelligence/verification.py b/src/devtime/intelligence/verification.py index 8b03b7e..fae5473 100644 --- a/src/devtime/intelligence/verification.py +++ b/src/devtime/intelligence/verification.py @@ -149,8 +149,17 @@ class ClaimDefinition: statement="Incoming billing webhooks verify the payment provider's signature.", category="billing", ), + "jwt-authentication": ClaimDefinition( + slug="jwt-authentication", + name="JWT Authentication", + statement="Authentication uses JWT access tokens.", + category="authentication", + ), } +# JWT library names for dependency evidence. +_JWT_DEP_TOKENS = ("jsonwebtoken", "pyjwt", "jose", "njwt", "jwt-decode", "fast-jwt") + _LIMITATIONS = [ "Heuristic scanner: evidence comes from static patterns, not execution.", "Signature verification is recognized for known provider patterns " @@ -225,7 +234,8 @@ def verify_claim(conn: sqlite3.Connection, slug: str) -> VerificationResult: ) rows = _load_signals(conn, scan_id) - return _verify_billing_webhook_signature(definition, rows, scan_id) + evaluator = _EVALUATORS[slug] + return evaluator(definition, rows, scan_id) def _verify_billing_webhook_signature( @@ -348,6 +358,135 @@ def _verify_billing_webhook_signature( ) +def _verify_jwt_authentication( + definition: ClaimDefinition, rows: list[sqlite3.Row], scan_id: str +) -> VerificationResult: + """Verify JWT access-token authentication, with the documentation-vs- + implementation contradiction detector. + + The purpose classifier (Trust Repair v0.0.6) distinguishes access tokens + from invitation/verification tokens. Documentation claiming JWT while the + only JWT usage found is invitation-purpose is a real, both-sided conflict. + Documentation with NO usage found at all is missing evidence (WEAK), not a + contradiction: absence is not positive conflicting evidence. + """ + access_usage: list[EvidenceRef] = [] + invitation_usage: list[EvidenceRef] = [] + unclear_usage: list[EvidenceRef] = [] + jwt_docs: list[EvidenceRef] = [] + jwt_deps: list[EvidenceRef] = [] + + for row in rows: + hay = _hay(row) + kind = row["kind"] + + if kind == "token_usage": + try: + purpose = json.loads(row["metadata_json"] or "{}").get("purpose", "unclear") + except json.JSONDecodeError: + purpose = "unclear" + if purpose == "access": + access_usage.append( + _ref(row, "JWT used as an access token (login/bearer context).", "strong") + ) + elif purpose == "invitation": + invitation_usage.append( + _ref( + row, + "JWT used for invitation/verification tokens, not access.", + "moderate", + ) + ) + else: + unclear_usage.append( + _ref(row, "JWT usage found; its purpose is unclear.", "weak") + ) + elif kind in ("doc", "decision") and "jwt" in hay: + jwt_docs.append( + _ref( + row, + "Documentation or decision record references JWT.", + "moderate" if kind == "decision" else "weak", + ) + ) + elif kind == "dependency" and any(t in hay for t in _JWT_DEP_TOKENS): + jwt_deps.append( + _ref(row, "JWT library dependency is declared.", "weak") + ) + + why: list[str] = [] + missing: list[str] = [] + contradictions: list[Contradiction] = [] + supporting: list[EvidenceRef] = [] + + if access_usage: + status = SUPPORTED + supporting = access_usage + jwt_docs + jwt_deps + why.append("JWT access-token usage evidence was found.") + if jwt_docs: + why.append("Documentation or a decision record corroborates JWT usage.") + else: + missing.append("A decision record explaining the JWT choice.") + elif jwt_docs and invitation_usage: + status = CONTRADICTED + supporting = invitation_usage + doc = jwt_docs[0] + inv = invitation_usage[0] + contradictions.append( + Contradiction( + summary="Documentation claims JWT authentication, but the only JWT " + "usage found is invitation/verification tokens.", + claimed_side=f"{doc.path} references JWT in a documentation or " + "decision context, implying JWT-based authentication.", + observed_side=f"{inv.path} uses JWT for invitation/verification " + "tokens. Invitation tokens are not access-token authentication, " + "and no access-token usage was found.", + evidence=[doc, inv], + ) + ) + why.append( + "Documentation references JWT, but no access-token usage exists; " + "the only JWT usage found is invitation-purpose." + ) + missing.append("JWT access-token usage (login/bearer/authorization context).") + elif jwt_docs or jwt_deps or invitation_usage or unclear_usage: + status = WEAK + supporting = jwt_docs + jwt_deps + invitation_usage + unclear_usage + why.append( + "JWT surface exists (documentation, dependencies, or unclear usage), " + "but access-token authentication is not established." + ) + missing.append("JWT access-token usage (login/bearer/authorization context).") + else: + status = UNKNOWN + why.append( + "No JWT evidence was found in the scanned files. The claim does not " + "apply, or the usage is outside scanner coverage." + ) + missing.append("Any JWT usage, dependency, or documentation.") + + return VerificationResult( + claim_slug=definition.slug, + claim_name=definition.name, + statement=definition.statement, + status=status, + why=why, + supporting=supporting, + contradictions=contradictions, + missing=missing, + limitations=_LIMITATIONS, + scan_id=scan_id, + verified_at=_now(), + engine_version=__version__, + ) + + +_EVALUATORS = { + "billing-webhook-signature": _verify_billing_webhook_signature, + "jwt-authentication": _verify_jwt_authentication, +} + + def _ref(row: sqlite3.Row, observation: str, strength: str) -> EvidenceRef: return EvidenceRef( path=row["path"], diff --git a/src/devtime/scanner/signals.py b/src/devtime/scanner/signals.py index 24b1551..eddffe0 100644 --- a/src/devtime/scanner/signals.py +++ b/src/devtime/scanner/signals.py @@ -10,6 +10,7 @@ from __future__ import annotations import hashlib +import json import sqlite3 import uuid from dataclasses import dataclass, field @@ -214,7 +215,10 @@ def run_scan( s.start_line, s.end_line, s.confidence, - "{}", + # v0.3.0: persist extractor metadata (was hardcoded '{}', + # silently dropping e.g. the JWT purpose classification + # that verification reads back from the database). + json.dumps(s.metadata or {}), ), ) conn.commit() diff --git a/tests/integration/test_evidence_precision.py b/tests/integration/test_evidence_precision.py index 0a1a86e..99b811a 100644 --- a/tests/integration/test_evidence_precision.py +++ b/tests/integration/test_evidence_precision.py @@ -14,14 +14,14 @@ # --- version ------------------------------------------------------------------ -def test_version_is_release_0_2_0(): - # v0.2.0 release: package metadata and __version__ agree on the release version. +def test_version_is_release_0_3_0(): + # v0.3.0 release: package metadata and __version__ agree on the release version. import importlib.metadata as m - assert devtime.__version__ == "0.2.0" + assert devtime.__version__ == "0.3.0" # Distribution is published as "devtime-ei" (the name "devtime" is reserved on # PyPI); the import package and the dtc command stay "devtime"/"dtc". - assert m.version("devtime-ei") == "0.2.0" + assert m.version("devtime-ei") == "0.3.0" # --- P0 Authentication headline precision ------------------------------------ diff --git a/tests/integration/test_verification.py b/tests/integration/test_verification.py index e8b7217..b4b13c8 100644 --- a/tests/integration/test_verification.py +++ b/tests/integration/test_verification.py @@ -247,3 +247,85 @@ def test_mcp_exposes_verify_claim(tmp_path, monkeypatch): text = str(result) assert "SUPPORTED" in text assert "stripe-webhook" in text + + +# --- jwt-authentication claim (v0.3.0) ------------------------------------------- + +ACCESS_JWT = """ +import jwt from "jsonwebtoken"; +export function login(user) { + return jwt.sign({ sub: user.id }, process.env.SECRET, { expiresIn: "1h" }); +} +""" + +INVITE_JWT = """ +import jwt from "jsonwebtoken"; +export function createInviteToken(email) { + // invitation link token for email verification + return jwt.sign({ email, invite: true }, process.env.SECRET); +} +""" + +JWT_DOC = "# Authentication\n\n## Use JWT for API authentication\n\nWe sign JWTs.\n" + + +def test_jwt_supported_with_access_usage(tmp_path, monkeypatch): + _repo(tmp_path, {"src/auth/login.ts": ACCESS_JWT}) + _init_scan(tmp_path, monkeypatch) + result = _verify("jwt-authentication") + assert result.status == ver.SUPPORTED + + +def test_jwt_docs_only_is_weak_not_contradicted(tmp_path, monkeypatch): + # Absence of usage is missing evidence, never a contradiction. + _repo(tmp_path, {"docs/auth.md": JWT_DOC}) + _init_scan(tmp_path, monkeypatch) + result = _verify("jwt-authentication") + assert result.status == ver.WEAK + assert not result.contradictions + + +def test_jwt_docs_vs_invitation_only_is_contradicted(tmp_path, monkeypatch): + _repo(tmp_path, { + "docs/auth.md": JWT_DOC, + "src/tokens/invite.ts": INVITE_JWT, + }) + _init_scan(tmp_path, monkeypatch) + result = _verify("jwt-authentication") + assert result.status == ver.CONTRADICTED + c = result.contradictions[0] + assert c.claimed_side and c.observed_side + assert "invitation" in c.observed_side.lower() + + +def test_jwt_unknown_without_any_jwt_surface(tmp_path, monkeypatch): + _repo(tmp_path, {"src/util/math.ts": "export const add = (a, b) => a + b;\n"}) + _init_scan(tmp_path, monkeypatch) + result = _verify("jwt-authentication") + assert result.status == ver.UNKNOWN + + +def test_signal_metadata_survives_persistence(tmp_path, monkeypatch): + # Regression: metadata_json was hardcoded to '{}' at INSERT, silently + # dropping the JWT purpose classification the verifier reads back. + _repo(tmp_path, {"src/auth/login.ts": ACCESS_JWT}) + _init_scan(tmp_path, monkeypatch) + conn = connection.connect() + try: + row = conn.execute( + "SELECT metadata_json FROM signals WHERE kind='token_usage' LIMIT 1" + ).fetchone() + assert row is not None + assert json.loads(row["metadata_json"]).get("purpose") == "access" + finally: + conn.close() + + +def test_verify_all_returns_both_claims(tmp_path, monkeypatch): + _repo(tmp_path, {"src/auth/login.ts": ACCESS_JWT}) + _init_scan(tmp_path, monkeypatch) + result = runner.invoke(app, ["verify", "--json"]) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + ids = {r["claim_id"] for r in payload["results"]} + assert ids == {"billing-webhook-signature", "jwt-authentication"}