diff --git a/.gitignore b/.gitignore index 8ba6358..5f97fad 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ release-artifacts *.njsproj *.sln *.sw? + +# Builder handoff briefs — not product truth, never released +docs/prompts/ diff --git a/docs/connector-hardening-audit.md b/docs/connector-hardening-audit.md index 4c4df23..427aa6b 100644 --- a/docs/connector-hardening-audit.md +++ b/docs/connector-hardening-audit.md @@ -1,18 +1,23 @@ # Connector Super-Hardening Audit -Generated 2026-06-25T02:24:05.607Z from the live connector registry (CONNECTOR_AUDIT_DEFINITIONS). +Generated 2026-06-29T20:37:46.794Z from the live connector registry (CONNECTOR_AUDIT_DEFINITIONS). Metadata-based hardening score (catalog attributes), not a line-by-line code audit. No secrets printed. -- Connectors: **38** +- Connectors: **48** - Average hardening score: **100/100** -- Fully hardened (no catalog gaps): **37** +- Fully hardened (no catalog gaps): **47** - With gaps: **1** ## Coverage table | Connector | Domain | Cadence | Trust | Key | Source trail | Persistence | Resolver/Exposure | Score | Next action | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| arXiv AI | AI research metadata | periodic | official-api | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | +| ECB Press RSS | central bank policy releases | periodic | rss-public | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | +| Federal Reserve Press RSS | central bank policy releases | periodic | rss-public | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | | SEC EDGAR | company disclosure | periodic | public-disclosure | yes | yes | yes | yes/curated-reference | 100 | None — fully hardened per catalog attributes. | +| SEC Press RSS | company disclosure and enforcement releases | periodic | rss-public | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | +| Public Crypto Realtime | crypto market data | realtime | public-unauthenticated | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | | CISA Advisories | defensive security | periodic | official-api | no | yes | yes | partial/identifier-only | 100 | None — fully hardened per catalog attributes. | | CISA KEV | defensive security | periodic | official-api | no | yes | yes | partial/identifier-only | 100 | None — fully hardened per catalog attributes. | | GitHub Advisories | defensive security | periodic | public-unauthenticated | no | yes | yes | partial/identifier-only | 100 | None — fully hardened per catalog attributes. | @@ -20,9 +25,10 @@ Metadata-based hardening score (catalog attributes), not a line-by-line code aud | OSV.dev | defensive security | periodic | official-api | no | yes | yes | partial/identifier-only | 100 | None — fully hardened per catalog attributes. | | Crossref | DOI metadata | periodic | official-api | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | | EIA | energy and commodities | periodic | official-api | yes | yes | yes | yes/curated-reference | 100 | None — fully hardened per catalog attributes. | +| EIA Public Bulk | energy and commodities | periodic | official-api | no | yes | yes | yes/curated-reference | 100 | None — fully hardened per catalog attributes. | | EIA Nuclear Plants | energy facilities | periodic | official-api | yes | yes | yes | partial/curated-reference | 100 | None — fully hardened per catalog attributes. | | EIA Power Plants | energy facilities | periodic | official-api | yes | yes | yes | partial/curated-reference | 100 | None — fully hardened per catalog attributes. | -| EIA Refineries | energy facilities | periodic | official-api | no | yes | yes | partial/curated-reference | 100 | None — fully hardened per catalog attributes. | +| EIA Refineries | energy facilities | periodic | official-api | yes | yes | yes | partial/curated-reference | 100 | None — fully hardened per catalog attributes. | | LNG Terminals | energy facilities | periodic | official-api | yes | yes | yes | partial/curated-reference | 100 | None — fully hardened per catalog attributes. | | NRC Reactor Status | energy operations | periodic | official-api | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | | ETF Holdings | ETF basket exposure | periodic | public-disclosure | no | yes | yes | partial/curated-reference | 100 | None — fully hardened per catalog attributes. | @@ -31,24 +37,28 @@ Metadata-based hardening score (catalog attributes), not a line-by-line code aud | SEC Form 4 | insider transactions | periodic | public-disclosure | yes | yes | yes | yes/curated-reference | 100 | None — fully hardened per catalog attributes. | | SEC Form 13F | institutional holdings | periodic | public-disclosure | yes | yes | yes | partial/curated-reference | 100 | None — fully hardened per catalog attributes. | | BLS | labor and prices | periodic | official-api | no | yes | yes | partial/identifier-only | 100 | None — fully hardened per catalog attributes. | -| Congress.gov | legislative pipeline | periodic | official-api | yes | yes | yes | identifier-only/future | 100 | None — fully hardened per catalog attributes. | -| FRED | macro time series | periodic | official-api | yes | yes | yes | partial/identifier-only | 100 | None — fully hardened per catalog attributes. | +| Congress.gov | legislative pipeline | periodic | official-api | no | yes | yes | identifier-only/future | 100 | None — fully hardened per catalog attributes. | +| FRED | macro time series | periodic | official-api | no | yes | yes | partial/identifier-only | 100 | None — fully hardened per catalog attributes. | +| WSJ Markets RSS | market headline observation | periodic | rss-public | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | | Market Reference Master | market identity | periodic | official-api | no | yes | yes | yes/curated-reference | 100 | None — fully hardened per catalog attributes. | | Equity/ETF Quotes | market price data | realtime | auth-gated | yes | yes | n/a (realtime) | no/none | 100 | None — fully hardened per catalog attributes. | -| USGS Minerals | materials | periodic | official-api | yes | yes | yes | partial/curated-reference | 100 | None — fully hardened per catalog attributes. | +| USGS Minerals | materials | periodic | official-api | no | yes | yes | partial/curated-reference | 100 | None — fully hardened per catalog attributes. | | GDELT (media) | media observation | periodic | media-observation | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | | BEA | national accounts | periodic | official-api | yes | yes | yes | partial/identifier-only | 100 | None — fully hardened per catalog attributes. | | GitHub Releases | open-source activity | periodic | public-unauthenticated | no | yes | yes | yes/curated-reference | 100 | None — fully hardened per catalog attributes. | +| GitHub Public Repository Search | open-source repository metadata | periodic | official-api | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | | Options chain / open interest | options market data | realtime | auth-gated | yes | yes | n/a (realtime) | no/none | 90 | Wire the live fetch -> renderer store when prioritized. | | USPTO PatentsView | patent intelligence | periodic | official-api | yes | yes | yes | yes/curated-reference | 100 | None — fully hardened per catalog attributes. | | World Port Index | physical ports | periodic | official-api | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | | USGS Earthquakes | physical-world events | periodic | official-api | no | yes | yes | partial/future | 100 | None — fully hardened per catalog attributes. | | Federal Register | regulatory documents | periodic | official-api | no | yes | yes | partial/future | 100 | None — fully hardened per catalog attributes. | | SEC Company Facts | reported fundamentals | periodic | public-disclosure | yes | yes | yes | yes/curated-reference | 100 | None — fully hardened per catalog attributes. | -| OpenAlex | research metadata | periodic | official-api | yes | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | +| OpenAlex | research metadata | periodic | official-api | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | | OFAC SDN | sanctions enforcement | periodic | official-api | no | yes | yes | identifier-only/future | 100 | None — fully hardened per catalog attributes. | +| NASA News RSS | space and science releases | periodic | rss-public | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | +| Launch Library 2 | space launch schedule | periodic | public-unauthenticated | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | | UN Comtrade | trade flows | periodic | official-api | yes | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | -| UN/LOCODE | trade/logistics location codes | periodic | official-api | yes | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | +| UN/LOCODE | trade/logistics location codes | periodic | official-api | no | yes | yes | no/none | 100 | None — fully hardened per catalog attributes. | | NOAA/NWS Alerts | weather disruption | periodic | official-api | no | yes | yes | partial/future | 100 | None — fully hardened per catalog attributes. | ## Weakest connectors (lowest score first) @@ -59,6 +69,12 @@ Metadata-based hardening score (catalog attributes), not a line-by-line code aud - Gaps: Live runtime wiring deferred (built + tested, not yet flowing). - Next action: Wire the live fetch -> renderer store when prioritized. +### arXiv AI — 100/100 +- Source: arXiv public API (https://export.arxiv.org/api/query) +- Key required: no +- Gaps: none +- Next action: None — fully hardened per catalog attributes. + ### BEA — 100/100 - Source: U.S. Bureau of Economic Analysis (https://www.bea.gov/resources/for-developers) - Key required: ATLASZ_BEA_API_KEY @@ -77,9 +93,3 @@ Metadata-based hardening score (catalog attributes), not a line-by-line code aud - Gaps: none - Next action: None — fully hardened per catalog attributes. -### CISA KEV — 100/100 -- Source: CISA Known Exploited Vulnerabilities catalog (https://www.cisa.gov/known-exploited-vulnerabilities-catalog) -- Key required: no -- Gaps: none -- Next action: None — fully hardened per catalog attributes. - diff --git a/docs/data-connectivity-matrix.md b/docs/data-connectivity-matrix.md new file mode 100644 index 0000000..b14a135 --- /dev/null +++ b/docs/data-connectivity-matrix.md @@ -0,0 +1,98 @@ +# Connector Reality Matrix + + + +Generated 2026-06-29T20:37:46.141Z from the live connector registry. +Structural truth only — what each connector needs to run. Env-var NAMES only, +never values. "Online" is never claimed here; live status comes from the runtime +audit (`npx tsx scripts/runtimeVerification.mts`). + +- Connectors: **48** +- no-key public: **33** +- no-key public (config-required): **4** +- key-gated: **8** +- configured-only: **2** +- deferred wiring: **1** +- not-wired: **0** + +## Matrix + +| Connector | Domain | Official owner | Cold-start status | Auth (secret key) | No-key mode | Configured URL | Required env | Cadence | Trust | Freshness window | Source trail / persistence | Decision | Blocker | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| arXiv AI | AI research metadata | arXiv public API | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h (unresolved-by-design) | arXiv research source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| ECB Press RSS | central bank policy releases | European Central Bank press releases RSS | pending-first-fetch | no | yes | no | — | periodic | rss-public | 30h (unresolved-by-design) | ECB RSS source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| Federal Reserve Press RSS | central bank policy releases | Board of Governors of the Federal Reserve System press releases RSS | pending-first-fetch | no | yes | no | — | periodic | rss-public | 30h (unresolved-by-design) | Federal Reserve RSS source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| SEC EDGAR | company disclosure | U.S. Securities and Exchange Commission | missing-key | no | yes | no | ATLASZ_SEC_USER_AGENT | periodic | public-disclosure | 30h | SEC filing source trail — tables: sec_company_filings, world_intel_events, source_audit_log | no-key public (config-required) | Requires contact User-Agent: ATLASZ_SEC_USER_AGENT | +| SEC Press RSS | company disclosure and enforcement releases | U.S. Securities and Exchange Commission press releases RSS | pending-first-fetch | no | yes | no | — | periodic | rss-public | 30h (unresolved-by-design) | SEC press-release RSS source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| Public Crypto Realtime | crypto market data | CoinGecko public REST plus Coinbase/Binance public WebSocket market data | pending-first-fetch | no | yes | no | — | realtime | public-unauthenticated | 30h (unresolved-by-design) | Realtime widgets / sampled market tick source trail — tables: market_ticks_daily | no-key public | — | +| CISA Advisories | defensive security | CISA Cybersecurity Advisories RSS | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | CVE source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| CISA KEV | defensive security | CISA Known Exploited Vulnerabilities catalog | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | CVE source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| GitHub Advisories | defensive security | GitHub Security Advisory Database | pending-first-fetch | no | yes | no | — | periodic | public-unauthenticated | 30h | CVE source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| NVD CVEs | defensive security | NIST National Vulnerability Database | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | CVE source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| OSV.dev | defensive security | OSV.dev public API | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | CVE source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| Crossref | DOI metadata | Crossref REST API | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h (unresolved-by-design) | Crossref DOI metadata source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| EIA | energy and commodities | U.S. Energy Information Administration | missing-key | yes | no | no | ATLASZ_EIA_API_KEY | periodic | official-api | 30h | Energy source trail cards — tables: eia_energy_records, world_intel_events, source_audit_log | key-gated | Missing secret key: ATLASZ_EIA_API_KEY | +| EIA Public Bulk | energy and commodities | U.S. Energy Information Administration public bulk files | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | Energy source trail cards — tables: eia_energy_records, world_intel_events, source_audit_log | no-key public | — | +| EIA Nuclear Plants | energy facilities | EIA-860M filtered to nuclear | missing-key | yes | no | no | ATLASZ_EIA_API_KEY | periodic | official-api | 30h | Nuclear plant source trail — tables: world_intel_events, source_audit_log | key-gated | Missing secret key: ATLASZ_EIA_API_KEY | +| EIA Power Plants | energy facilities | EIA-860M operating generator capacity | missing-key | yes | no | no | ATLASZ_EIA_API_KEY | periodic | official-api | 30h | EIA facility source trail — tables: world_intel_events, source_audit_log | key-gated | Missing secret key: ATLASZ_EIA_API_KEY | +| EIA Refineries | energy facilities | EIA U.S. Energy Atlas Petroleum Refineries (EIA-820) | missing-key | no | yes | yes | ATLASZ_EIA_REFINERIES_URL | periodic | official-api | 30h | EIA refinery source trail — tables: world_intel_events, source_audit_log | configured-only | Requires official URL: ATLASZ_EIA_REFINERIES_URL | +| LNG Terminals | energy facilities | EIA Atlas / FERC LNG terminals | missing-key | no | yes | yes | ATLASZ_LNG_TERMINALS_URL | periodic | official-api | 30h | LNG terminal source trail — tables: world_intel_events, source_audit_log | configured-only | Requires official URL: ATLASZ_LNG_TERMINALS_URL | +| NRC Reactor Status | energy operations | NRC Power Reactor Status Report | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h (unresolved-by-design) | NRC reactor status source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| ETF Holdings | ETF basket exposure | Official issuer-published ETF holdings files | pending-first-fetch | no | yes | no | — | periodic | public-disclosure | 30h | ETF holdings source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| Treasury Fiscal Data | government finance | U.S. Treasury Fiscal Data | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | Fiscal source trail cards — tables: treasury_fiscal_records, world_intel_events, source_audit_log | no-key public | — | +| EIA Grid Regions | grid geography | EIA electricity/rto respondent facet | missing-key | yes | no | no | ATLASZ_EIA_API_KEY | periodic | official-api | 30h | Grid region source trail — tables: world_intel_events, source_audit_log | key-gated | Missing secret key: ATLASZ_EIA_API_KEY | +| SEC Form 4 | insider transactions | SEC Form 4 ownership filings | missing-key | no | yes | no | ATLASZ_SEC_USER_AGENT | periodic | public-disclosure | 30h | SEC Form 4 source trail — tables: world_intel_events, source_audit_log | no-key public (config-required) | Requires contact User-Agent: ATLASZ_SEC_USER_AGENT | +| SEC Form 13F | institutional holdings | SEC Form 13F-HR filings | missing-key | no | yes | no | ATLASZ_SEC_USER_AGENT | periodic | public-disclosure | 30h | SEC Form 13F source trail — tables: world_intel_events, source_audit_log | no-key public (config-required) | Requires contact User-Agent: ATLASZ_SEC_USER_AGENT | +| BLS | labor and prices | U.S. Bureau of Labor Statistics | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | Macro context cards — tables: world_intel_events, source_audit_log | no-key public | — | +| Congress.gov | legislative pipeline | Library of Congress / Congress.gov API | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | Congress bill/action source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| FRED | macro time series | Federal Reserve Bank of St. Louis FRED | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | Macro context cards — tables: fred_macro_observations, world_intel_events, source_audit_log | no-key public | — | +| WSJ Markets RSS | market headline observation | Dow Jones / WSJ public Markets RSS feed | pending-first-fetch | no | yes | no | — | periodic | rss-public | 30h (unresolved-by-design) | Public RSS headline source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| Market Reference Master | market identity | SEC company_tickers.json | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | Market identity source trail — tables: market_identity_master, world_intel_events, source_audit_log | no-key public | — | +| Equity/ETF Quotes | market price data | Alpaca Market Data (IEX) latest trades | missing-key | yes | no | no | ATLASZ_ALPACA_API_KEY, ATLASZ_ALPACA_SECRET_KEY | realtime | auth-gated | 30h (unresolved-by-design) | Market Quote source trail — no persistence | key-gated | Missing secret key: ATLASZ_ALPACA_API_KEY, ATLASZ_ALPACA_SECRET_KEY | +| USGS Minerals | materials | USGS Mineral Resources (MRDS default, optional USMIN) | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | USGS mineral site source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| GDELT (media) | media observation | GDELT Project DOC 2.0 API | pending-first-fetch | no | yes | no | — | periodic | media-observation | 30h (unresolved-by-design) | GDELT media-observation source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| BEA | national accounts | U.S. Bureau of Economic Analysis | missing-key | yes | no | no | ATLASZ_BEA_API_KEY | periodic | official-api | 30h | BEA macro context cards — tables: bea_observations, world_intel_events, source_audit_log | key-gated | Missing secret key: ATLASZ_BEA_API_KEY | +| GitHub Releases | open-source activity | GitHub REST Releases API | pending-first-fetch | no | yes | no | — | periodic | public-unauthenticated | 30h | Timeline/source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| GitHub Public Repository Search | open-source repository metadata | GitHub REST Search API | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h (unresolved-by-design) | GitHub public repository source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| Options chain / open interest | options market data | Alpaca Market Data options snapshots | deferred | yes | no | no | ATLASZ_ALPACA_API_KEY, ATLASZ_ALPACA_SECRET_KEY, ATLASZ_OPTIONS_UNDERLYINGS | realtime | auth-gated | 30h (unresolved-by-design) | Options source trail — no persistence | deferred wiring | Missing secret key: ATLASZ_ALPACA_API_KEY, ATLASZ_ALPACA_SECRET_KEY | +| USPTO PatentsView | patent intelligence | USPTO PatentsView API | missing-key | yes | no | no | ATLASZ_PATENTSVIEW_API_KEY | periodic | official-api | 30h | Patent source trail — tables: world_intel_events, source_audit_log | key-gated | Missing secret key: ATLASZ_PATENTSVIEW_API_KEY | +| World Port Index | physical ports | NGA World Port Index (Pub 150) | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h (unresolved-by-design) | World Port Index source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| USGS Earthquakes | physical-world events | USGS Earthquake Hazards Program | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | Timeline/event source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| Federal Register | regulatory documents | Office of the Federal Register / FederalRegister.gov API | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | Regulatory document source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| SEC Company Facts | reported fundamentals | SEC companyfacts XBRL API | missing-key | no | yes | no | ATLASZ_SEC_USER_AGENT | periodic | public-disclosure | 30h | SEC Company Facts source trail — tables: world_intel_events, source_audit_log | no-key public (config-required) | Requires contact User-Agent: ATLASZ_SEC_USER_AGENT | +| OpenAlex | research metadata | OpenAlex Works API | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h (unresolved-by-design) | OpenAlex research source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| OFAC SDN | sanctions enforcement | U.S. Treasury OFAC Sanctions List Service | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | OFAC sanctions source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| NASA News RSS | space and science releases | NASA news-release RSS | pending-first-fetch | no | yes | no | — | periodic | rss-public | 30h (unresolved-by-design) | NASA RSS source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| Launch Library 2 | space launch schedule | The Space Devs Launch Library 2 public API | pending-first-fetch | no | yes | no | — | periodic | public-unauthenticated | 30h (unresolved-by-design) | Launch Library source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| UN Comtrade | trade flows | United Nations Comtrade | missing-key | yes | no | no | ATLASZ_UN_COMTRADE_API_KEY | periodic | official-api | 30h (unresolved-by-design) | UN Comtrade trade-flow source trail — tables: world_intel_events, source_audit_log | key-gated | Missing secret key: ATLASZ_UN_COMTRADE_API_KEY | +| UN/LOCODE | trade/logistics location codes | UNECE UN/LOCODE code list | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h (unresolved-by-design) | Port / UN/LOCODE source trail — tables: world_intel_events, source_audit_log | no-key public | — | +| NOAA/NWS Alerts | weather disruption | National Weather Service alerts API | pending-first-fetch | no | yes | no | — | periodic | official-api | 30h | NOAA weather source trail — tables: world_intel_events, source_audit_log | no-key public | — | + +## Locked connectors — blocker and no-key options + +Each row below cannot complete a first fetch until its blocker is resolved. +A same-domain no-key sibling means partial coverage already exists without a key. + +| Connector | Decision | Blocker | Official docs | Same-domain no-key sibling | +| --- | --- | --- | --- | --- | +| EIA (`eia`) | key-gated | Missing secret key: ATLASZ_EIA_API_KEY | https://www.eia.gov/opendata/ | eia-bulk-public | +| EIA Nuclear Plants (`eia-nuclear`) | key-gated | Missing secret key: ATLASZ_EIA_API_KEY | https://www.eia.gov/electricity/data/eia860m/ | — (none) | +| EIA Power Plants (`eia-power-plants`) | key-gated | Missing secret key: ATLASZ_EIA_API_KEY | https://www.eia.gov/electricity/data/eia860m/ | — (none) | +| EIA Refineries (`eia-refineries`) | configured-only | Requires official URL: ATLASZ_EIA_REFINERIES_URL | https://atlas.eia.gov/datasets/petroleum-refineries | — (none) | +| LNG Terminals (`lng-terminals`) | configured-only | Requires official URL: ATLASZ_LNG_TERMINALS_URL | https://atlas.eia.gov/datasets/liquefied-natural-gas-lng-import-and-export-terminals | — (none) | +| EIA Grid Regions (`eia-balancing-authorities`) | key-gated | Missing secret key: ATLASZ_EIA_API_KEY | https://www.eia.gov/opendata/browser/electricity/rto/region-data | — (none) | +| Equity/ETF Quotes (`equities-prices`) | key-gated | Missing secret key: ATLASZ_ALPACA_API_KEY, ATLASZ_ALPACA_SECRET_KEY | https://alpaca.markets/data | — (none) | +| BEA (`bea`) | key-gated | Missing secret key: ATLASZ_BEA_API_KEY | https://www.bea.gov/resources/for-developers | — (none) | +| USPTO PatentsView (`uspto`) | key-gated | Missing secret key: ATLASZ_PATENTSVIEW_API_KEY | https://developer.uspto.gov/api-catalog | — (none) | +| UN Comtrade (`un-comtrade`) | key-gated | Missing secret key: ATLASZ_UN_COMTRADE_API_KEY | https://comtradeplus.un.org/ | — (none) | + +## How to read "Decision" + +- **no-key public** — runs with no key and no extra config. +- **no-key public (config-required)** — no secret key, but needs a contact User-Agent or an allowlist/config value. +- **key-gated** — requires a secret API key (named in Required env). Fails closed until configured. +- **configured-only** — requires an official source URL to be pinned (named in Required env). No stale default. +- **deferred wiring** — built and tested, live runtime wiring intentionally deferred. +- **not-wired** — no runtime adapter yet. diff --git a/scripts/connectorRealityMatrix.mts b/scripts/connectorRealityMatrix.mts new file mode 100644 index 0000000..83de1e0 --- /dev/null +++ b/scripts/connectorRealityMatrix.mts @@ -0,0 +1,106 @@ +/* + * Generates docs/data-connectivity-matrix.md from the real connector registry + * (CONNECTOR_AUDIT_DEFINITIONS). Deterministic; prints env-var NAMES only, never + * values. Run: npx tsx scripts/connectorRealityMatrix.mts + * + * This matrix is derived structural truth (what each connector needs to run). It + * never claims a connector is "online" — live runtime status comes from the audit + * (scripts/runtimeVerification.mts). Regenerate after editing the registry. + */ +import { writeFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { dirname, join } from 'node:path' +import { + buildConnectorMatrix, + summarizeMatrix, + DECISION_ORDER, + type ConnectorMatrixRow, +} from '../src/engine/connectorRealityMatrix' + +const rows = buildConnectorMatrix() +const summary = summarizeMatrix(rows) +const generatedAt = new Date().toISOString() + +function cell(value: string): string { + return value.replace(/\|/g, '\\|').replace(/\n/g, ' ') +} + +function yn(value: boolean): string { + return value ? 'yes' : 'no' +} + +function envList(row: ConnectorMatrixRow): string { + return row.requiredEnv.length > 0 ? row.requiredEnv.join(', ') : '—' +} + +const lines: string[] = [] +lines.push('# Connector Reality Matrix') +lines.push('') +lines.push('') +lines.push('') +lines.push(`Generated ${generatedAt} from the live connector registry.`) +lines.push('Structural truth only — what each connector needs to run. Env-var NAMES only,') +lines.push('never values. "Online" is never claimed here; live status comes from the runtime') +lines.push('audit (`npx tsx scripts/runtimeVerification.mts`).') +lines.push('') +lines.push(`- Connectors: **${summary.total}**`) +for (const decision of DECISION_ORDER) { + lines.push(`- ${decision}: **${summary.byDecision[decision]}**`) +} +lines.push('') +lines.push('## Matrix') +lines.push('') +lines.push( + '| Connector | Domain | Official owner | Cold-start status | Auth (secret key) | No-key mode | Configured URL | Required env | Cadence | Trust | Freshness window | Source trail / persistence | Decision | Blocker |', +) +lines.push( + '| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |', +) +for (const r of rows) { + const trail = [r.sourceTrailUi, r.persistenceTables.length > 0 ? `tables: ${r.persistenceTables.join(', ')}` : 'no persistence'] + .filter(Boolean) + .join(' — ') + const freshness = `${r.freshnessWindowHours}h${r.unresolvedByDesign ? ' (unresolved-by-design)' : ''}` + lines.push( + `| ${cell(r.label)} | ${cell(r.domain)} | ${cell(r.owner)} | ${r.coldStartStatus} | ${yn(r.authRequired)} | ${yn(r.noKeyMode)} | ${yn(r.configuredUrlRequired)} | ${cell(envList(r))} | ${r.cadence} | ${r.trust} | ${cell(freshness)} | ${cell(trail)} | ${cell(r.decision)} | ${cell(r.blocker)} |`, + ) +} +lines.push('') + +// Locked connectors: exactly what is blocking each one + any no-key path. +const locked = rows.filter((r) => r.decision === 'key-gated' || r.decision === 'configured-only') +lines.push('## Locked connectors — blocker and no-key options') +lines.push('') +if (locked.length === 0) { + lines.push('_None — every connector runs with no secret key or configured URL._') +} else { + lines.push('Each row below cannot complete a first fetch until its blocker is resolved.') + lines.push('A same-domain no-key sibling means partial coverage already exists without a key.') + lines.push('') + lines.push('| Connector | Decision | Blocker | Official docs | Same-domain no-key sibling |') + lines.push('| --- | --- | --- | --- | --- |') + for (const r of locked) { + const siblings = r.noKeySiblingIds.length > 0 ? r.noKeySiblingIds.join(', ') : '— (none)' + lines.push( + `| ${cell(r.label)} (\`${r.id}\`) | ${r.decision} | ${cell(r.blocker)} | ${cell(r.officialUrl)} | ${cell(siblings)} |`, + ) + } +} +lines.push('') +lines.push('## How to read "Decision"') +lines.push('') +lines.push('- **no-key public** — runs with no key and no extra config.') +lines.push('- **no-key public (config-required)** — no secret key, but needs a contact User-Agent or an allowlist/config value.') +lines.push('- **key-gated** — requires a secret API key (named in Required env). Fails closed until configured.') +lines.push('- **configured-only** — requires an official source URL to be pinned (named in Required env). No stale default.') +lines.push('- **deferred wiring** — built and tested, live runtime wiring intentionally deferred.') +lines.push('- **not-wired** — no runtime adapter yet.') + +const outPath = join(dirname(fileURLToPath(import.meta.url)), '..', 'docs', 'data-connectivity-matrix.md') +writeFileSync(outPath, `${lines.join('\n')}\n`, 'utf8') +console.log( + `Wrote ${outPath} — ${summary.total} connectors: ${summary.noKeyPublicCount} no-key, ` + + `${summary.byDecision['key-gated']} key-gated, ${summary.byDecision['configured-only']} configured-only`, +) diff --git a/src/engine/connectorRealityMatrix.ts b/src/engine/connectorRealityMatrix.ts new file mode 100644 index 0000000..d718c17 --- /dev/null +++ b/src/engine/connectorRealityMatrix.ts @@ -0,0 +1,182 @@ +/* + * Connector Reality Matrix. + * + * Derives a deterministic, honest connectivity matrix from the REAL registry + * (CONNECTOR_AUDIT_DEFINITIONS). Every column is computed from catalog facts — + * access model, requiredEnv, cadence, freshness window, unresolved-by-design — so + * the matrix can never drift from the code. It prints env-var NAMES only, never + * values. It does not claim a connector is online; runtime status comes from the + * live audit. This is the structural truth a human can act on: + * - which connectors need a secret key (and exactly which env var), + * - which need an official configured URL, + * - which already run with no key, + * - and which key-gated connectors have a same-domain no-key sibling/mode. + */ +import { + CONNECTOR_AUDIT_DEFINITIONS, + type ConnectorAuditDefinition, + type ConnectorRuntimeStatus, +} from './runtimeAudit' + +const HOUR_MS = 60 * 60 * 1000 + +/** Classifies each required env var by suffix — no value is ever read. */ +export type EnvClassification = { + secretKeys: string[] + configuredUrls: string[] + userAgents: string[] + otherConfig: string[] +} + +export type MatrixDecision = + | 'no-key public' + | 'no-key public (config-required)' + | 'key-gated' + | 'configured-only' + | 'deferred wiring' + | 'not-wired' + +export type ConnectorMatrixRow = { + id: string + label: string + domain: string + owner: string + officialUrl: string + /** Cold-start status with no source snapshot — mirrors runtimeAudit.connectorStatus(). */ + coldStartStatus: ConnectorRuntimeStatus + /** True only when a secret API/secret key is required. */ + authRequired: boolean + /** True when the connector needs no secret key (UA/URL/allowlist config may still apply). */ + noKeyMode: boolean + configuredUrlRequired: boolean + requiredEnv: string[] + env: EnvClassification + cadence: ConnectorAuditDefinition['cadence'] + trust: ConnectorAuditDefinition['trust'] + freshnessWindowHours: number + unresolvedByDesign: boolean + sourceTrailUi: string + persistenceTables: string[] + decision: MatrixDecision + /** Human-readable blocker, or '—' when nothing is blocking a first fetch. */ + blocker: string + /** Same-domain connectors that already run with no secret key. */ + noKeySiblingIds: string[] +} + +export function classifyEnv(vars: string[]): EnvClassification { + const secretKeys: string[] = [] + const configuredUrls: string[] = [] + const userAgents: string[] = [] + const otherConfig: string[] = [] + for (const name of vars) { + if (/(_API_KEY|_SECRET_KEY|_TOKEN)$/.test(name)) secretKeys.push(name) + else if (/_URL$/.test(name)) configuredUrls.push(name) + else if (/_USER_AGENT$/.test(name)) userAgents.push(name) + else otherConfig.push(name) + } + return { secretKeys, configuredUrls, userAgents, otherConfig } +} + +/** Cold-start status (no source snapshot), mirroring runtimeAudit.connectorStatus(). */ +function coldStartStatus(d: ConnectorAuditDefinition): ConnectorRuntimeStatus { + if (!d.implemented) return 'not-wired' + if (d.liveWiringDeferred) return 'deferred' + if (d.requiredEnv.length > 0) return 'missing-key' + return 'pending-first-fetch' +} + +function decide(d: ConnectorAuditDefinition, env: EnvClassification): MatrixDecision { + if (!d.implemented) return 'not-wired' + if (d.liveWiringDeferred) return 'deferred wiring' + if (env.secretKeys.length > 0) return 'key-gated' + if (env.configuredUrls.length > 0) return 'configured-only' + if (env.userAgents.length > 0 || env.otherConfig.length > 0) return 'no-key public (config-required)' + return 'no-key public' +} + +function blockerFor(d: ConnectorAuditDefinition, env: EnvClassification): string { + if (!d.implemented) return 'No runtime adapter wired.' + if (env.secretKeys.length > 0) return `Missing secret key: ${env.secretKeys.join(', ')}` + if (env.configuredUrls.length > 0) return `Requires official URL: ${env.configuredUrls.join(', ')}` + if (env.userAgents.length > 0) return `Requires contact User-Agent: ${env.userAgents.join(', ')}` + if (env.otherConfig.length > 0) return `Requires config: ${env.otherConfig.join(', ')}` + return '—' +} + +export function buildConnectorMatrix( + definitions: ConnectorAuditDefinition[] = CONNECTOR_AUDIT_DEFINITIONS, +): ConnectorMatrixRow[] { + // No-key, implemented connectors grouped by domain, for sibling cross-reference. + const noKeyByDomain = new Map() + for (const d of definitions) { + const env = classifyEnv(d.requiredEnv) + if (d.implemented && env.secretKeys.length === 0 && env.configuredUrls.length === 0) { + const list = noKeyByDomain.get(d.domain) ?? [] + list.push(d.id) + noKeyByDomain.set(d.domain, list) + } + } + + return definitions + .map((d): ConnectorMatrixRow => { + const env = classifyEnv(d.requiredEnv) + const noKeyMode = env.secretKeys.length === 0 + const noKeySiblingIds = (noKeyByDomain.get(d.domain) ?? []).filter((id) => id !== d.id) + return { + id: d.id, + label: d.label, + domain: d.domain, + owner: d.sourceIdentity, + officialUrl: d.officialUrl, + coldStartStatus: coldStartStatus(d), + authRequired: env.secretKeys.length > 0, + noKeyMode, + configuredUrlRequired: env.configuredUrls.length > 0, + requiredEnv: d.requiredEnv, + env, + cadence: d.cadence, + trust: d.trust, + freshnessWindowHours: Math.round((d.staleAfterMs ?? 30 * HOUR_MS) / HOUR_MS), + unresolvedByDesign: Boolean(d.unresolvedByDesign), + sourceTrailUi: d.sourceTrailUi, + persistenceTables: d.persistenceTables, + decision: decide(d, env), + blocker: blockerFor(d, env), + noKeySiblingIds, + } + }) + .sort((a, b) => a.domain.localeCompare(b.domain) || a.label.localeCompare(b.label)) +} + +export type MatrixSummary = { + total: number + byDecision: Record + keyGatedIds: string[] + configuredOnlyIds: string[] + noKeyPublicCount: number +} + +const DECISION_ORDER: MatrixDecision[] = [ + 'no-key public', + 'no-key public (config-required)', + 'key-gated', + 'configured-only', + 'deferred wiring', + 'not-wired', +] + +export function summarizeMatrix(rows: ConnectorMatrixRow[] = buildConnectorMatrix()): MatrixSummary { + const byDecision = Object.fromEntries(DECISION_ORDER.map((d) => [d, 0])) as Record + for (const r of rows) byDecision[r.decision] += 1 + return { + total: rows.length, + byDecision, + keyGatedIds: rows.filter((r) => r.decision === 'key-gated').map((r) => r.id).sort(), + configuredOnlyIds: rows.filter((r) => r.decision === 'configured-only').map((r) => r.id).sort(), + noKeyPublicCount: + byDecision['no-key public'] + byDecision['no-key public (config-required)'], + } +} + +export { DECISION_ORDER } diff --git a/test/connectorRealityMatrix.test.ts b/test/connectorRealityMatrix.test.ts new file mode 100644 index 0000000..93f415e --- /dev/null +++ b/test/connectorRealityMatrix.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest' +import { + buildConnectorMatrix, + summarizeMatrix, + classifyEnv, +} from '../src/engine/connectorRealityMatrix' +import { CONNECTOR_AUDIT_DEFINITIONS } from '../src/engine/runtimeAudit' + +describe('connector reality matrix', () => { + const rows = buildConnectorMatrix() + const byId = new Map(rows.map((r) => [r.id, r])) + + it('covers every registry connector exactly once', () => { + expect(rows.length).toBe(CONNECTOR_AUDIT_DEFINITIONS.length) + expect(new Set(rows.map((r) => r.id)).size).toBe(rows.length) + }) + + it('classifies env vars by suffix without reading values', () => { + const c = classifyEnv(['ATLASZ_EIA_API_KEY', 'ATLASZ_LNG_TERMINALS_URL', 'ATLASZ_SEC_USER_AGENT', 'ATLASZ_OPTIONS_UNDERLYINGS']) + expect(c.secretKeys).toEqual(['ATLASZ_EIA_API_KEY']) + expect(c.configuredUrls).toEqual(['ATLASZ_LNG_TERMINALS_URL']) + expect(c.userAgents).toEqual(['ATLASZ_SEC_USER_AGENT']) + expect(c.otherConfig).toEqual(['ATLASZ_OPTIONS_UNDERLYINGS']) + }) + + it('marks secret-keyed connectors key-gated with a missing-key cold start', () => { + const eia = byId.get('eia') + expect(eia?.decision).toBe('key-gated') + expect(eia?.authRequired).toBe(true) + expect(eia?.noKeyMode).toBe(false) + expect(eia?.coldStartStatus).toBe('missing-key') + expect(eia?.blocker).toContain('ATLASZ_EIA_API_KEY') + }) + + it('marks URL-only connectors configured-only, not key-gated', () => { + const lng = byId.get('lng-terminals') + expect(lng?.decision).toBe('configured-only') + expect(lng?.authRequired).toBe(false) + expect(lng?.configuredUrlRequired).toBe(true) + expect(lng?.blocker).toContain('ATLASZ_LNG_TERMINALS_URL') + }) + + it('treats SEC user-agent connectors as no-key (config-required), not key-gated', () => { + const edgar = byId.get('sec-edgar') + expect(edgar?.authRequired).toBe(false) + expect(edgar?.noKeyMode).toBe(true) + expect(edgar?.decision).toBe('no-key public (config-required)') + }) + + it('cross-references same-domain no-key siblings for locked connectors', () => { + // eia (key-gated) shares the "energy and commodities" domain with eia-bulk-public (no-key). + expect(byId.get('eia')?.noKeySiblingIds).toContain('eia-bulk-public') + expect(byId.get('eia-bulk-public')?.noKeyMode).toBe(true) + }) + + it('never emits a secret value — only env var names appear', () => { + for (const r of rows) { + for (const name of r.requiredEnv) { + expect(name).toMatch(/^[A-Z0-9_]+$/) + } + } + }) + + it('summary decision counts add up to the connector total', () => { + const summary = summarizeMatrix(rows) + const sum = Object.values(summary.byDecision).reduce((a, b) => a + b, 0) + expect(sum).toBe(summary.total) + expect(summary.total).toBe(CONNECTOR_AUDIT_DEFINITIONS.length) + }) +})