From 22a0378e5bce1710ac7251bf82b8878edeaba270 Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Thu, 16 Jul 2026 18:27:10 -0400 Subject: [PATCH 1/2] providers: add yfinance price source + MSCI EM/EAFE held-out markets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A third price provider (after Norgate/databento) for markets those don't cover. Registry Symbol gains an optional `yahoo` ticker; `providers/yfinance.py` fetches Yahoo OHLCV and writes the store's standard Open/High/Low/Close/Volume + Date-index shape (ETF/spot proxies have no roll, so backadj == unadj). CLI: --prices-yahoo. Research-grade (Yahoo is free/unofficial — gaps/revisions/API breakage possible). Adds MSCI Emerging Markets (MME → EEM) and MSCI EAFE (MFS → EFA): the liquid, out-of-selection equity indices needed for a clean generalization test — COT is the real ICE futures positioning, price is the ETF proxy (no roll gaps). ~15y COT + deep ETF history each. Hermetic provider tests (mock download + write_prices); golden registry tests bumped 47→49 / Equities 6→8 / checksum. Full suite 58 passed. Co-Authored-By: Claude Opus 4.8 --- src/cotdata/providers/yfinance.py | 64 +++++++++++++++++++++++++++++++ src/cotdata/registry.py | 4 ++ src/cotdata/registry.yaml | 9 +++++ src/cotdata/update.py | 14 ++++++- tests/test_registry.py | 8 ++-- tests/test_yfinance_provider.py | 54 ++++++++++++++++++++++++++ 6 files changed, 147 insertions(+), 6 deletions(-) create mode 100644 src/cotdata/providers/yfinance.py create mode 100644 tests/test_yfinance_provider.py diff --git a/src/cotdata/providers/yfinance.py b/src/cotdata/providers/yfinance.py new file mode 100644 index 0000000..76b7aff --- /dev/null +++ b/src/cotdata/providers/yfinance.py @@ -0,0 +1,64 @@ +"""yfinance price provider — free Yahoo Finance OHLCV for registry symbols that carry +a ``yahoo`` ticker. + +For markets Norgate/databento don't cover (e.g. MSCI EM / EAFE, priced via the EEM / +EFA ETF proxies). Research-grade: Yahoo is a free, unofficial feed — expect occasional +gaps, silent revisions, and API breakage; not a production replacement for Norgate. + +Writes the same Open/High/Low/Close/Volume frame with a tz-naive DatetimeIndex named +``Date`` that the store's other price providers use, so ``cotdata.get_prices`` stays +source-agnostic. ETF/spot proxies have no futures roll, so backadj == unadj (both are +written, since consumers ask for ``backadj``). +""" +from __future__ import annotations + +import pandas as pd + +from .. import store +from ..registry import all_symbols + + +def _fetch(ticker: str) -> pd.DataFrame: + import yfinance as yf + raw = yf.download(ticker, period="max", auto_adjust=True, + progress=False, threads=False) + if raw is None or raw.empty: + return pd.DataFrame() + # yfinance returns a (field, ticker) column MultiIndex even for a single symbol. + if isinstance(raw.columns, pd.MultiIndex): + raw.columns = raw.columns.get_level_values(0) + keep = [c for c in ("Open", "High", "Low", "Close", "Volume") if c in raw.columns] + df = raw[keep].copy() + df.index = pd.to_datetime(df.index).tz_localize(None) + df.index.name = "Date" + return df.dropna(subset=["Close"]).sort_index() + + +def update(symbols=None) -> dict: + """Fetch Yahoo OHLCV for registry symbols with a ``yahoo`` ticker (default: all + that have one; pass ``symbols`` to scope). Returns {kind, ok, wrote}.""" + targets = [s for s in all_symbols() + if s.yahoo and (symbols is None or s.internal in symbols)] + if not targets: + print("yfinance: no registry symbols with a 'yahoo' ticker" + + (f" among {symbols}" if symbols else "")) + return {"kind": "prices_yahoo", "ok": True, "wrote": 0} + + wrote, failed = 0, 0 + for s in targets: + try: + df = _fetch(s.yahoo) + except Exception as e: # noqa: BLE001 — yfinance/network is flaky by nature + print(f"{s.internal}: yfinance fetch failed ({s.yahoo}) — {e}") + failed += 1 + continue + if df.empty: + print(f"{s.internal}: yfinance returned no data ({s.yahoo})") + failed += 1 + continue + for adj in ("backadj", "unadj"): + store.write_prices(s.internal, adj, df, source="yahoo") + wrote += 1 + print(f"{s.internal}: {len(df):5d} bars ({s.yahoo}) " + f"{df.index.min().date()}..{df.index.max().date()} -> store") + return {"kind": "prices_yahoo", "ok": failed == 0, "wrote": wrote} diff --git a/src/cotdata/registry.py b/src/cotdata/registry.py index 5ecc154..d425822 100644 --- a/src/cotdata/registry.py +++ b/src/cotdata/registry.py @@ -31,6 +31,9 @@ class Symbol: is_equity: bool report_type: str = "disagg" # "tff" for financials, "disagg" for commodities cftc_code: Optional[str] = None + # Optional Yahoo Finance ticker for markets Norgate/databento don't cover (the + # yfinance provider prices these; e.g. "EEM"/"EFA" as ETF proxies for MSCI EM/EAFE). + yahoo: Optional[str] = None # Predecessor CFTC codes from earlier exchange/contract listings of the SAME # instrument, stitched in chronologically behind cftc_code by get_cot. Each # entry is a bare code string (scale 1.0) or a (code, scale) tuple. Kept as a @@ -107,6 +110,7 @@ def load_registry(yaml_path=None) -> Dict[str, Symbol]: report_type=attrs.get("report_type", "tff" if asset_class in ("Equities", "FX", "Rates") else "disagg"), cftc_code=attrs["cftc_code"], hist_codes=_coerce_hist_codes(attrs.get("hist_codes")), + yahoo=attrs.get("yahoo"), ) return registry diff --git a/src/cotdata/registry.yaml b/src/cotdata/registry.yaml index 4ed7646..ad55fc9 100644 --- a/src/cotdata/registry.yaml +++ b/src/cotdata/registry.yaml @@ -15,6 +15,15 @@ Equities: cftc_code: "33874A" NKD: # Nikkei 225 ($-denominated, CME) cftc_code: "240741" + # MSCI international indices — the liquid out-of-selection held-out markets. Priced + # off their ETF proxies via Yahoo (Norgate/databento don't cover them; ETFs have no + # roll gaps). COT is the real futures positioning (ICE Futures U.S.). + MME: # MSCI Emerging Markets Index (ICE) — priced via EEM + cftc_code: "244042" + yahoo: "EEM" + MFS: # MSCI EAFE Index (ICE) — priced via EFA + cftc_code: "244041" + yahoo: "EFA" Metals: GC: diff --git a/src/cotdata/update.py b/src/cotdata/update.py index 973cddf..5f6de15 100644 --- a/src/cotdata/update.py +++ b/src/cotdata/update.py @@ -14,6 +14,9 @@ def main() -> None: p = argparse.ArgumentParser(description="cotdata producer — fetch sources into the store.") p.add_argument("--prices", action="store_true", help="Update Norgate price bars (Windows).") p.add_argument("--metadata", action="store_true", help="Update Norgate contract metadata (Windows).") + p.add_argument("--prices-yahoo", action="store_true", + help="Update prices from Yahoo Finance for registry symbols with a " + "'yahoo' ticker (cross-platform; research-grade).") p.add_argument("--cot-legacy", action="store_true", help="Update CFTC COT Legacy (cross-platform).") p.add_argument("--cot-disagg", action="store_true", help="Update CFTC COT Disaggregated Futures-Only (cross-platform).") p.add_argument("--cot-tff", action="store_true", help="Update Traders in Financial Futures (TFF) COT (cross-platform).") @@ -61,8 +64,8 @@ def main() -> None: "at": _dt.datetime.utcnow().isoformat(timespec="seconds") + "Z"}) return - if not (args.prices or args.metadata or args.cot_legacy or args.cot_disagg or args.cot_tff or args.cot_all): - p.error("nothing to do — pass --check, --prices, --metadata, --cot-legacy, --cot-disagg, --cot-tff, or --cot-all") + if not (args.prices or args.metadata or args.prices_yahoo or args.cot_legacy or args.cot_disagg or args.cot_tff or args.cot_all): + p.error("nothing to do — pass --check, --prices, --prices-yahoo, --metadata, --cot-legacy, --cot-disagg, --cot-tff, or --cot-all") if args.prices or args.metadata: from .providers import norgate @@ -89,6 +92,13 @@ def main() -> None: norgate.update_metadata(symbols=args.symbols) kinds.append("metadata") + if args.prices_yahoo: + from .providers import yfinance as yprov + r = yprov.update(symbols=args.symbols) + kinds.append("prices_yahoo") + if not (r or {}).get("ok", True): + failed_kinds.append("prices_yahoo") + if args.cot_legacy or args.cot_all: from .providers import cftc r = cftc.update() diff --git a/tests/test_registry.py b/tests/test_registry.py index 5ae156b..a7fe096 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -11,7 +11,7 @@ # ── loading & basic shape ──────────────────────────────────────────────────── def test_registry_loads_all_symbols(): symbols = all_symbols() - assert len(symbols) == 47, f"Expected 47 symbols, got {len(symbols)}" + assert len(symbols) == 49, f"Expected 49 symbols, got {len(symbols)}" def test_basic_symbol_loading(): @@ -49,7 +49,7 @@ def test_hist_code_scales_normalization(): def test_by_asset_class(): equities = by_asset_class("Equities") - assert len(equities) == 6 # ES NQ YM RTY + held-out EMD, NKD + assert len(equities) == 8 # ES NQ YM RTY + held-out EMD, NKD, MME, MFS assert all(eq.is_equity for eq in equities) dairy = by_asset_class("Dairy") # new held-out class @@ -62,7 +62,7 @@ def test_by_asset_class(): # ── hashability (frozen dataclass must stay hashable, incl. scaled hist_codes) ─ def test_symbols_are_hashable(): # Would raise TypeError: unhashable type 'list' if hist_codes held a list. - assert len(set(all_symbols())) == 47 + assert len(set(all_symbols())) == 49 assert symbol("LBR") in {symbol("LBR")} hash(symbol("LBR")) # scaled hist_codes — the tricky one @@ -151,5 +151,5 @@ def test_golden_identity_checksum(): (registry.yaml carries FIXED identity only; unintended drift is a bug).""" ident = sorted((s.internal, s.cftc_code, s.asset_class) for s in all_symbols()) digest = hashlib.md5(repr(ident).encode()).hexdigest() - assert digest == "427c5d59ee4a2e5a09fd7dc6f8b5103a", ( + assert digest == "e6b0b92caa0f25f68dea62024bddf899", ( "registry identity facts changed; update the expected checksum if intended") diff --git a/tests/test_yfinance_provider.py b/tests/test_yfinance_provider.py new file mode 100644 index 0000000..9ae924e --- /dev/null +++ b/tests/test_yfinance_provider.py @@ -0,0 +1,54 @@ +"""Hermetic tests for the yfinance price provider — no network, no store writes. + +Mocks yfinance.download (so no Yahoo call) and store.write_prices (so nothing is +written), and checks the provider normalizes Yahoo's (field, ticker) MultiIndex frame +to the store's Open/High/Low/Close/Volume + DatetimeIndex('Date') shape, for both the +backadj and unadj adjustments.""" +import pandas as pd +import pytest + + +def _fake_yahoo_frame(): + # yfinance returns a (field, ticker) column MultiIndex even for one symbol. + idx = pd.to_datetime(["2020-01-02", "2020-01-03", "2020-01-06"]) + cols = pd.MultiIndex.from_product( + [["Open", "High", "Low", "Close", "Volume"], ["EEM"]]) + return pd.DataFrame([[10, 11, 9, 10.5, 1000], + [10.5, 11.5, 10, 11, 1200], + [11, 12, 10.8, 11.8, 900]], index=idx, columns=cols) + + +def test_yfinance_update_normalizes_and_writes_both_adjustments(monkeypatch): + import yfinance + from cotdata import store + from cotdata.providers import yfinance as yprov + + monkeypatch.setattr(yfinance, "download", lambda *a, **k: _fake_yahoo_frame()) + written = {} + monkeypatch.setattr(store, "write_prices", + lambda sym, adj, df, source: written.__setitem__((sym, adj), (df, source))) + + res = yprov.update(symbols=["MME"]) # MME carries yahoo="EEM" in the registry + assert res["wrote"] == 1 and res["ok"] + assert ("MME", "backadj") in written and ("MME", "unadj") in written # ETF proxy → both + + df, source = written[("MME", "backadj")] + assert source == "yahoo" + assert list(df.columns) == ["Open", "High", "Low", "Close", "Volume"] # flattened + assert df.index.name == "Date" and df.index.tz is None # tz-naive Date index + assert len(df) == 3 and df["Close"].iloc[-1] == 11.8 + + +def test_yfinance_update_skips_symbols_without_ticker(monkeypatch): + from cotdata.providers import yfinance as yprov + # GC has no yahoo ticker → nothing to do, no fetch attempted. + res = yprov.update(symbols=["GC"]) + assert res["wrote"] == 0 and res["ok"] + + +def test_yfinance_update_reports_empty_as_failure(monkeypatch): + import yfinance + from cotdata.providers import yfinance as yprov + monkeypatch.setattr(yfinance, "download", lambda *a, **k: pd.DataFrame()) + res = yprov.update(symbols=["MME"]) + assert res["wrote"] == 0 and res["ok"] is False From 408748e773c8f5e19a9fbf0c4fe257cfd255a22a Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Thu, 16 Jul 2026 22:08:25 -0400 Subject: [PATCH 2/2] test(yfinance): stub yfinance via sys.modules; add [yahoo] extra The provider tests imported the real yfinance, which isn't installed in CI (it's an optional dep). Inject a fake yfinance module into sys.modules instead, so the tests run without the package. Declare yfinance as the [yahoo] optional extra (matching norgate/databento). Co-Authored-By: Claude Opus 4.8 --- pyproject.toml | 2 ++ tests/test_yfinance_provider.py | 27 +++++++++++++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a4e1340..29abe13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,8 @@ Issues = "https://github.com/mspinola/cotdata/issues" norgate = ["norgatedata"] # Dormant provider, retained for the intraday news-failure work + settlement cross-check. databento = ["databento>=0.30"] + +yahoo = ["yfinance"] dev = ["pytest>=7"] [project.scripts] diff --git a/tests/test_yfinance_provider.py b/tests/test_yfinance_provider.py index 9ae924e..a67f870 100644 --- a/tests/test_yfinance_provider.py +++ b/tests/test_yfinance_provider.py @@ -1,11 +1,20 @@ -"""Hermetic tests for the yfinance price provider — no network, no store writes. +"""Hermetic tests for the yfinance price provider — no network, no store writes, and +no real yfinance dependency (it's an optional [yahoo] extra, absent in CI). + +Injects a stub `yfinance` module into sys.modules (the provider does `import yfinance` +lazily inside _fetch) and mocks store.write_prices, then checks the provider normalizes +Yahoo's (field, ticker) MultiIndex frame to the store's Open/High/Low/Close/Volume + +DatetimeIndex('Date') shape for both the backadj and unadj adjustments.""" +import sys +import types -Mocks yfinance.download (so no Yahoo call) and store.write_prices (so nothing is -written), and checks the provider normalizes Yahoo's (field, ticker) MultiIndex frame -to the store's Open/High/Low/Close/Volume + DatetimeIndex('Date') shape, for both the -backadj and unadj adjustments.""" import pandas as pd -import pytest + + +def _install_fake_yfinance(monkeypatch, download): + mod = types.ModuleType("yfinance") + mod.download = download + monkeypatch.setitem(sys.modules, "yfinance", mod) def _fake_yahoo_frame(): @@ -19,11 +28,10 @@ def _fake_yahoo_frame(): def test_yfinance_update_normalizes_and_writes_both_adjustments(monkeypatch): - import yfinance from cotdata import store from cotdata.providers import yfinance as yprov - monkeypatch.setattr(yfinance, "download", lambda *a, **k: _fake_yahoo_frame()) + _install_fake_yfinance(monkeypatch, lambda *a, **k: _fake_yahoo_frame()) written = {} monkeypatch.setattr(store, "write_prices", lambda sym, adj, df, source: written.__setitem__((sym, adj), (df, source))) @@ -47,8 +55,7 @@ def test_yfinance_update_skips_symbols_without_ticker(monkeypatch): def test_yfinance_update_reports_empty_as_failure(monkeypatch): - import yfinance from cotdata.providers import yfinance as yprov - monkeypatch.setattr(yfinance, "download", lambda *a, **k: pd.DataFrame()) + _install_fake_yfinance(monkeypatch, lambda *a, **k: pd.DataFrame()) res = yprov.update(symbols=["MME"]) assert res["wrote"] == 0 and res["ok"] is False