providers: yfinance price source + MSCI EM/EAFE held-out markets

pyproject.toml

@@ -36,6 +36,8 @@ Issues = "https://github.com/mspinola/cotdata/issues"
 norgate = ["norgatedata"]
 # Dormant provider, retained for the intraday news-failure work + settlement cross-check.
 databento = ["databento>=0.30"]
+
+yahoo = ["yfinance"]
 dev = ["pytest>=7"]
 
 [project.scripts]

src/cotdata/providers/yfinance.py

@@ -0,0 +1,64 @@
+"""yfinance price provider — free Yahoo Finance OHLCV for registry symbols that carry
+a ``yahoo`` ticker.
+
+For markets Norgate/databento don't cover (e.g. MSCI EM / EAFE, priced via the EEM /
+EFA ETF proxies). Research-grade: Yahoo is a free, unofficial feed — expect occasional
+gaps, silent revisions, and API breakage; not a production replacement for Norgate.
+
+Writes the same Open/High/Low/Close/Volume frame with a tz-naive DatetimeIndex named
+``Date`` that the store's other price providers use, so ``cotdata.get_prices`` stays
+source-agnostic. ETF/spot proxies have no futures roll, so backadj == unadj (both are
+written, since consumers ask for ``backadj``).
+"""
+from __future__ import annotations
+
+import pandas as pd
+
+from .. import store
+from ..registry import all_symbols
+
+
+def _fetch(ticker: str) -> pd.DataFrame:
+    import yfinance as yf
+    raw = yf.download(ticker, period="max", auto_adjust=True,
+                      progress=False, threads=False)
+    if raw is None or raw.empty:
+        return pd.DataFrame()
+    # yfinance returns a (field, ticker) column MultiIndex even for a single symbol.
+    if isinstance(raw.columns, pd.MultiIndex):
+        raw.columns = raw.columns.get_level_values(0)
+    keep = [c for c in ("Open", "High", "Low", "Close", "Volume") if c in raw.columns]
+    df = raw[keep].copy()
+    df.index = pd.to_datetime(df.index).tz_localize(None)
+    df.index.name = "Date"
+    return df.dropna(subset=["Close"]).sort_index()
+
+
+def update(symbols=None) -> dict:
+    """Fetch Yahoo OHLCV for registry symbols with a ``yahoo`` ticker (default: all
+    that have one; pass ``symbols`` to scope). Returns {kind, ok, wrote}."""
+    targets = [s for s in all_symbols()
+               if s.yahoo and (symbols is None or s.internal in symbols)]
+    if not targets:
+        print("yfinance: no registry symbols with a 'yahoo' ticker"
+              + (f" among {symbols}" if symbols else ""))
+        return {"kind": "prices_yahoo", "ok": True, "wrote": 0}
+
+    wrote, failed = 0, 0
+    for s in targets:
+        try:
+            df = _fetch(s.yahoo)
+        except Exception as e:  # noqa: BLE001 — yfinance/network is flaky by nature
+            print(f"{s.internal}: yfinance fetch failed ({s.yahoo}) — {e}")
+            failed += 1
+            continue
+        if df.empty:
+            print(f"{s.internal}: yfinance returned no data ({s.yahoo})")
+            failed += 1
+            continue
+        for adj in ("backadj", "unadj"):
+            store.write_prices(s.internal, adj, df, source="yahoo")
+        wrote += 1
+        print(f"{s.internal}: {len(df):5d} bars ({s.yahoo}) "
+              f"{df.index.min().date()}..{df.index.max().date()} -> store")
+    return {"kind": "prices_yahoo", "ok": failed == 0, "wrote": wrote}

src/cotdata/registry.py

@@ -31,6 +31,9 @@ class Symbol:
     is_equity: bool
     report_type: str = "disagg"   # "tff" for financials, "disagg" for commodities
     cftc_code: Optional[str] = None
+    # Optional Yahoo Finance ticker for markets Norgate/databento don't cover (the
+    # yfinance provider prices these; e.g. "EEM"/"EFA" as ETF proxies for MSCI EM/EAFE).
+    yahoo: Optional[str] = None
     # Predecessor CFTC codes from earlier exchange/contract listings of the SAME
     # instrument, stitched in chronologically behind cftc_code by get_cot. Each
     # entry is a bare code string (scale 1.0) or a (code, scale) tuple. Kept as a
@@ -107,6 +110,7 @@ def load_registry(yaml_path=None) -> Dict[str, Symbol]:
                 report_type=attrs.get("report_type", "tff" if asset_class in ("Equities", "FX", "Rates") else "disagg"),
                 cftc_code=attrs["cftc_code"],
                 hist_codes=_coerce_hist_codes(attrs.get("hist_codes")),
+                yahoo=attrs.get("yahoo"),
             )
     return registry
 

src/cotdata/registry.yaml

@@ -15,6 +15,15 @@ Equities:
     cftc_code: "33874A"
   NKD:                      # Nikkei 225 ($-denominated, CME)
     cftc_code: "240741"
+  # MSCI international indices — the liquid out-of-selection held-out markets. Priced
+  # off their ETF proxies via Yahoo (Norgate/databento don't cover them; ETFs have no
+  # roll gaps). COT is the real futures positioning (ICE Futures U.S.).
+  MME:                      # MSCI Emerging Markets Index (ICE) — priced via EEM
+    cftc_code: "244042"
+    yahoo: "EEM"
+  MFS:                      # MSCI EAFE Index (ICE) — priced via EFA
+    cftc_code: "244041"
+    yahoo: "EFA"
 
 Metals:
   GC:

src/cotdata/update.py

@@ -14,6 +14,9 @@ def main() -> None:
     p = argparse.ArgumentParser(description="cotdata producer — fetch sources into the store.")
     p.add_argument("--prices", action="store_true", help="Update Norgate price bars (Windows).")
     p.add_argument("--metadata", action="store_true", help="Update Norgate contract metadata (Windows).")
+    p.add_argument("--prices-yahoo", action="store_true",
+                   help="Update prices from Yahoo Finance for registry symbols with a "
+                        "'yahoo' ticker (cross-platform; research-grade).")
     p.add_argument("--cot-legacy", action="store_true", help="Update CFTC COT Legacy (cross-platform).")
     p.add_argument("--cot-disagg", action="store_true", help="Update CFTC COT Disaggregated Futures-Only (cross-platform).")
     p.add_argument("--cot-tff", action="store_true", help="Update Traders in Financial Futures (TFF) COT (cross-platform).")
@@ -61,8 +64,8 @@ def main() -> None:
                                                "at": _dt.datetime.utcnow().isoformat(timespec="seconds") + "Z"})
         return
 
-    if not (args.prices or args.metadata or args.cot_legacy or args.cot_disagg or args.cot_tff or args.cot_all):
-        p.error("nothing to do — pass --check, --prices, --metadata, --cot-legacy, --cot-disagg, --cot-tff, or --cot-all")
+    if not (args.prices or args.metadata or args.prices_yahoo or args.cot_legacy or args.cot_disagg or args.cot_tff or args.cot_all):
+        p.error("nothing to do — pass --check, --prices, --prices-yahoo, --metadata, --cot-legacy, --cot-disagg, --cot-tff, or --cot-all")
 
     if args.prices or args.metadata:
         from .providers import norgate
@@ -89,6 +92,13 @@ def main() -> None:
         norgate.update_metadata(symbols=args.symbols)
         kinds.append("metadata")
 
+    if args.prices_yahoo:
+        from .providers import yfinance as yprov
+        r = yprov.update(symbols=args.symbols)
+        kinds.append("prices_yahoo")
+        if not (r or {}).get("ok", True):
+            failed_kinds.append("prices_yahoo")
+
     if args.cot_legacy or args.cot_all:
         from .providers import cftc
         r = cftc.update()

tests/test_registry.py

@@ -11,7 +11,7 @@
 # ── loading & basic shape ────────────────────────────────────────────────────
 def test_registry_loads_all_symbols():
     symbols = all_symbols()
-    assert len(symbols) == 47, f"Expected 47 symbols, got {len(symbols)}"
+    assert len(symbols) == 49, f"Expected 49 symbols, got {len(symbols)}"
 
 
 def test_basic_symbol_loading():
@@ -49,7 +49,7 @@ def test_hist_code_scales_normalization():
 
 def test_by_asset_class():
     equities = by_asset_class("Equities")
-    assert len(equities) == 6          # ES NQ YM RTY + held-out EMD, NKD
+    assert len(equities) == 8          # ES NQ YM RTY + held-out EMD, NKD, MME, MFS
     assert all(eq.is_equity for eq in equities)
 
     dairy = by_asset_class("Dairy")    # new held-out class
@@ -62,7 +62,7 @@ def test_by_asset_class():
 # ── hashability (frozen dataclass must stay hashable, incl. scaled hist_codes) ─
 def test_symbols_are_hashable():
     # Would raise TypeError: unhashable type 'list' if hist_codes held a list.
-    assert len(set(all_symbols())) == 47
+    assert len(set(all_symbols())) == 49
     assert symbol("LBR") in {symbol("LBR")}
     hash(symbol("LBR"))  # scaled hist_codes — the tricky one
 
@@ -151,5 +151,5 @@ def test_golden_identity_checksum():
     (registry.yaml carries FIXED identity only; unintended drift is a bug)."""
     ident = sorted((s.internal, s.cftc_code, s.asset_class) for s in all_symbols())
     digest = hashlib.md5(repr(ident).encode()).hexdigest()
-    assert digest == "427c5d59ee4a2e5a09fd7dc6f8b5103a", (
+    assert digest == "e6b0b92caa0f25f68dea62024bddf899", (
         "registry identity facts changed; update the expected checksum if intended")

tests/test_yfinance_provider.py

@@ -0,0 +1,61 @@
+"""Hermetic tests for the yfinance price provider — no network, no store writes, and
+no real yfinance dependency (it's an optional [yahoo] extra, absent in CI).
+
+Injects a stub `yfinance` module into sys.modules (the provider does `import yfinance`
+lazily inside _fetch) and mocks store.write_prices, then checks the provider normalizes
+Yahoo's (field, ticker) MultiIndex frame to the store's Open/High/Low/Close/Volume +
+DatetimeIndex('Date') shape for both the backadj and unadj adjustments."""
+import sys
+import types
+
+import pandas as pd
+
+
+def _install_fake_yfinance(monkeypatch, download):
+    mod = types.ModuleType("yfinance")
+    mod.download = download
+    monkeypatch.setitem(sys.modules, "yfinance", mod)
+
+
+def _fake_yahoo_frame():
+    # yfinance returns a (field, ticker) column MultiIndex even for one symbol.
+    idx = pd.to_datetime(["2020-01-02", "2020-01-03", "2020-01-06"])
+    cols = pd.MultiIndex.from_product(
+        [["Open", "High", "Low", "Close", "Volume"], ["EEM"]])
+    return pd.DataFrame([[10, 11, 9, 10.5, 1000],
+                         [10.5, 11.5, 10, 11, 1200],
+                         [11, 12, 10.8, 11.8, 900]], index=idx, columns=cols)
+
+
+def test_yfinance_update_normalizes_and_writes_both_adjustments(monkeypatch):
+    from cotdata import store
+    from cotdata.providers import yfinance as yprov
+
+    _install_fake_yfinance(monkeypatch, lambda *a, **k: _fake_yahoo_frame())
+    written = {}
+    monkeypatch.setattr(store, "write_prices",
+                        lambda sym, adj, df, source: written.__setitem__((sym, adj), (df, source)))
+
+    res = yprov.update(symbols=["MME"])          # MME carries yahoo="EEM" in the registry
+    assert res["wrote"] == 1 and res["ok"]
+    assert ("MME", "backadj") in written and ("MME", "unadj") in written   # ETF proxy → both
+
+    df, source = written[("MME", "backadj")]
+    assert source == "yahoo"
+    assert list(df.columns) == ["Open", "High", "Low", "Close", "Volume"]  # flattened
+    assert df.index.name == "Date" and df.index.tz is None                 # tz-naive Date index
+    assert len(df) == 3 and df["Close"].iloc[-1] == 11.8
+
+
+def test_yfinance_update_skips_symbols_without_ticker(monkeypatch):
+    from cotdata.providers import yfinance as yprov
+    # GC has no yahoo ticker → nothing to do, no fetch attempted.
+    res = yprov.update(symbols=["GC"])
+    assert res["wrote"] == 0 and res["ok"]
+
+
+def test_yfinance_update_reports_empty_as_failure(monkeypatch):
+    from cotdata.providers import yfinance as yprov
+    _install_fake_yfinance(monkeypatch, lambda *a, **k: pd.DataFrame())
+    res = yprov.update(symbols=["MME"])
+    assert res["wrote"] == 0 and res["ok"] is False