fix(prices): skip Yahoo-only markets (MME/MFS) in the Norgate producer

CONTRIBUTING.md

@@ -14,29 +14,58 @@ Thank you for your interest in contributing to `cotdata`! This project provides
 
 ## Development Setup
 
-This project uses `uv` (or standard `pip` environments) for dependency management.
+This project uses `uv` for fast, reliable dependency management. Standard `pip` venv also works if preferred.
 
-1. **Clone the repository:**
+### Using `uv` (Recommended)
+
+1. **Install uv** (if needed):
+   ```bash
+   curl -LsSf https://astral.sh/uv/install.sh | sh
+   ```
+   See [uv documentation](https://docs.astral.sh/uv/) for other installation methods.
+
+2. **Clone the repository:**
    ```bash
    git clone https://github.com/your-username/cotdata.git
    cd cotdata
    ```
 
-2. **Set up a virtual environment and install dev dependencies:**
+3. **Create and activate a virtual environment:**
    ```bash
    uv venv
+   source .venv/bin/activate  # Mac/Linux
+   # OR
+   .venv\Scripts\activate     # Windows (cmd)
+   # OR
+   .venv\Scripts\Activate.ps1 # Windows (PowerShell)
+   ```
+
+4. **Install the package in development mode with dev dependencies:**
+   ```bash
    uv pip install -e ".[dev]"
    ```
-   *Note: If you are on Windows and intend to work on the Norgate integration, you should install the Norgate extras: `uv pip install -e ".[dev,norgate]"`.*
+   If you're on Windows and working on the Norgate integration:
+   ```bash
+   uv pip install -e ".[dev,norgate]"
+   ```
 
-3. **Set the temporary datastore:**
-   For local development, you should point the store to a local temporary folder to avoid overwriting your live data:
+5. **Set the temporary datastore for local testing:**
    ```bash
    export COTDATA_STORE=/tmp/cotdata_test_store  # Mac/Linux
    # OR
    $env:COTDATA_STORE = "C:\temp\cotdata_test_store"  # Windows
    ```
 
+### Using standard pip + venv
+
+If you prefer not to install `uv`:
+```bash
+python3 -m venv .venv
+source .venv/bin/activate  # Mac/Linux / Windows (bash)
+pip install -e ".[dev]"
+export COTDATA_STORE=/tmp/cotdata_test_store  # Set your test store
+```
+
 ## Running Tests
 
 We use `pytest` for all unit and integration tests.

README.md

@@ -26,7 +26,7 @@ cotdata separates *fetching* data (a "producer" that talks to vendors) from *usi
 
 ## Contents
 
-- [Quickstart](#quickstart) · [How it works](#how-it-works) · [Reading data](#reading-data-consumer) · [Producing data](#producing-data-producer) · [Scheduling on Windows](#scheduling-on-windows-task-scheduler) · [Operations](#operations) · [Concepts & design](#concepts--design) · [Reference: schemas](#reference-data-schemas) · [Reference: COT formats](#reference-cot-formats-explained) · [Diagnostics](#diagnostics) · [Contributing](#contributing) · [License](#license)
+- [Quickstart](#quickstart) · [How it works](#how-it-works) · [Reading data](#reading-data-consumer) · [Producing data](#producing-data-producer) · [Scheduling on Windows](#scheduling-on-windows-task-scheduler) · [Operations](#operations) · [Concepts & design](#concepts--design) · [Reference: schemas](#reference-data-schemas) · [Reference: COT formats](#reference-cot-formats-explained) · [Diagnostics](#diagnostics) · [Development](#development) · [Contributing](#contributing) · [License](#license)
 
 ## Quickstart
 
@@ -417,6 +417,14 @@ The flow runs one direction: **`cotdata` (data) → your signal → `crucible`
 (edge)**. Neither imports the other, so cotdata stays useful on its own for any
 COT/futures research — crucible is just the most common thing to point at it next.
 
+## Development
+
+Want to contribute or work on cotdata locally? See [CONTRIBUTING.md](CONTRIBUTING.md) for:
+- Virtual environment setup with `uv` or standard `pip`
+- Running the test suite
+- Platform-specific notes (Norgate is Windows-only; CFTC parsing runs anywhere)
+- Code style guidelines
+
 ## Contributing
 
 Issues and pull requests are welcome. Please see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, tests, and conventions. When filing a bug, include your OS — Norgate features require Windows, while store reads and CFTC COT run anywhere.

src/cotdata/providers/norgate.py

@@ -275,6 +275,23 @@ def finals_ready(cutoff: str = DEFAULT_FINAL_CUTOFF, now=None):
     return _finals_ready(times, cutoff, now)
 
 
+def _norgate_covered(symbols):
+    """Resolve requested internal symbols to those Norgate actually carries.
+
+    Yahoo-only markets (registry `norgate: null` — e.g. the MSCI MME/MFS indices
+    priced off ETF proxies) have no `&SYM_CCB` continuous series. Fetching them
+    errors on every field and, for metadata, silently writes null-filled rows, so
+    drop them here (with a note) rather than hitting Norgate for a symbol it can't
+    serve. The yfinance provider prices these instead."""
+    requested = symbols or [s.internal for s in all_symbols()]
+    covered = [s for s in requested if REGISTRY[s].norgate]
+    skipped = [s for s in requested if not REGISTRY[s].norgate]
+    if skipped:
+        print(f"  skipping {len(skipped)} symbol(s) with no Norgate coverage "
+              f"(priced elsewhere): {', '.join(skipped)}")
+    return covered
+
+
 def update(symbols=None, full: bool = False) -> None:
     """Fetch + write to the store for the given internal symbols (backadj and unadj).
 
@@ -285,7 +302,7 @@ def update(symbols=None, full: bool = False) -> None:
     import time
     from .. import status
 
-    syms = symbols or [s.internal for s in all_symbols()]
+    syms = _norgate_covered(symbols)
     prior = store.load_manifest().get("prices", {})  # to report per-symbol date deltas
     t0 = time.time()
     ok, failed, total_rows, newest = [], [], 0, None
@@ -384,7 +401,7 @@ def update_metadata(symbols=None) -> None:
     """
     import concurrent.futures
     scoped = symbols is not None
-    syms = symbols or [s.internal for s in all_symbols()]
+    syms = _norgate_covered(symbols)
 
     print(f"Fetching metadata for {len(syms)} symbols...")
     metadata_rows = []

src/cotdata/registry.py

@@ -26,7 +26,9 @@
 @dataclass(frozen=True)
 class Symbol:
     internal: str                 # pipeline root, e.g. "ES"
-    norgate: str                  # Norgate continuous symbol, e.g. "&ES"
+    norgate: Optional[str]        # Norgate continuous symbol, e.g. "&ES"; None when
+                                  # Norgate has no series (norgate: null in YAML) —
+                                  # the Norgate producer skips these (priced elsewhere)
     asset_class: str
     is_equity: bool
     report_type: str = "disagg"   # "tff" for financials, "disagg" for commodities

src/cotdata/registry.yaml

@@ -20,10 +20,12 @@ Equities:
   # roll gaps). COT is the real futures positioning (ICE Futures U.S.).
   MME:                      # MSCI Emerging Markets Index (ICE) — priced via EEM
     cftc_code: "244042"
+    norgate: null           # no Norgate continuous series — priced off the ETF proxy
     yahoo: "EEM"
   MFS:                      # MSCI EAFE Index (ICE) — priced via EFA
     cftc_code: "244041"
-    yahoo: "EFA"
+    norgate: null           # (Norgate doesn't carry MSCI intl indices) — skip in the
+    yahoo: "EFA"            #  Norgate producer; the yfinance provider prices these
 
 Metals:
   GC:

tests/test_norgate_provider.py

@@ -338,6 +338,41 @@ def test_full_update_metadata_replaces_table(tmp_path, monkeypatch):
     assert set(store.read_metadata()["Symbol"]) == {"ES", "NQ"}  # OLD gone
 
 
+def test_metadata_skips_symbols_without_norgate_coverage(tmp_path, monkeypatch):
+    """Yahoo-only markets (registry norgate=None, e.g. MME/MFS) must be skipped by
+    the Norgate metadata producer — never fetched, never written as null rows. The
+    regression: `&MME_CCB not found` spam + all-null spec rows in contract_specs."""
+    monkeypatch.setenv("COTDATA_STORE", str(tmp_path))
+    from cotdata import store
+
+    sym_es = mock.Mock(internal="ES", norgate="&ES")
+    sym_mme = mock.Mock(internal="MME", norgate=None)   # no Norgate coverage
+    called = []
+
+    def fake_meta(sym):
+        called.append(sym)
+        return {"Symbol": sym, "Tick Size": 1.0}
+
+    with mock.patch("cotdata.providers.norgate.all_symbols",
+                    return_value=[sym_es, sym_mme]), \
+         mock.patch.dict("cotdata.providers.norgate.REGISTRY",
+                         {"ES": sym_es, "MME": sym_mme}), \
+         mock.patch("cotdata.providers.norgate.get_symbol_metadata",
+                    side_effect=fake_meta):
+        norgate.update_metadata()  # full run
+
+    assert called == ["ES"]                                   # MME never fetched
+    assert set(store.read_metadata()["Symbol"]) == {"ES"}     # no null MME row
+
+
+def test_covered_filter_drops_none_norgate():
+    """Unit: _norgate_covered keeps only symbols whose registry norgate is truthy."""
+    with mock.patch.dict("cotdata.providers.norgate.REGISTRY",
+                         {"ES": mock.Mock(norgate="&ES"),
+                          "MME": mock.Mock(norgate=None)}):
+        assert norgate._norgate_covered(["ES", "MME"]) == ["ES"]
+
+
 import datetime as _dt
 def test_finals_ready_pure_logic():
     from cotdata.providers.norgate import _finals_ready

tests/test_registry.py

@@ -24,6 +24,18 @@ def test_basic_symbol_loading():
     assert es.hist_codes == ()
 
 
+def test_yahoo_only_market_has_no_norgate_symbol():
+    """MSCI MME/MFS are priced off ETF proxies (EEM/EFA); Norgate carries no series
+    for them, so norgate is None (registry `norgate: null`) — that's the signal the
+    Norgate producer filters on. A defaulted '&MME' would send it fetching a
+    nonexistent &MME_CCB."""
+    for s in ("MME", "MFS"):
+        assert symbol(s).norgate is None, s
+        assert symbol(s).yahoo in ("EEM", "EFA")
+    # Norgate-covered markets still default to '&<internal>'.
+    assert symbol("ES").norgate == "&ES"
+
+
 def test_symbol_with_simple_hist_codes():
     rty = symbol("RTY")
     assert rty.asset_class == "Equities"