From 741b5097e1241e0a793777286b5011c9e66dfbfe Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Mon, 1 Dec 2025 10:43:39 -0800 Subject: [PATCH 01/26] move quality gate thresholds to configs/evaluation.json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New configs/evaluation.json with all gate thresholds - EvaluateDetectorFlow now uses eval_config instead of Parameters - Increased max_rate_diff from 10% to 25% (crypto volatility) - Remove hardcoded defaults from src/eval.py in favor of config 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- configs/evaluation.json | 7 +++++++ flows/evaluate/flow.py | 21 +++++++-------------- src/eval.py | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) create mode 100644 configs/evaluation.json diff --git a/configs/evaluation.json b/configs/evaluation.json new file mode 100644 index 0000000..826bebf --- /dev/null +++ b/configs/evaluation.json @@ -0,0 +1,7 @@ +{ + "max_anomaly_rate": 0.20, + "min_anomaly_rate": 0.02, + "max_rate_diff": 0.25, + "min_silhouette": 0.0, + "min_score_gap": 0.05 +} diff --git a/flows/evaluate/flow.py b/flows/evaluate/flow.py index 69341d1..b9d6956 100644 --- a/flows/evaluate/flow.py +++ b/flows/evaluate/flow.py @@ -29,19 +29,9 @@ class EvaluateDetectorFlow(ProjectFlow): Fetches new data (not from asset) to test model generalization. """ - # Centralized config - num_coins comes from training_config.num_coins + # Centralized configs training_config = Config("training_config", default="configs/training.json") - - max_anomaly_rate = Parameter( - "max_anomaly_rate", - default=0.20, - help="Maximum acceptable anomaly rate" - ) - min_anomaly_rate = Parameter( - "min_anomaly_rate", - default=0.02, - help="Minimum acceptable anomaly rate" - ) + eval_config = Config("eval_config", default="configs/evaluation.json") @step def start(self): @@ -134,8 +124,11 @@ def evaluate(self): self.eval_result = evaluation.run_quality_gates( self.prediction, training_anomaly_rate=training_rate, - max_anomaly_rate=self.max_anomaly_rate, - min_anomaly_rate=self.min_anomaly_rate, + max_anomaly_rate=self.eval_config.get("max_anomaly_rate", 0.20), + min_anomaly_rate=self.eval_config.get("min_anomaly_rate", 0.02), + max_rate_diff=self.eval_config.get("max_rate_diff", 0.25), + min_silhouette=self.eval_config.get("min_silhouette", 0.0), + min_score_gap=self.eval_config.get("min_score_gap", 0.05), features=self.feature_set.features, # Enable silhouette score ) diff --git a/src/eval.py b/src/eval.py index ecab01b..de98197 100644 --- a/src/eval.py +++ b/src/eval.py @@ -177,7 +177,7 @@ def run_quality_gates( training_anomaly_rate: float, max_anomaly_rate: float = 0.20, min_anomaly_rate: float = 0.02, - max_rate_diff: float = 0.10, + max_rate_diff: float = 0.25, min_silhouette: float = 0.0, min_score_gap: float = 0.05, features: Optional[List[List[float]]] = None, From 704adb2d13f66fc336b242069626121317ebc572 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Mon, 1 Dec 2025 10:43:47 -0800 Subject: [PATCH 02/26] add ds-strategist agent for data science critique MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent that: - Critiques the business value of current ML task - Identifies gaps in evaluation methodology - Proposes evolution paths toward practical value - Provides implementation guidance for improvements Uses opus model for deeper analysis and web search for industry context. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .claude/agents/ds-strategist.md | 150 ++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 .claude/agents/ds-strategist.md diff --git a/.claude/agents/ds-strategist.md b/.claude/agents/ds-strategist.md new file mode 100644 index 0000000..4be35cd --- /dev/null +++ b/.claude/agents/ds-strategist.md @@ -0,0 +1,150 @@ +--- +name: ds-strategist +description: Critiques the data science approach and proposes evolutions toward practical business value +tools: Read, Edit, Write, Bash, Glob, Grep, WebSearch, WebFetch +model: opus +--- + +You are a senior data science strategist who critiques ML reference architectures and proposes evolutions that increase practical business value. Your role is to challenge assumptions, identify gaps, and suggest concrete improvements. + +## Context: What This Project Is + +This is a **reference implementation** showing how to build production ML systems with Metaflow + Outerbounds. The current example is crypto anomaly detection, but the architecture patterns matter more than the specific use case. + +### Current Architecture (What's Been Built) + +``` +Data Pipeline: +IngestMarketDataFlow → BuildDatasetFlow → TrainDetectorFlow → EvaluateDetectorFlow → PromoteDetectorFlow + (hourly) (accumulates) (trains) (quality gates) (champion) + +Model Lifecycle: +candidate → evaluated → champion + ↑ ↑ ↑ + Training QA Gates Human/Auto +``` + +### Key Files to Understand + +| File | What It Does | +|------|--------------| +| `src/data.py` | Data fetching from CoinGecko, feature engineering | +| `src/model.py` | Isolation Forest / LOF anomaly detection | +| `src/eval.py` | Quality gates (rate bounds, silhouette, score gap) | +| `src/registry.py` | Model versioning, status management | +| `flows/*/flow.py` | Metaflow orchestration for each lifecycle stage | +| `deployments/dashboard/` | FastAPI dashboard showing model registry state | + +### What the Reference Architecture Demonstrates + +1. **Model Lifecycle Management** - Create, version, evaluate, promote models +2. **Data Lineage** - Track which dataset trained which model +3. **Quality Gates** - Automated checks before promotion +4. **Human-in-the-Loop** - Optional approval workflows +5. **Observability** - Dashboard showing registry state, Metaflow cards for viz + +## Your Task: Critique and Evolve + +When invoked, you should: + +### 1. Critique the Current Data Science Task + +The current task is **crypto anomaly detection** using unsupervised learning. Be honest: + +- **What's the actual business value?** Who would pay for this? What decisions does it enable? +- **Is the evaluation meaningful?** Without ground truth, are the quality gates actually measuring anything useful? +- **Does the model generalize?** Crypto markets are notoriously non-stationary. Does training on historical data predict future anomalies? +- **What's the feedback loop?** How would we know if the model is working in production? + +### 2. Propose Evolution Paths + +Suggest 2-3 concrete evolutions that would make this reference architecture more valuable to real companies. Consider: + +**Domain pivots** (keep the architecture, change the problem): +- Fraud detection (has ground truth labels eventually) +- Infrastructure monitoring (anomaly → incident correlation) +- Financial transaction monitoring (regulatory requirements) +- Manufacturing quality control (sensor data, defect labels) + +**Architecture evolutions** (keep the domain, add capabilities): +- A/B testing framework for challenger models +- Drift detection triggering automatic retraining +- Feature store integration +- Online learning / incremental updates +- Explainability (why is this point anomalous?) + +**Evaluation improvements**: +- Synthetic anomaly injection for testing +- Backtesting framework with historical "known bad" events +- Business metric correlation (did flagged anomalies lead to real events?) + +### 3. Provide Implementation Guidance + +For your top recommendation, provide: +- Which files need to change +- What new flows/modules to add +- Estimated complexity (small/medium/large) +- How it demonstrates additional reference architecture value + +## How to Work + +1. **Read the codebase first** - Use Glob/Grep/Read to understand what exists +2. **Search for context** - Use WebSearch to find industry examples, papers, or benchmarks +3. **Be specific** - Reference actual files, functions, and code patterns +4. **Be honest** - If something is demo-ware with limited practical value, say so +5. **Be constructive** - Critique should lead to actionable improvements + +## What Previous Contributors Have Built + +The conversation history shows iterative improvements: + +1. **Flow refactoring** - Renamed flows from `*AnomalyFlow` to `*DetectorFlow` +2. **Card visualizations** - Added `src/cards.py` with reusable Metaflow card components +3. **Dashboard improvements** - Card embedding, timestamp formatting, error handling +4. **Playwright testing** - Automated UI validation in `scripts/validate_dashboard.py` +5. **Claude agents** - `ui-improver` and `lifecycle-runner` for iterative development + +Build on this foundation. Don't reinvent what's already working. + +## Output Format + +Structure your response as: + +```markdown +## Critique: Current State + +### Business Value Assessment +[Honest assessment of who would use this and why] + +### Technical Gaps +[What's missing or broken] + +### Evaluation Quality +[Are we actually measuring anything meaningful?] + +## Recommended Evolution: [Name] + +### Why This Evolution +[Business case] + +### Implementation Plan +[Specific files, flows, changes] + +### Complexity & Dependencies +[What it takes to build] + +### Reference Architecture Value +[What new patterns does this demonstrate?] + +## Alternative Evolutions + +### Option B: [Name] +[Brief description] + +### Option C: [Name] +[Brief description] +``` + +## Remember + +You are not here to praise the existing work. You are here to make it better by identifying what's weak and proposing concrete improvements. The goal is a reference architecture that companies actually want to use as a starting point for their ML platforms. From 020e769f744df9b6fcb85b476da49d70cc7ac907 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Mon, 1 Dec 2025 10:59:59 -0800 Subject: [PATCH 03/26] fix timestamp validation to check visible text only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raw timestamps in title="" attributes are intentional for hover tooltips. The check now uses inner_text() to only validate visible text, not HTML attributes. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- scripts/validate_dashboard.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/validate_dashboard.py b/scripts/validate_dashboard.py index 887b7e9..95a2379 100644 --- a/scripts/validate_dashboard.py +++ b/scripts/validate_dashboard.py @@ -211,10 +211,12 @@ async def validate_data_page(self) -> ValidationResult: checks["shows_samples"] = "sample" in page_content.lower() or "100" in page_content print(f" Sample counts shown: {'PASS' if checks['shows_samples'] else 'FAIL'}") - # Check timestamps are formatted (not raw ISO) - # Good: "2025-12-01 05:41:00", Bad: "2025-12-01T05:41:00.065898+00:00" - raw_timestamp = "+00:00" in page_content and "T" in page_content - checks["timestamps_formatted"] = not raw_timestamp or "format_timestamp" in page_content + # Check timestamps are formatted (not raw ISO) in visible text + # Raw timestamps in title="" attributes are OK (for hover tooltips) + # Good: "2025-12-01 05:41:00", Bad: visible "2025-12-01T05:41:00.065898+00:00" + visible_text = await self.page.inner_text("body") + raw_timestamp_visible = "+00:00" in visible_text and "T" in visible_text + checks["timestamps_formatted"] = not raw_timestamp_visible print(f" Timestamps formatted: {'PASS' if checks['timestamps_formatted'] else 'FAIL'}") # Check for feature schema section From 7f799d007e13ad641643c3703ece5266eae20424 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Mon, 1 Dec 2025 12:01:12 -0800 Subject: [PATCH 04/26] add temporal features and prediction logging for crash prediction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Data Engineering: - src/features.py: Temporal feature engineering (lag, rolling stats) - src/predictions.py: Prediction logging for outcome validation - BuildDatasetFlow: No longer deduplicates! Keeps all temporal data - Adds lag features (1h, 6h, 24h) - Adds rolling statistics (mean, std over 6h, 24h) - Time-based train/holdout split Prediction Logging: - TrainFlow now logs all predictions with timestamps - Stored in predictions/dt=YYYY-MM-DD/hour=HH/ - Enables future outcome validation (did flagged coins actually crash?) This transforms the model from "static anomaly detector" to "crash early warning system" that can be validated against actual price movements. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- flows/build_dataset/flow.py | 297 +++++++++++++++++++--------------- flows/train/flow.py | 30 +++- src/__init__.py | 2 +- src/features.py | 314 ++++++++++++++++++++++++++++++++++++ src/predictions.py | 307 +++++++++++++++++++++++++++++++++++ 5 files changed, 817 insertions(+), 133 deletions(-) create mode 100644 src/features.py create mode 100644 src/predictions.py diff --git a/flows/build_dataset/flow.py b/flows/build_dataset/flow.py index f8ab2e5..9755b26 100644 --- a/flows/build_dataset/flow.py +++ b/flows/build_dataset/flow.py @@ -1,37 +1,48 @@ """ Build training and evaluation datasets from accumulated snapshots. -This flow implements the "fast bakery" pattern: -1. Lists N recent market snapshot paths from Parquet storage -2. Splits paths by time (train = older, holdout = recent) -3. Loads, deduplicates, and writes each dataset in a single step -4. Registers as DataAssets for downstream consumption -5. Publishes 'dataset_ready' event to trigger TrainFlow - -Dataset Split Strategy: ------------------------ -For temporal data, we split by snapshot paths (not loaded data) to prevent -memory issues with large datasets: - - |<------- Training Paths ------->|<-- Holdout Paths -->| - | snapshots 1 to N-holdout_n | last holdout_n | - | (older data) | (recent data) | - -This ensures: -- No data leakage (eval uses "future" data) -- Memory efficient (load/write each split independently) -- Scalable (never holds full dataset in memory) +This flow implements temporal feature engineering for crash prediction: +1. Lists recent market snapshot paths from Parquet storage +2. Loads ALL snapshots (no deduplication - we need temporal history) +3. Adds lag features and rolling statistics per coin +4. Splits by TIME (train on past, evaluate on "future") +5. Registers as DataAssets for downstream consumption + +Dataset Strategy: +----------------- +Unlike typical ML where we deduplicate, we KEEP all temporal observations: + + Snapshot t=1: [BTC, ETH, ...] 100 coins + Snapshot t=2: [BTC, ETH, ...] 100 coins + ... + Snapshot t=24: [BTC, ETH, ...] 100 coins + + Total: 24 × 100 = 2,400 rows (not deduplicated!) + +Each row gets lag features computed from prior snapshots of the same coin: + - price_change_pct_24h_lag_1h (what was this 1 hour ago?) + - price_change_pct_24h_lag_6h + - price_change_pct_24h_rolling_mean_24h + +This enables the model to learn TEMPORAL patterns, not just static anomalies. + +Time Split: +----------- + |<------- Training (older) ------->|<-- Holdout (recent) -->| + + Training: Learn patterns from historical data + Holdout: Test if model predicts on "future" data it hasn't seen Triggering: - @schedule(daily=True) - runs daily to build fresh datasets - Publishes 'dataset_ready' event for TrainFlow Usage: - # Use last 24 snapshots, hold out last 3 for eval - python flow.py run --n_snapshots 24 --holdout_snapshots 3 + # Use last 168 hours (7 days), hold out last 24 hours + python flow.py run --max_history_hours 168 --holdout_hours 24 - # Quick test with fewer snapshots - python flow.py run --n_snapshots 5 --holdout_snapshots 1 + # Quick test with less data + python flow.py run --max_history_hours 48 --holdout_hours 6 """ from metaflow import step, card, Parameter, schedule @@ -47,82 +58,77 @@ class BuildDatasetFlow(ProjectFlow): """ Build training and evaluation datasets from accumulated snapshots. - Memory-efficient: splits by paths, loads each split independently, - never holds full dataset in Metaflow artifacts. + Preserves ALL temporal observations and adds lag/rolling features + for crash prediction. """ - n_snapshots = Parameter( - "n_snapshots", + max_history_hours = Parameter( + "max_history_hours", + default=168, # 7 days + type=int, + help="Maximum hours of history to include (rolling window)" + ) + + holdout_hours = Parameter( + "holdout_hours", default=24, type=int, - help="Number of recent snapshots to include in dataset" + help="Hours of most recent data to hold out for evaluation" ) - holdout_snapshots = Parameter( - "holdout_snapshots", - default=3, + min_snapshots_per_coin = Parameter( + "min_snapshots_per_coin", + default=6, type=int, - help="Number of most recent snapshots to hold out for evaluation" + help="Minimum snapshots needed per coin to compute lag features" ) - deduplicate = Parameter( - "deduplicate", - default=True, + add_targets = Parameter( + "add_targets", + default=False, type=bool, - help="Deduplicate by coin_id within each snapshot window" + help="Add future crash targets (for evaluation/backtesting only)" ) @step def start(self): - """List snapshots and split paths by time.""" + """Load all snapshots and prepare for temporal processing.""" from src.storage import SnapshotStore print(f"Project: {self.prj.project}, Branch: {self.prj.branch}") - print(f"\nBuilding dataset from {self.n_snapshots} snapshots") - print(f"Holding out {self.holdout_snapshots} for evaluation") + print(f"\nBuilding dataset with temporal features") + print(f" Max history: {self.max_history_hours} hours") + print(f" Holdout: {self.holdout_hours} hours") + print(f" Min snapshots per coin: {self.min_snapshots_per_coin}") - # List available snapshots (newest first) + # List all available snapshots store = SnapshotStore(self.prj.project, self.prj.branch) all_paths = store.list_snapshots() print(f"\nFound {len(all_paths)} snapshots in storage") - if len(all_paths) < self.n_snapshots: - print(f"[WARN] Only {len(all_paths)} snapshots available, using all") - snapshot_paths = all_paths - else: - snapshot_paths = all_paths[:self.n_snapshots] - - if len(snapshot_paths) < self.holdout_snapshots + 1: + if len(all_paths) < self.min_snapshots_per_coin + 1: raise ValueError( - f"Need at least {self.holdout_snapshots + 1} snapshots " - f"(got {len(snapshot_paths)})" + f"Need at least {self.min_snapshots_per_coin + 1} snapshots " + f"(got {len(all_paths)}). Run IngestFlow more times first." ) - # Split by PATHS, not by loaded data - # Paths are sorted newest-first, so holdout = first N paths - self.holdout_paths = snapshot_paths[:self.holdout_snapshots] - self.train_paths = snapshot_paths[self.holdout_snapshots:] - - print(f"\nPath split:") - print(f" Training: {len(self.train_paths)} snapshots (older)") - print(f" Holdout: {len(self.holdout_paths)} snapshots (recent)") - - for i, p in enumerate(self.train_paths[:2]): - print(f" Train[{i}]: {p.split('/')[-1]}") - if len(self.train_paths) > 2: - print(f" ... and {len(self.train_paths) - 2} more") + # We'll load ALL snapshots and filter by timestamp later + # This is more accurate than filtering by path count + self.snapshot_paths = all_paths + print(f"Will process {len(self.snapshot_paths)} snapshots") self.next(self.build_and_write) @card @step def build_and_write(self): - """Load, deduplicate, and write datasets - single step, no artifacts.""" + """Load snapshots, add temporal features, split by time, write datasets.""" from src.storage import SnapshotStore, DatasetStore, get_datastore_root - from src.data import FEATURE_COLS + from src.features import DatasetConfig, build_ml_dataset, get_feature_columns from metaflow import current from metaflow.cards import Markdown, Table - import pyarrow as pa + from datetime import timedelta + import pandas as pd snapshot_store = SnapshotStore(self.prj.project, self.prj.branch) dataset_store = DatasetStore(self.prj.project, self.prj.branch) @@ -130,101 +136,127 @@ def build_and_write(self): root, provider = get_datastore_root() print(f"Storage: {provider} ({root})") - # --- Process Training Dataset --- - print(f"\nProcessing training dataset ({len(self.train_paths)} snapshots)...") - train_table = snapshot_store.load_snapshots(self.train_paths) - print(f" Loaded {train_table.num_rows} rows") + # --- Load ALL snapshots --- + print(f"\nLoading {len(self.snapshot_paths)} snapshots...") + raw_table = snapshot_store.load_snapshots(self.snapshot_paths) + print(f" Loaded {raw_table.num_rows} raw rows (NOT deduplicated)") + + # --- Configure feature engineering --- + config = DatasetConfig( + max_history_hours=self.max_history_hours, + min_snapshots_per_coin=self.min_snapshots_per_coin, + lag_hours=[1, 6, 24], + rolling_windows_hours=[6, 24], + ) - if self.deduplicate: - train_table = self._deduplicate_table(train_table) - print(f" After dedup: {train_table.num_rows} rows") + # --- Build ML dataset with temporal features --- + print(f"\nAdding temporal features...") + ml_table, feature_cols = build_ml_dataset( + raw_table, + config=config, + include_targets=self.add_targets, + ) + print(f" After feature engineering: {ml_table.num_rows} rows") + print(f" Feature columns: {len(feature_cols)}") - train_ts = train_table.column('snapshot_timestamp').to_pylist() - train_ts_range = (min(train_ts), max(train_ts)) if train_ts else ('', '') + # --- Split by TIME (not by path count) --- + df = ml_table.to_pandas() + df['snapshot_timestamp'] = pd.to_datetime(df['snapshot_timestamp']) + + max_ts = df['snapshot_timestamp'].max() + holdout_cutoff = max_ts - timedelta(hours=self.holdout_hours) + + train_df = df[df['snapshot_timestamp'] < holdout_cutoff] + holdout_df = df[df['snapshot_timestamp'] >= holdout_cutoff] + + print(f"\nTime-based split:") + print(f" Training: {len(train_df)} rows (before {holdout_cutoff})") + print(f" Holdout: {len(holdout_df)} rows (after {holdout_cutoff})") + + # --- Write Training Dataset --- + import pyarrow as pa + train_table = pa.Table.from_pandas(train_df, preserve_index=False) + train_ts = train_df['snapshot_timestamp'] + train_ts_range = (str(train_ts.min()), str(train_ts.max())) if len(train_ts) > 0 else ('', '') + + # Count unique snapshots (not rows) + train_snapshot_count = train_df['snapshot_timestamp'].nunique() self.train_meta = dataset_store.write_dataset( name="training_dataset", table=train_table, version=current.run_id, metadata={ - 'snapshot_count': len(self.train_paths), + 'snapshot_count': train_snapshot_count, 'snapshot_range': train_ts_range, - 'n_features': len(FEATURE_COLS), - 'feature_names': FEATURE_COLS, + 'n_features': len(feature_cols), + 'feature_names': feature_cols, 'builder_flow': current.flow_name, 'builder_run_id': current.run_id, + 'temporal_features': True, + 'deduplicated': False, # Important: we keep all rows! } ) print(f" Wrote training_dataset: {self.train_meta.total_samples} samples") - # Free memory before loading holdout - del train_table - - # --- Process Holdout Dataset --- - print(f"\nProcessing holdout dataset ({len(self.holdout_paths)} snapshots)...") - holdout_table = snapshot_store.load_snapshots(self.holdout_paths) - print(f" Loaded {holdout_table.num_rows} rows") - - if self.deduplicate: - holdout_table = self._deduplicate_table(holdout_table) - print(f" After dedup: {holdout_table.num_rows} rows") + del train_table, train_df - holdout_ts = holdout_table.column('snapshot_timestamp').to_pylist() - holdout_ts_range = (min(holdout_ts), max(holdout_ts)) if holdout_ts else ('', '') + # --- Write Holdout Dataset --- + holdout_table = pa.Table.from_pandas(holdout_df, preserve_index=False) + holdout_ts = holdout_df['snapshot_timestamp'] + holdout_ts_range = (str(holdout_ts.min()), str(holdout_ts.max())) if len(holdout_ts) > 0 else ('', '') + holdout_snapshot_count = holdout_df['snapshot_timestamp'].nunique() self.holdout_meta = dataset_store.write_dataset( name="eval_holdout", table=holdout_table, version=current.run_id, metadata={ - 'snapshot_count': len(self.holdout_paths), + 'snapshot_count': holdout_snapshot_count, 'snapshot_range': holdout_ts_range, - 'n_features': len(FEATURE_COLS), - 'feature_names': FEATURE_COLS, + 'n_features': len(feature_cols), + 'feature_names': feature_cols, 'builder_flow': current.flow_name, 'builder_run_id': current.run_id, + 'temporal_features': True, + 'deduplicated': False, } ) print(f" Wrote eval_holdout: {self.holdout_meta.total_samples} samples") - del holdout_table + # Store feature cols for downstream + self.feature_cols = feature_cols + + del holdout_table, holdout_df # Build card - current.card.append(Markdown("# Dataset Builder")) + current.card.append(Markdown("# Dataset Builder (Temporal)")) current.card.append(Markdown(f"**Run ID:** {current.run_id}")) current.card.append(Markdown(f"**Storage:** {provider}")) + current.card.append(Markdown(f"**Temporal Features:** Yes (lag + rolling)")) current.card.append(Markdown("## Training Dataset")) current.card.append(Table([ - ["Samples", str(self.train_meta.total_samples)], - ["Snapshots", str(self.train_meta.snapshot_count)], - ["From", self.train_meta.snapshot_range[0][:19] if self.train_meta.snapshot_range[0] else ""], - ["To", self.train_meta.snapshot_range[1][:19] if self.train_meta.snapshot_range[1] else ""], - ["Features", str(self.train_meta.n_features)], + ["Rows (observations)", str(self.train_meta.total_samples)], + ["Unique Snapshots", str(self.train_meta.snapshot_count)], + ["From", train_ts_range[0][:19] if train_ts_range[0] else ""], + ["To", train_ts_range[1][:19] if train_ts_range[1] else ""], + ["Features", str(len(feature_cols))], + ["Deduplicated", "No (kept all temporal data)"], ], headers=["Property", "Value"])) current.card.append(Markdown("## Evaluation Holdout")) current.card.append(Table([ - ["Samples", str(self.holdout_meta.total_samples)], - ["Snapshots", str(self.holdout_meta.snapshot_count)], - ["From", self.holdout_meta.snapshot_range[0][:19] if self.holdout_meta.snapshot_range[0] else ""], - ["To", self.holdout_meta.snapshot_range[1][:19] if self.holdout_meta.snapshot_range[1] else ""], + ["Rows (observations)", str(self.holdout_meta.total_samples)], + ["Unique Snapshots", str(self.holdout_meta.snapshot_count)], + ["From", holdout_ts_range[0][:19] if holdout_ts_range[0] else ""], + ["To", holdout_ts_range[1][:19] if holdout_ts_range[1] else ""], ], headers=["Property", "Value"])) - self.next(self.register) - - def _deduplicate_table(self, table): - """Deduplicate by coin_id, keeping most recent entry.""" - import pyarrow as pa + current.card.append(Markdown("## Feature Columns")) + current.card.append(Markdown(f"```\n{chr(10).join(feature_cols[:20])}\n{'...' if len(feature_cols) > 20 else ''}\n```")) - if table.num_rows == 0: - return table - - df = table.to_pandas() - df = df.sort_values('snapshot_timestamp', ascending=False) - df = df.drop_duplicates(subset=['coin_id'], keep='first') - - return pa.Table.from_pandas(df, preserve_index=False) + self.next(self.register) @step def register(self): @@ -241,14 +273,15 @@ def register(self): "snapshot_count": self.train_meta.snapshot_count, "snapshot_range_start": self.train_meta.snapshot_range[0], "snapshot_range_end": self.train_meta.snapshot_range[1], - "n_features": self.train_meta.n_features, - "feature_names": ",".join(self.train_meta.feature_names), + "n_features": len(self.feature_cols), + "feature_names": ",".join(self.feature_cols[:20]), # First 20 to avoid overflow "builder_flow": current.flow_name, "builder_run_id": current.run_id, - "deduplicated": str(self.deduplicate), + "temporal_features": "True", + "deduplicated": "False", }, - tags={"dataset_type": "training"}, - description=f"Training: {self.train_meta.total_samples} samples from {self.train_meta.snapshot_count} snapshots" + tags={"dataset_type": "training", "has_temporal_features": "true"}, + description=f"Training: {self.train_meta.total_samples} rows from {self.train_meta.snapshot_count} snapshots (temporal features)" ) # Register holdout dataset @@ -261,19 +294,22 @@ def register(self): "snapshot_count": self.holdout_meta.snapshot_count, "snapshot_range_start": self.holdout_meta.snapshot_range[0], "snapshot_range_end": self.holdout_meta.snapshot_range[1], - "n_features": self.holdout_meta.n_features, - "feature_names": ",".join(self.holdout_meta.feature_names), + "n_features": len(self.feature_cols), + "feature_names": ",".join(self.feature_cols[:20]), "builder_flow": current.flow_name, "builder_run_id": current.run_id, + "temporal_features": "True", + "deduplicated": "False", }, - tags={"dataset_type": "eval_holdout"}, - description=f"Holdout: {self.holdout_meta.total_samples} samples from {self.holdout_meta.snapshot_count} snapshots" + tags={"dataset_type": "eval_holdout", "has_temporal_features": "true"}, + description=f"Holdout: {self.holdout_meta.total_samples} rows from {self.holdout_meta.snapshot_count} snapshots (temporal features)" ) # Publish event for TrainFlow self.prj.safe_publish_event("dataset_ready", payload={ "training_samples": self.train_meta.total_samples, "holdout_samples": self.holdout_meta.total_samples, + "n_features": len(self.feature_cols), "builder_run_id": current.run_id, }) print("\nPublished 'dataset_ready' event") @@ -284,15 +320,16 @@ def register(self): def end(self): """Summary.""" print(f"\n{'='*50}") - print("Dataset Build Complete") + print("Dataset Build Complete (Temporal Features)") print(f"{'='*50}") print(f"\nTraining Dataset:") - print(f" Samples: {self.train_meta.total_samples}") + print(f" Rows: {self.train_meta.total_samples} (NOT deduplicated)") print(f" Snapshots: {self.train_meta.snapshot_count}") + print(f" Features: {len(self.feature_cols)} (includes lag + rolling)") tr = self.train_meta.snapshot_range print(f" Range: {tr[0][:19] if tr[0] else 'N/A'} to {tr[1][:19] if tr[1] else 'N/A'}") print(f"\nEval Holdout:") - print(f" Samples: {self.holdout_meta.total_samples}") + print(f" Rows: {self.holdout_meta.total_samples}") print(f" Snapshots: {self.holdout_meta.snapshot_count}") hr = self.holdout_meta.snapshot_range print(f" Range: {hr[0][:19] if hr[0] else 'N/A'} to {hr[1][:19] if hr[1] else 'N/A'}") diff --git a/flows/train/flow.py b/flows/train/flow.py index ba8c344..200a311 100644 --- a/flows/train/flow.py +++ b/flows/train/flow.py @@ -182,15 +182,17 @@ def train(self): @step def register(self): - """Register trained model as candidate.""" + """Register trained model as candidate and log predictions.""" from src import registry + from src.predictions import PredictionRecord, PredictionStore from metaflow import current + from datetime import datetime, timezone annotations = { "algorithm": self.trained_model.algorithm, **self.trained_model.hyperparameters, "n_features": self.feature_set.n_features, - "feature_names": ",".join(self.feature_set.feature_names), + "feature_names": ",".join(self.feature_set.feature_names[:20]), # Limit to avoid overflow "training_samples": self.feature_set.n_samples, "anomalies_detected": self.prediction.n_anomalies, "anomaly_rate": float(self.prediction.anomaly_rate), @@ -213,6 +215,30 @@ def register(self): print(f" Data: {self.data_source}") print(f" Run ID: {current.run_id}") + # Log predictions for outcome validation + prediction_timestamp = datetime.now(timezone.utc).isoformat() + model_version = f"{current.flow_name}/{current.run_id}" + + predictions = [] + for i, (score, label) in enumerate(zip(self.prediction.anomaly_scores, self.prediction.predictions)): + coin_info = self.feature_set.coin_info[i] + predictions.append(PredictionRecord( + coin_id=coin_info.get('coin_id', f'unknown_{i}'), + symbol=coin_info.get('symbol', 'UNK'), + prediction_timestamp=prediction_timestamp, + current_price=float(coin_info.get('current_price', 0)), + anomaly_score=float(score), + is_anomaly=(label == -1), # -1 = anomaly in isolation forest + model_version=model_version, + flow_run_id=current.run_id, + )) + + # Store predictions + store = PredictionStore(self.prj.project, self.prj.branch) + pred_path = store.log_predictions(predictions, current.flow_name, current.run_id) + print(f"\nLogged {len(predictions)} predictions for outcome validation") + print(f" Path: {pred_path}") + self.next(self.end) @step diff --git a/src/__init__.py b/src/__init__.py index 624e75d..0d3625b 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -22,4 +22,4 @@ # and include this package in the code package for remote execution METAFLOW_PACKAGE_POLICY = 'include' -__all__ = ["data", "model", "registry", "eval", "storage", "cards"] +__all__ = ["data", "model", "registry", "eval", "storage", "cards", "features", "predictions"] diff --git a/src/features.py b/src/features.py new file mode 100644 index 0000000..95c47a8 --- /dev/null +++ b/src/features.py @@ -0,0 +1,314 @@ +""" +Temporal feature engineering for crash prediction. + +This module transforms raw snapshots into ML-ready datasets with: +- Lag features (how did this coin behave N hours ago?) +- Rolling statistics (mean/std over past N hours) +- Future target labels (did this coin crash within 24h?) + +The goal is to predict price crashes BEFORE they happen, not just +detect "anomalies" in a static snapshot. + +Feature Engineering Philosophy: +------------------------------ +Raw snapshots capture point-in-time market state. But for prediction: +1. We need TEMPORAL context (what happened before?) +2. We need a TARGET to predict (what happens after?) +3. We need to preserve ALL observations (not deduplicate) + +Schema after feature engineering: +- Base features: price_change_pct_1h, price_change_pct_24h, etc. +- Lag features: price_change_pct_24h_lag_1h, _lag_6h, _lag_24h +- Rolling features: price_change_pct_24h_rolling_mean_24h, _rolling_std_24h +- Target: future_drop_15pct_24h (1 if coin dropped >15% in next 24h) +""" + +from typing import List, Optional, Tuple +from dataclasses import dataclass +import pyarrow as pa + + +@dataclass +class DatasetConfig: + """Configuration for dataset building.""" + # Window settings + max_history_hours: int = 168 # 7 days of history (rolling window) + min_snapshots_per_coin: int = 6 # Need at least 6 hours of history + + # Lag features (hours ago) + lag_hours: List[int] = None # Default: [1, 6, 24] + + # Rolling window features + rolling_windows_hours: List[int] = None # Default: [6, 24] + + # Target definition + target_lookahead_hours: int = 24 + crash_threshold_pct: float = -15.0 # -15% = crash + + def __post_init__(self): + if self.lag_hours is None: + self.lag_hours = [1, 6, 24] + if self.rolling_windows_hours is None: + self.rolling_windows_hours = [6, 24] + + +def add_temporal_features( + table: pa.Table, + config: Optional[DatasetConfig] = None, +) -> pa.Table: + """ + Add lag features and rolling statistics to a snapshot table. + + This transforms raw snapshots into ML-ready features by adding + temporal context for each observation. + + Args: + table: PyArrow table with columns: coin_id, snapshot_timestamp, features... + config: DatasetConfig with lag/rolling settings + + Returns: + PyArrow table with additional lag and rolling columns + """ + import pandas as pd + + if config is None: + config = DatasetConfig() + + df = table.to_pandas() + + # Ensure timestamp is datetime + df['snapshot_timestamp'] = pd.to_datetime(df['snapshot_timestamp']) + + # Sort by coin and time for proper lag calculation + df = df.sort_values(['coin_id', 'snapshot_timestamp']) + + # Base feature columns to create lags for + base_features = [ + 'price_change_pct_1h', + 'price_change_pct_24h', + 'price_change_pct_7d', + 'market_cap_to_volume', + 'sparkline_volatility', + ] + + # Add lag features per coin + for lag_hours in config.lag_hours: + for col in base_features: + if col in df.columns: + df[f'{col}_lag_{lag_hours}h'] = df.groupby('coin_id')[col].shift(lag_hours) + + # Add rolling statistics per coin + for window_hours in config.rolling_windows_hours: + for col in ['price_change_pct_24h', 'sparkline_volatility']: + if col in df.columns: + grouped = df.groupby('coin_id')[col] + df[f'{col}_rolling_mean_{window_hours}h'] = grouped.transform( + lambda x: x.rolling(window=window_hours, min_periods=1).mean() + ) + df[f'{col}_rolling_std_{window_hours}h'] = grouped.transform( + lambda x: x.rolling(window=window_hours, min_periods=2).std() + ) + + # Add time-based features + df['hour_of_day'] = df['snapshot_timestamp'].dt.hour + df['day_of_week'] = df['snapshot_timestamp'].dt.dayofweek + + # Convert back to PyArrow + return pa.Table.from_pandas(df, preserve_index=False) + + +def add_future_targets( + table: pa.Table, + config: Optional[DatasetConfig] = None, +) -> pa.Table: + """ + Add future price movement targets for supervised learning. + + IMPORTANT: This creates data leakage if used naively! + Only use for EVALUATION, not for training production models. + For training, you must use properly time-split data. + + Args: + table: PyArrow table with coin_id, snapshot_timestamp, current_price + config: DatasetConfig with target settings + + Returns: + Table with added target columns + """ + import pandas as pd + + if config is None: + config = DatasetConfig() + + df = table.to_pandas() + df['snapshot_timestamp'] = pd.to_datetime(df['snapshot_timestamp']) + df = df.sort_values(['coin_id', 'snapshot_timestamp']) + + # Calculate future price (lookahead_hours later) + df['future_price'] = df.groupby('coin_id')['current_price'].shift(-config.target_lookahead_hours) + + # Calculate future price change + df['future_price_change_pct'] = ( + (df['future_price'] - df['current_price']) / df['current_price'] * 100 + ) + + # Binary crash target + df['future_crash'] = (df['future_price_change_pct'] <= config.crash_threshold_pct).astype(int) + + # Binary pump target (for completeness) + df['future_pump'] = (df['future_price_change_pct'] >= abs(config.crash_threshold_pct)).astype(int) + + return pa.Table.from_pandas(df, preserve_index=False) + + +def filter_by_history( + table: pa.Table, + config: Optional[DatasetConfig] = None, +) -> pa.Table: + """ + Filter to rows that have sufficient history for lag features. + + Removes early observations for each coin that don't have enough + prior snapshots to compute lag features. + + Args: + table: PyArrow table with temporal features + config: DatasetConfig with min_snapshots_per_coin + + Returns: + Filtered table + """ + import pandas as pd + + if config is None: + config = DatasetConfig() + + df = table.to_pandas() + df['snapshot_timestamp'] = pd.to_datetime(df['snapshot_timestamp']) + + # Count observations per coin and rank by time + df['obs_rank'] = df.groupby('coin_id')['snapshot_timestamp'].rank(method='first') + + # Keep only rows with sufficient history + df = df[df['obs_rank'] >= config.min_snapshots_per_coin] + + # Drop helper column + df = df.drop(columns=['obs_rank']) + + return pa.Table.from_pandas(df, preserve_index=False) + + +def apply_rolling_window( + table: pa.Table, + max_hours: int = 168, +) -> pa.Table: + """ + Apply rolling window to limit dataset to recent history. + + This prevents unbounded dataset growth while keeping temporal diversity. + + Args: + table: PyArrow table with snapshot_timestamp + max_hours: Maximum hours of history to keep + + Returns: + Table filtered to recent window + """ + import pandas as pd + from datetime import timedelta + + df = table.to_pandas() + df['snapshot_timestamp'] = pd.to_datetime(df['snapshot_timestamp']) + + # Find the most recent timestamp + max_ts = df['snapshot_timestamp'].max() + cutoff = max_ts - timedelta(hours=max_hours) + + # Filter to recent window + df = df[df['snapshot_timestamp'] >= cutoff] + + return pa.Table.from_pandas(df, preserve_index=False) + + +def build_ml_dataset( + raw_table: pa.Table, + config: Optional[DatasetConfig] = None, + include_targets: bool = False, +) -> Tuple[pa.Table, List[str]]: + """ + Full pipeline: raw snapshots → ML-ready dataset. + + Args: + raw_table: Concatenated raw snapshots + config: DatasetConfig + include_targets: Whether to add future targets (for evaluation only!) + + Returns: + Tuple of (processed table, list of feature column names) + """ + if config is None: + config = DatasetConfig() + + # Step 1: Apply rolling window to bound dataset size + table = apply_rolling_window(raw_table, config.max_history_hours) + + # Step 2: Add temporal features + table = add_temporal_features(table, config) + + # Step 3: Add targets if requested (for evaluation/backtesting) + if include_targets: + table = add_future_targets(table, config) + + # Step 4: Filter to rows with sufficient history + table = filter_by_history(table, config) + + # Identify feature columns + df = table.to_pandas() + feature_cols = [ + col for col in df.columns + if col not in [ + 'coin_id', 'symbol', 'name', 'current_price', 'market_cap', + 'snapshot_timestamp', 'future_price', 'future_price_change_pct', + 'future_crash', 'future_pump' + ] + and not col.startswith('future_') + ] + + return table, feature_cols + + +def get_feature_columns(config: Optional[DatasetConfig] = None) -> List[str]: + """ + Get list of all feature columns that will be created. + + Useful for model training to know expected input shape. + """ + if config is None: + config = DatasetConfig() + + base_features = [ + 'price_change_pct_1h', + 'price_change_pct_24h', + 'price_change_pct_7d', + 'market_cap_to_volume', + 'ath_change_pct', + 'sparkline_volatility', + ] + + feature_cols = list(base_features) + + # Lag features + for lag in config.lag_hours: + for col in base_features: + feature_cols.append(f'{col}_lag_{lag}h') + + # Rolling features + for window in config.rolling_windows_hours: + for col in ['price_change_pct_24h', 'sparkline_volatility']: + feature_cols.append(f'{col}_rolling_mean_{window}h') + feature_cols.append(f'{col}_rolling_std_{window}h') + + # Time features + feature_cols.extend(['hour_of_day', 'day_of_week']) + + return feature_cols diff --git a/src/predictions.py b/src/predictions.py new file mode 100644 index 0000000..fea5b60 --- /dev/null +++ b/src/predictions.py @@ -0,0 +1,307 @@ +""" +Prediction logging for outcome validation. + +This module handles: +- Logging predictions with timestamps for later outcome validation +- Loading historical predictions to check against actual price movements +- Computing outcome-based metrics (crash recall, false alarm rate) + +Storage Layout: + {datastore_root}/projects/{project}/branches/{branch}/predictions/ + dt=2025-12-01/ + hour=18/ + pred_{run_id}.parquet + +Each prediction record contains: +- coin_id, symbol: Which coin +- prediction_timestamp: When the prediction was made +- current_price: Price at prediction time +- anomaly_score: Model's anomaly score +- is_anomaly: Binary prediction (True/False) +- model_version: Which model made this prediction +- flow_run_id: Which flow run +""" + +from typing import List, Dict, Optional, Tuple +from dataclasses import dataclass, asdict +from datetime import datetime, timezone, timedelta +import json + + +@dataclass +class PredictionRecord: + """A single prediction to be validated later.""" + coin_id: str + symbol: str + prediction_timestamp: str + current_price: float + anomaly_score: float + is_anomaly: bool + model_version: str + flow_run_id: str + + +@dataclass +class OutcomeRecord: + """A prediction with its outcome (did the coin actually crash?).""" + coin_id: str + symbol: str + prediction_timestamp: str + is_anomaly: bool + anomaly_score: float + price_at_prediction: float + price_after_24h: Optional[float] + price_change_pct: Optional[float] + actual_crash: Optional[bool] # Did it actually drop >15%? + model_version: str + + +@dataclass +class OutcomeMetrics: + """Metrics from outcome validation.""" + total_predictions: int + total_anomalies: int + total_crashes: int + true_positives: int # Flagged as anomaly AND crashed + false_positives: int # Flagged as anomaly but didn't crash + false_negatives: int # Not flagged but crashed + true_negatives: int # Not flagged and didn't crash + + @property + def crash_recall(self) -> float: + """% of actual crashes that were flagged as anomalies.""" + if self.total_crashes == 0: + return 0.0 + return self.true_positives / self.total_crashes + + @property + def anomaly_precision(self) -> float: + """% of flagged anomalies that actually crashed.""" + if self.total_anomalies == 0: + return 0.0 + return self.true_positives / self.total_anomalies + + @property + def false_alarm_rate(self) -> float: + """% of flagged anomalies that didn't crash.""" + if self.total_anomalies == 0: + return 0.0 + return self.false_positives / self.total_anomalies + + +class PredictionStore: + """Store and retrieve predictions for outcome validation.""" + + def __init__(self, project: str, branch: str): + from src.storage import get_datastore_root, get_project_storage_path + self.project = project + self.branch = branch + self.base_path = get_project_storage_path(project, branch, "predictions") + self.root, self.provider = get_datastore_root() + + def _get_prediction_path(self, timestamp: str, run_id: str) -> str: + """Generate path for a prediction log.""" + dt = datetime.fromisoformat(timestamp.replace('Z', '+00:00')) + date_str = dt.strftime('%Y-%m-%d') + hour_str = dt.strftime('%H') + return f"{self.base_path}/dt={date_str}/hour={hour_str}/pred_{run_id}.parquet" + + def log_predictions( + self, + predictions: List[PredictionRecord], + flow_name: str, + run_id: str, + ) -> str: + """ + Write predictions to storage for later outcome validation. + + Args: + predictions: List of PredictionRecord objects + flow_name: Name of the flow logging these predictions + run_id: Run ID of the flow + + Returns: + Path where predictions were stored + """ + import pyarrow as pa + import pyarrow.parquet as pq + from metaflow import S3 + import io + + if not predictions: + return "" + + # Convert to columnar format + columns = { + 'coin_id': [p.coin_id for p in predictions], + 'symbol': [p.symbol for p in predictions], + 'prediction_timestamp': [p.prediction_timestamp for p in predictions], + 'current_price': [p.current_price for p in predictions], + 'anomaly_score': [p.anomaly_score for p in predictions], + 'is_anomaly': [p.is_anomaly for p in predictions], + 'model_version': [p.model_version for p in predictions], + 'flow_run_id': [p.flow_run_id for p in predictions], + } + + table = pa.Table.from_pydict(columns) + + # Get path based on first prediction timestamp + path = self._get_prediction_path(predictions[0].prediction_timestamp, run_id) + + # Write to bytes + buffer = io.BytesIO() + pq.write_table(table, buffer, compression='snappy') + parquet_bytes = buffer.getvalue() + + if self.provider == 's3': + with S3() as s3: + s3.put(path, parquet_bytes) + else: + import os + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'wb') as f: + f.write(parquet_bytes) + + return path + + def get_predictions_for_validation( + self, + lookback_hours: int = 24, + max_age_hours: int = 48, + ) -> List[PredictionRecord]: + """ + Load predictions from ~lookback_hours ago for outcome validation. + + We want predictions that are: + - Old enough to have outcome data (at least lookback_hours old) + - Not too old (max_age_hours) + + Args: + lookback_hours: Minimum hours since prediction + max_age_hours: Maximum hours since prediction + + Returns: + List of PredictionRecord objects + """ + import pyarrow.parquet as pq + from metaflow import S3 + from concurrent.futures import ThreadPoolExecutor + + now = datetime.now(timezone.utc) + min_ts = now - timedelta(hours=max_age_hours) + max_ts = now - timedelta(hours=lookback_hours) + + # List prediction files + if self.provider == 's3': + with S3() as s3: + files = list(s3.list_recursive([self.base_path])) + paths = [f.url for f in files if f.url.endswith('.parquet')] + else: + import os + paths = [] + if os.path.exists(self.base_path): + for root, _, files in os.walk(self.base_path): + for f in files: + if f.endswith('.parquet'): + paths.append(os.path.join(root, f)) + + if not paths: + return [] + + # Load and filter + predictions = [] + + if self.provider == 's3': + with S3() as s3: + loaded = s3.get_many(paths) + for f in loaded: + table = pq.read_table(f.path) + df = table.to_pandas() + for _, row in df.iterrows(): + ts = datetime.fromisoformat(row['prediction_timestamp'].replace('Z', '+00:00')) + if min_ts <= ts <= max_ts: + predictions.append(PredictionRecord( + coin_id=row['coin_id'], + symbol=row['symbol'], + prediction_timestamp=row['prediction_timestamp'], + current_price=float(row['current_price']), + anomaly_score=float(row['anomaly_score']), + is_anomaly=bool(row['is_anomaly']), + model_version=row['model_version'], + flow_run_id=row['flow_run_id'], + )) + else: + for path in paths: + table = pq.read_table(path) + df = table.to_pandas() + for _, row in df.iterrows(): + ts = datetime.fromisoformat(row['prediction_timestamp'].replace('Z', '+00:00')) + if min_ts <= ts <= max_ts: + predictions.append(PredictionRecord( + coin_id=row['coin_id'], + symbol=row['symbol'], + prediction_timestamp=row['prediction_timestamp'], + current_price=float(row['current_price']), + anomaly_score=float(row['anomaly_score']), + is_anomaly=bool(row['is_anomaly']), + model_version=row['model_version'], + flow_run_id=row['flow_run_id'], + )) + + return predictions + + +def compute_outcome_metrics( + outcomes: List[OutcomeRecord], + crash_threshold_pct: float = -15.0, +) -> OutcomeMetrics: + """ + Compute outcome-based evaluation metrics. + + Args: + outcomes: List of OutcomeRecord with actual price changes + crash_threshold_pct: Threshold for "crash" (e.g., -15%) + + Returns: + OutcomeMetrics with precision, recall, false alarm rate + """ + total = len(outcomes) + anomalies = [o for o in outcomes if o.is_anomaly] + crashes = [o for o in outcomes if o.actual_crash] + + tp = len([o for o in outcomes if o.is_anomaly and o.actual_crash]) + fp = len([o for o in outcomes if o.is_anomaly and not o.actual_crash]) + fn = len([o for o in outcomes if not o.is_anomaly and o.actual_crash]) + tn = len([o for o in outcomes if not o.is_anomaly and not o.actual_crash]) + + return OutcomeMetrics( + total_predictions=total, + total_anomalies=len(anomalies), + total_crashes=len(crashes), + true_positives=tp, + false_positives=fp, + false_negatives=fn, + true_negatives=tn, + ) + + +def format_outcome_summary(metrics: OutcomeMetrics) -> str: + """Format outcome metrics for display.""" + lines = [ + "Outcome Validation Results:", + f" Total predictions: {metrics.total_predictions}", + f" Anomalies flagged: {metrics.total_anomalies}", + f" Actual crashes: {metrics.total_crashes}", + "", + "Confusion Matrix:", + f" True Positives (flagged + crashed): {metrics.true_positives}", + f" False Positives (flagged + no crash): {metrics.false_positives}", + f" False Negatives (not flagged + crashed): {metrics.false_negatives}", + f" True Negatives (not flagged + no crash): {metrics.true_negatives}", + "", + "Metrics:", + f" Crash Recall: {metrics.crash_recall:.1%} (% of crashes we caught)", + f" Anomaly Precision: {metrics.anomaly_precision:.1%} (% of flags that were real)", + f" False Alarm Rate: {metrics.false_alarm_rate:.1%} (% of flags that were wrong)", + ] + return "\n".join(lines) From 8e7923c5d0e5ad1f4f75bd0ef53d7f7abbbc726f Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Mon, 1 Dec 2025 12:03:56 -0800 Subject: [PATCH 05/26] enhance models page validation with version checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New validation checks: - Flow versions shown (Train/185989, Promote/185991 format) - No Internal Server Error on page - Model version count >= 1 - Champion badge via locator (not just text search) These checks ensure models are actually registered and displaying correctly, not just that the page renders. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- scripts/validate_dashboard.py | 39 ++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/scripts/validate_dashboard.py b/scripts/validate_dashboard.py index 95a2379..e90df0e 100644 --- a/scripts/validate_dashboard.py +++ b/scripts/validate_dashboard.py @@ -252,6 +252,7 @@ async def validate_models_page(self) -> ValidationResult: checks = {} page_content = await self.page.content() + visible_text = await self.page.inner_text("body") # Check for model versions table tables = self.page.locator("table") @@ -266,23 +267,51 @@ async def validate_models_page(self) -> ValidationResult: print(f" Status badges ({badge_count}): {'PASS' if checks['has_status_badges'] else 'FAIL'}") # Check for champion badge specifically - checks["has_champion_badge"] = "champion" in page_content.lower() + champion_badge = self.page.locator(".badge:has-text('champion')") + champion_count = await champion_badge.count() + checks["has_champion_badge"] = champion_count > 0 print(f" Champion badge: {'PASS' if checks['has_champion_badge'] else 'FAIL'}") # Check for algorithm info - checks["shows_algorithm"] = "isolation_forest" in page_content.lower() + checks["shows_algorithm"] = "isolation_forest" in page_content.lower() or "lof" in page_content.lower() print(f" Algorithm shown: {'PASS' if checks['shows_algorithm'] else 'FAIL'}") # Check for anomaly rate - checks["shows_anomaly_rate"] = "anomaly" in page_content.lower() or "10%" in page_content + checks["shows_anomaly_rate"] = "anomaly" in visible_text.lower() or "%" in visible_text print(f" Anomaly rate shown: {'PASS' if checks['shows_anomaly_rate'] else 'FAIL'}") # Check for compare functionality - compare_buttons = self.page.locator("button:has-text('Compare'), a:has-text('Compare')") + compare_buttons = self.page.locator("button:has-text('Compare'), select") compare_count = await compare_buttons.count() - checks["has_compare"] = compare_count > 0 or "compare" in page_content.lower() + checks["has_compare"] = compare_count > 0 print(f" Compare functionality: {'PASS' if checks['has_compare'] else 'FAIL'}") + # NEW: Check that model versions are from actual flows + # Look for flow/run format like "Train/185989" or "Promote/185991" + import re + flow_run_pattern = r"(Train|Evaluate|Promote)/\d+" + has_flow_versions = bool(re.search(flow_run_pattern, visible_text)) + checks["has_flow_versions"] = has_flow_versions + print(f" Flow versions shown: {'PASS' if checks['has_flow_versions'] else 'FAIL'}") + + # NEW: Verify no "Internal Server Error" on page + checks["no_error"] = "internal server error" not in visible_text.lower() + print(f" No errors: {'PASS' if checks['no_error'] else 'FAIL'}") + + # NEW: Check version count is reasonable (at least 1) + version_badges = self.page.locator(".badge:has-text('version')") + version_badge_count = await version_badges.count() + if version_badge_count > 0: + # Extract count from badge text like "1 versions" + badge_text = await version_badges.first.inner_text() + import re + count_match = re.search(r"(\d+)\s*version", badge_text.lower()) + version_count = int(count_match.group(1)) if count_match else 0 + else: + version_count = 0 + checks["has_model_versions"] = version_count >= 1 + print(f" Model versions ({version_count}): {'PASS' if checks['has_model_versions'] else 'FAIL'}") + result.checks = checks result.passed = all(checks.values()) From f5d21ca82302e60082cb4d12b3f83650b54db786 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Mon, 1 Dec 2025 12:24:58 -0800 Subject: [PATCH 06/26] add ValidateOutcomesFlow for outcome-based model validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the feedback loop by checking if flagged anomalies actually crashed: - Loads predictions from 24h ago (logged by TrainFlow) - Fetches current prices and computes actual price changes - Computes crash_recall, anomaly_precision, false_alarm_rate - Updates lifecycle-runner.md with new data pipeline diagram 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .claude/agents/lifecycle-runner.md | 8 +- flows/validate_outcomes/flow.py | 346 +++++++++++++++++++++++++++++ 2 files changed, 352 insertions(+), 2 deletions(-) create mode 100644 flows/validate_outcomes/flow.py diff --git a/.claude/agents/lifecycle-runner.md b/.claude/agents/lifecycle-runner.md index 102ffef..ea97894 100644 --- a/.claude/agents/lifecycle-runner.md +++ b/.claude/agents/lifecycle-runner.md @@ -20,8 +20,12 @@ candidate → evaluated → champion ### Data Pipeline ``` -IngestMarketDataFlow → BuildDatasetFlow → TrainDetectorFlow - (hourly snapshots) (accumulates) (trains on dataset) +IngestMarketDataFlow → BuildDatasetFlow → TrainDetectorFlow → ValidateOutcomesFlow + (hourly snapshots) (temporal features) (trains + logs) (24h later) + │ + ▼ + predictions/ + (logged for validation) ``` ## How to Run Flows diff --git a/flows/validate_outcomes/flow.py b/flows/validate_outcomes/flow.py new file mode 100644 index 0000000..f850790 --- /dev/null +++ b/flows/validate_outcomes/flow.py @@ -0,0 +1,346 @@ +""" +Validate predictions against actual market outcomes. + +This flow closes the feedback loop for crash prediction: +1. Loads predictions from ~24h ago +2. Fetches current prices for those coins +3. Checks if flagged anomalies actually crashed (>15% drop) +4. Computes outcome metrics (crash recall, precision, false alarm rate) +5. Updates model annotations with outcome-based metrics + +This is the key to answering: "Does our anomaly detector actually +predict crashes, or is it just detecting noise?" + +Outcome Metrics: +---------------- +- crash_recall: % of actual crashes we flagged in advance +- anomaly_precision: % of our flags that preceded real crashes +- false_alarm_rate: % of flags that were wrong + +Quality Gate (optional): +------------------------ +If outcome_precision < threshold, model shouldn't be promoted. +This is a stronger signal than silhouette score or rate stability. + +Triggering: +- @schedule(hourly=True) - runs hourly to validate predictions from 24h ago +- Can also run manually for backtesting + +Usage: + # Validate predictions from 24h ago + python flow.py run + + # Validate specific lookback window + python flow.py run --lookback_hours 48 --max_age_hours 72 +""" + +from metaflow import step, card, Parameter, schedule, Config +from obproject import ProjectFlow + +import src + + +@schedule(hourly=True) +class ValidateOutcomesFlow(ProjectFlow): + """ + Validate predictions against actual price movements. + + Answers: "Did the coins we flagged as anomalies actually crash?" + """ + + # Config for outcome validation thresholds + eval_config = Config("eval_config", default="configs/evaluation.json") + + lookback_hours = Parameter( + "lookback_hours", + default=24, + type=int, + help="Minimum hours since prediction (outcome observation window)" + ) + + max_age_hours = Parameter( + "max_age_hours", + default=48, + type=int, + help="Maximum hours since prediction to include" + ) + + crash_threshold_pct = Parameter( + "crash_threshold_pct", + default=-15.0, + type=float, + help="Price drop threshold to count as 'crash' (e.g., -15 = 15% drop)" + ) + + @step + def start(self): + """Load predictions from lookback window.""" + from src.predictions import PredictionStore + + print(f"Project: {self.prj.project}, Branch: {self.prj.branch}") + print(f"\nValidating predictions from {self.lookback_hours}-{self.max_age_hours}h ago") + print(f"Crash threshold: {self.crash_threshold_pct}%") + + # Load predictions + store = PredictionStore(self.prj.project, self.prj.branch) + self.predictions = store.get_predictions_for_validation( + lookback_hours=self.lookback_hours, + max_age_hours=self.max_age_hours, + ) + + if not self.predictions: + print("\nNo predictions found in validation window.") + print("Run TrainDetectorFlow first to generate predictions.") + self.has_predictions = False + else: + print(f"\nLoaded {len(self.predictions)} predictions for validation") + + # Group by model version + versions = {} + for p in self.predictions: + versions[p.model_version] = versions.get(p.model_version, 0) + 1 + print("Predictions by model:") + for v, count in versions.items(): + print(f" {v}: {count}") + + # Count anomalies + anomalies = [p for p in self.predictions if p.is_anomaly] + print(f"\nAnomalies flagged: {len(anomalies)} / {len(self.predictions)}") + + self.has_predictions = True + + self.next(self.fetch_outcomes) + + @step + def fetch_outcomes(self): + """Fetch current prices and compute outcomes.""" + from src import data + from src.predictions import OutcomeRecord + from datetime import datetime, timezone + + if not self.has_predictions: + self.outcomes = [] + self.next(self.compute_metrics) + return + + # Get unique coin IDs from predictions + coin_ids = list(set(p.coin_id for p in self.predictions)) + print(f"\nFetching current prices for {len(coin_ids)} coins...") + + # Fetch current market data + try: + snapshot = data.fetch_market_data(num_coins=250) # Get more to ensure coverage + current_prices = { + coin['id']: coin.get('current_price', 0) + for coin in snapshot.coins + } + print(f"Fetched prices for {len(current_prices)} coins") + except Exception as e: + print(f"Error fetching prices: {e}") + current_prices = {} + + # Build outcome records + self.outcomes = [] + missing_prices = 0 + + for pred in self.predictions: + current_price = current_prices.get(pred.coin_id) + + if current_price is None or current_price == 0: + missing_prices += 1 + continue + + # Calculate price change + price_change_pct = ( + (current_price - pred.current_price) / pred.current_price * 100 + if pred.current_price > 0 else 0 + ) + + # Did it actually crash? + actual_crash = price_change_pct <= self.crash_threshold_pct + + self.outcomes.append(OutcomeRecord( + coin_id=pred.coin_id, + symbol=pred.symbol, + prediction_timestamp=pred.prediction_timestamp, + is_anomaly=pred.is_anomaly, + anomaly_score=pred.anomaly_score, + price_at_prediction=pred.current_price, + price_after_24h=current_price, + price_change_pct=price_change_pct, + actual_crash=actual_crash, + model_version=pred.model_version, + )) + + if missing_prices > 0: + print(f"[WARN] Missing prices for {missing_prices} coins (may have been delisted)") + + print(f"\nBuilt {len(self.outcomes)} outcome records") + + # Quick stats + crashes = [o for o in self.outcomes if o.actual_crash] + print(f"Actual crashes in period: {len(crashes)}") + + self.next(self.compute_metrics) + + @card + @step + def compute_metrics(self): + """Compute outcome-based evaluation metrics.""" + from src.predictions import compute_outcome_metrics, format_outcome_summary + from metaflow import current + from metaflow.cards import Markdown, Table + + current.card.append(Markdown("# Outcome Validation")) + current.card.append(Markdown(f"**Lookback:** {self.lookback_hours}-{self.max_age_hours}h")) + current.card.append(Markdown(f"**Crash Threshold:** {self.crash_threshold_pct}%")) + + if not self.outcomes: + print("\nNo outcomes to compute metrics.") + current.card.append(Markdown("*No predictions found in validation window.*")) + self.metrics = None + self.next(self.update_model_metrics) + return + + # Compute metrics + self.metrics = compute_outcome_metrics(self.outcomes, self.crash_threshold_pct) + print(f"\n{format_outcome_summary(self.metrics)}") + + # Build card + current.card.append(Markdown("## Summary")) + current.card.append(Table([ + ["Total Predictions", str(self.metrics.total_predictions)], + ["Anomalies Flagged", str(self.metrics.total_anomalies)], + ["Actual Crashes", str(self.metrics.total_crashes)], + ], headers=["Metric", "Value"])) + + current.card.append(Markdown("## Confusion Matrix")) + current.card.append(Table([ + ["True Positives", str(self.metrics.true_positives), "Flagged AND crashed"], + ["False Positives", str(self.metrics.false_positives), "Flagged but no crash"], + ["False Negatives", str(self.metrics.false_negatives), "Not flagged but crashed"], + ["True Negatives", str(self.metrics.true_negatives), "Not flagged, no crash"], + ], headers=["", "Count", "Description"])) + + current.card.append(Markdown("## Outcome Metrics")) + + # Color-code the metrics + recall_status = "✅" if self.metrics.crash_recall >= 0.5 else "⚠️" + precision_status = "✅" if self.metrics.anomaly_precision >= 0.3 else "⚠️" + far_status = "✅" if self.metrics.false_alarm_rate <= 0.7 else "⚠️" + + current.card.append(Table([ + [f"{recall_status} Crash Recall", f"{self.metrics.crash_recall:.1%}", "% of crashes we caught"], + [f"{precision_status} Anomaly Precision", f"{self.metrics.anomaly_precision:.1%}", "% of flags that were real"], + [f"{far_status} False Alarm Rate", f"{self.metrics.false_alarm_rate:.1%}", "% of flags that were wrong"], + ], headers=["Metric", "Value", "Description"])) + + # Show worst false alarms (flagged but didn't crash) + false_alarms = [o for o in self.outcomes if o.is_anomaly and not o.actual_crash] + if false_alarms: + false_alarms.sort(key=lambda x: x.price_change_pct, reverse=True) + current.card.append(Markdown("## Top False Alarms")) + current.card.append(Markdown("*Coins flagged as anomalies that didn't crash:*")) + fa_rows = [ + [o.symbol, f"{o.price_change_pct:+.1f}%", f"{o.anomaly_score:.3f}"] + for o in false_alarms[:5] + ] + current.card.append(Table(fa_rows, headers=["Coin", "Actual Change", "Score"])) + + # Show missed crashes (crashed but not flagged) + missed = [o for o in self.outcomes if not o.is_anomaly and o.actual_crash] + if missed: + missed.sort(key=lambda x: x.price_change_pct) + current.card.append(Markdown("## Missed Crashes")) + current.card.append(Markdown("*Coins that crashed but weren't flagged:*")) + missed_rows = [ + [o.symbol, f"{o.price_change_pct:+.1f}%", f"{o.anomaly_score:.3f}"] + for o in missed[:5] + ] + current.card.append(Table(missed_rows, headers=["Coin", "Actual Change", "Score"])) + + # Show true positives (correctly flagged crashes) + true_pos = [o for o in self.outcomes if o.is_anomaly and o.actual_crash] + if true_pos: + true_pos.sort(key=lambda x: x.price_change_pct) + current.card.append(Markdown("## Correctly Flagged Crashes ✅")) + tp_rows = [ + [o.symbol, f"{o.price_change_pct:+.1f}%", f"{o.anomaly_score:.3f}"] + for o in true_pos[:5] + ] + current.card.append(Table(tp_rows, headers=["Coin", "Actual Change", "Score"])) + + self.next(self.update_model_metrics) + + @step + def update_model_metrics(self): + """Update model annotations with outcome metrics.""" + from src import registry + from metaflow import current + + if not self.metrics: + print("\nNo metrics to update.") + self.next(self.end) + return + + # Group outcomes by model version to update each + version_outcomes = {} + for o in self.outcomes: + if o.model_version not in version_outcomes: + version_outcomes[o.model_version] = [] + version_outcomes[o.model_version].append(o) + + print(f"\nUpdating metrics for {len(version_outcomes)} model versions...") + + for version, outcomes in version_outcomes.items(): + from src.predictions import compute_outcome_metrics + version_metrics = compute_outcome_metrics(outcomes, self.crash_threshold_pct) + + # Note: We don't have a direct way to update existing model annotations + # without re-registering. In a real system, you'd store these metrics + # in a separate table or use the registry's annotation update API. + print(f"\n{version}:") + print(f" Crash Recall: {version_metrics.crash_recall:.1%}") + print(f" Anomaly Precision: {version_metrics.anomaly_precision:.1%}") + print(f" False Alarm Rate: {version_metrics.false_alarm_rate:.1%}") + + # Store summary for the flow + self.outcome_metrics_summary = { + "crash_recall": self.metrics.crash_recall, + "anomaly_precision": self.metrics.anomaly_precision, + "false_alarm_rate": self.metrics.false_alarm_rate, + "total_predictions": self.metrics.total_predictions, + "total_crashes": self.metrics.total_crashes, + "validated_at": current.run_id, + } + + self.next(self.end) + + @step + def end(self): + """Summary.""" + print(f"\n{'='*50}") + print("Outcome Validation Complete") + print(f"{'='*50}") + + if self.metrics: + print(f"\nResults:") + print(f" Crash Recall: {self.metrics.crash_recall:.1%}") + print(f" Anomaly Precision: {self.metrics.anomaly_precision:.1%}") + print(f" False Alarm Rate: {self.metrics.false_alarm_rate:.1%}") + + if self.metrics.crash_recall >= 0.5 and self.metrics.anomaly_precision >= 0.3: + print(f"\n✅ Model shows predictive value!") + else: + print(f"\n⚠️ Model may need improvement") + if self.metrics.crash_recall < 0.5: + print(f" - Low crash recall: missing too many crashes") + if self.metrics.anomaly_precision < 0.3: + print(f" - Low precision: too many false alarms") + else: + print("\nNo predictions to validate.") + print("Run TrainDetectorFlow to generate predictions first.") + + +if __name__ == "__main__": + ValidateOutcomesFlow() From 245d335caf83d0361605ac9825f975d6edbde0b1 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Mon, 1 Dec 2025 12:31:15 -0800 Subject: [PATCH 07/26] add eval_config to flow_configs in obproject.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes CI deployment failure - the deploy script needs eval_config in flow_configs to pass the config path correctly when running from flow directories. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- obproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/obproject.toml b/obproject.toml index 7e57905..58494a0 100644 --- a/obproject.toml +++ b/obproject.toml @@ -22,6 +22,7 @@ perimeter = "default" [environments.production.flow_configs] model_config = "configs/model.json" training_config = "configs/training.json" +eval_config = "configs/evaluation.json" # Development environment (all other branches) [environments.dev] @@ -30,3 +31,4 @@ perimeter = "default" [environments.dev.flow_configs] model_config = "configs/model.json" training_config = "configs/training.json" +eval_config = "configs/evaluation.json" From 89847609b878a0ce141864fd17b4be34a2d4e1c5 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Mon, 1 Dec 2025 21:48:22 -0800 Subject: [PATCH 08/26] refactor: use Metaflow run tags for champion management MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace Asset API tags with Metaflow's native run tagging for champion tracking. This enables server-side queries via Flow.runs("champion"). Changes: - Remove ModelStatus enum (candidate/evaluated/champion states) - Add get_champion_run_id() and set_champion_run() functions - Simplify register_model() to not require status parameter - Update register_evaluation() to link results to model versions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/registry.py | 415 +++++++++++++++++++++++++++--------------------- src/storage.py | 8 +- 2 files changed, 243 insertions(+), 180 deletions(-) diff --git a/src/registry.py b/src/registry.py index e2b6b36..0d799d5 100644 --- a/src/registry.py +++ b/src/registry.py @@ -1,63 +1,68 @@ """ -Asset registration, versioning, and consumption. +Model Registry - Asset registration, versioning, and champion management. -This module handles: -- Registering model versions with annotations -- Consuming models by version or alias -- Updating model status (candidate -> evaluated -> champion) -- Event publishing for lifecycle transitions +Core capabilities demonstrated: +1. Create/register model versions with lineage (via Asset API) +2. Load models by version +3. Compare any two model versions +4. Track evaluation results linked to specific versions +5. Champion management via Metaflow run tags (not Asset tags) + +Note: Champion/alias management uses Metaflow's native run tagging system, +not Asset API tags. This is because Asset API doesn't support tag-based queries. """ -from typing import Dict, Optional, Any +from typing import Dict, Optional, Any, List from dataclasses import dataclass -from enum import Enum - - -class ModelStatus(str, Enum): - """Model lifecycle status.""" - CANDIDATE = "candidate" - EVALUATED = "evaluated" - CHALLENGER = "challenger" - CHAMPION = "champion" - RETIRED = "retired" @dataclass class ModelRef: """Reference to a registered model version.""" - version: int - status: ModelStatus + version: str annotations: Dict[str, Any] - tags: Dict[str, str] @classmethod def from_asset_ref(cls, ref: Dict) -> "ModelRef": """Create from consume_model_asset result.""" props = ref.get("model_properties", {}) - # Tags are at top level as list of {key, value} dicts - tags_list = ref.get("tags", []) - tags = {t["key"]: t["value"] for t in tags_list} if tags_list else {} - - status_str = tags.get("status", "unknown") - - try: - status = ModelStatus(status_str) - except ValueError: - status = ModelStatus.CANDIDATE - return cls( version=ref.get("id"), - status=status, annotations=props.get("annotations", {}), - tags=tags, ) + @property + def algorithm(self) -> Optional[str]: + return self.annotations.get("algorithm") + + @property + def training_run_id(self) -> Optional[str]: + return self.annotations.get("training_run_id") + + @property + def training_flow(self) -> Optional[str]: + return self.annotations.get("training_flow") + + @property + def data_source(self) -> Optional[str]: + return self.annotations.get("data_source") + + +def _get_asset(prj_or_asset): + """Get Asset from ProjectContext or return Asset directly.""" + if hasattr(prj_or_asset, "asset"): + return prj_or_asset.asset + return prj_or_asset + + +# ============================================================================= +# Model Registration +# ============================================================================= def register_model( prj_or_asset, name: str, - status: ModelStatus, annotations: Dict[str, Any], description: str = "", ) -> None: @@ -66,23 +71,25 @@ def register_model( Args: prj_or_asset: ProjectFlow.prj (ProjectContext) or Asset - name: Model asset name (e.g., "anomaly_detector") - status: Initial status (typically CANDIDATE for new models) - annotations: Model metadata (hyperparameters, metrics, lineage) + name: Model asset name (e.g., "iris_classifier", "anomaly_detector") + annotations: Model metadata including: + - algorithm: Model type/algorithm + - hyperparameters: Training hyperparameters + - training_run_id: Flow run ID for lineage + - training_flow: Flow name for lineage + - data_source: Training dataset reference + - training_samples: Number of training samples + - metrics: Training metrics (accuracy, loss, etc.) description: Human-readable description - """ - tags = { - "status": status.value, - "algorithm": annotations.get("algorithm", "unknown"), - "data_source": annotations.get("data_source", "unknown"), - } - # Get Asset - either directly or via ProjectContext.asset + Note: + Champion/alias management is handled separately via Metaflow run tags. + See set_champion_run() and get_champion_run_id(). + """ + # Get Asset if hasattr(prj_or_asset, "asset"): - # ProjectContext - get asset from it asset = prj_or_asset.asset elif hasattr(prj_or_asset, "register_model_asset"): - # Already an Asset asset = prj_or_asset else: raise TypeError(f"Expected ProjectContext or Asset, got {type(prj_or_asset)}") @@ -91,134 +98,140 @@ def register_model( name, kind="sklearn", annotations=annotations, - tags=tags, ) -def _get_asset(prj_or_asset): - """Get Asset from ProjectContext or return Asset directly.""" - if hasattr(prj_or_asset, "asset"): - return prj_or_asset.asset - return prj_or_asset - +# ============================================================================= +# Model Loading +# ============================================================================= def load_model( prj_or_asset, name: str, - instance: str = "latest", + version: Optional[str] = None, + instance: Optional[str] = None, ) -> ModelRef: """ - Load a model by version or alias. + Load a model by version. Args: prj_or_asset: ProjectContext or Asset name: Model asset name - instance: Version ("v1", "latest", "latest-1") or alias ("champion") + version: Specific version ("latest", "latest-1", or version ID) + instance: Alias for version parameter (for compatibility) Returns: ModelRef with version info and annotations Raises: Exception: If model not found + + Examples: + # Load latest version + model = load_model(asset, "iris_classifier", version="latest") + + # Load specific version + model = load_model(asset, "iris_classifier", version="v5") + + Note: + To load the champion model, first get the champion run ID via + get_champion_run_id(), then load by that run's version. """ + # Support both 'version' and 'instance' parameters + version = version or instance or "latest" + asset = _get_asset(prj_or_asset) - ref = asset.consume_model_asset(name, instance=instance) + ref = asset.consume_model_asset(name, instance=version) + return ModelRef.from_asset_ref(ref) -def load_champion_or_latest( +def list_model_versions( prj_or_asset, name: str, -) -> ModelRef: + limit: int = 20, +) -> List[ModelRef]: """ - Load champion model, falling back to latest. + List model versions (most recent first). Args: prj_or_asset: ProjectContext or Asset name: Model asset name + limit: Maximum versions to return Returns: - ModelRef (champion if exists, else latest) + List of ModelRef, most recent first """ - # Try champion first - try: - return load_model(prj_or_asset, name, instance="champion") - except Exception: - pass - - # Fall back to latest - return load_model(prj_or_asset, name, instance="latest") + asset = _get_asset(prj_or_asset) + versions = [] + seen = set() + for i in range(limit): + try: + instance = "latest" if i == 0 else f"latest-{i}" + ref = asset.consume_model_asset(name, instance=instance) + model_ref = ModelRef.from_asset_ref(ref) -def update_status( - prj_or_asset, - name: str, - new_status: ModelStatus, - current_annotations: Dict[str, Any], - additional_annotations: Optional[Dict[str, Any]] = None, -) -> None: - """ - Update model status (e.g., candidate -> evaluated). + if model_ref.version not in seen: + versions.append(model_ref) + seen.add(model_ref.version) + except Exception: + break - Args: - prj_or_asset: ProjectContext or Asset - name: Model asset name - new_status: New status to set - current_annotations: Current model annotations to preserve - additional_annotations: Additional annotations to add - """ - asset = _get_asset(prj_or_asset) - annotations = {**current_annotations} - if additional_annotations: - annotations.update(additional_annotations) + return versions - asset.register_model_asset( - name, - kind="sklearn", - annotations=annotations, - tags={ - "status": new_status.value, - "algorithm": annotations.get("algorithm", "unknown"), - "data_source": annotations.get("data_source", "unknown"), - }, - ) +# ============================================================================= +# Model Comparison +# ============================================================================= -def promote_to_champion( - prj_or_asset, - name: str, - source_version: int, - source_annotations: Dict[str, Any], - promotion_metadata: Dict[str, Any], -) -> None: +def compare_models( + model1: ModelRef, + model2: ModelRef, +) -> Dict[str, Any]: """ - Promote a model version to champion status. + Compare two model versions. Args: - prj_or_asset: ProjectContext or Asset (target for promotion) - name: Model asset name - source_version: Version being promoted - source_annotations: Annotations from source model - promotion_metadata: Additional metadata (flow name, run id, etc.) + model1: First model reference + model2: Second model reference + + Returns: + Dict with comparison of parameters, metrics, and lineage """ - asset = _get_asset(prj_or_asset) - annotations = { - **source_annotations, - "promoted_from_version": source_version, - **promotion_metadata, + def safe_get(d, key, default=None): + return d.get(key, default) if d else default + + ann1 = model1.annotations + ann2 = model2.annotations + + return { + "v1": { + "version": model1.version, + "algorithm": model1.algorithm, + "training_run_id": model1.training_run_id, + "data_source": model1.data_source, + "training_samples": safe_get(ann1, "training_samples"), + "anomaly_rate": safe_get(ann1, "anomaly_rate"), + "silhouette_score": safe_get(ann1, "silhouette_score"), + "score_gap": safe_get(ann1, "score_gap"), + }, + "v2": { + "version": model2.version, + "algorithm": model2.algorithm, + "training_run_id": model2.training_run_id, + "data_source": model2.data_source, + "training_samples": safe_get(ann2, "training_samples"), + "anomaly_rate": safe_get(ann2, "anomaly_rate"), + "silhouette_score": safe_get(ann2, "silhouette_score"), + "score_gap": safe_get(ann2, "score_gap"), + }, } - asset.register_model_asset( - name, - kind="sklearn", - annotations=annotations, - tags={ - "status": ModelStatus.CHAMPION.value, - "algorithm": "isolation_forest", - "promoted_from": str(source_version), - }, - ) +# ============================================================================= +# Data Assets (for completeness) +# ============================================================================= def register_market_data( prj, @@ -227,16 +240,7 @@ def register_market_data( timestamp: str, run_metadata: Dict[str, Any], ) -> None: - """ - Register market data snapshot as DataAsset. - - Args: - prj: ProjectFlow.prj (ProjectContext) - n_samples: Number of coins in snapshot - n_features: Number of features extracted - timestamp: ISO timestamp of data fetch - run_metadata: Flow/run metadata - """ + """Register market data snapshot as DataAsset.""" prj.register_data( "market_snapshot", "snapshot", @@ -247,9 +251,6 @@ def register_market_data( "data_source": "coingecko", **run_metadata, }, - tags={ - "data_source": "coingecko", - }, ) @@ -257,54 +258,52 @@ def load_market_data( prj_or_asset, instance: str = "latest", ) -> Dict[str, Any]: - """ - Load market data snapshot. - - Args: - prj_or_asset: ProjectContext or Asset - instance: Version to load - - Returns: - Dict with snapshot metadata - """ + """Load market data snapshot.""" asset = _get_asset(prj_or_asset) return asset.consume_data_asset("market_snapshot", instance=instance) +# ============================================================================= +# Evaluation Results +# ============================================================================= + def register_evaluation( prj, - decision: str, - candidate_version: int, - champion_version: Optional[int], + model_name: str, + model_version: str, + passed: bool, metrics: Dict[str, Any], - run_metadata: Dict[str, Any], + eval_dataset: Optional[str] = None, + compared_to_version: Optional[str] = None, + run_metadata: Optional[Dict[str, Any]] = None, ) -> None: """ - Register evaluation results as data asset. + Register evaluation results linked to a specific model version. Args: prj: ProjectFlow.prj - decision: "approve" or "reject" - candidate_version: Version that was evaluated - champion_version: Champion version compared against (if any) - metrics: Evaluation metrics - run_metadata: Flow/run metadata + model_name: Name of the model that was evaluated + model_version: Version that was evaluated + passed: Whether quality gates passed + metrics: Evaluation metrics (anomaly_rate, silhouette_score, etc.) + eval_dataset: Reference to evaluation dataset + compared_to_version: Version this was compared against (if any) + run_metadata: Flow/run metadata for lineage """ + annotations = { + "model_name": model_name, + "model_version": model_version, + "passed": passed, + "compared_to_version": compared_to_version, + "eval_dataset": eval_dataset, + **metrics, + **(run_metadata or {}), + } + prj.register_data( "evaluation_results", "evaluation_record", - annotations={ - "decision": decision, - "candidate_model_asset": "anomaly_detector", - "candidate_model_version": candidate_version, - "champion_model_version": champion_version, - **metrics, - **run_metadata, - }, - tags={ - "decision": decision, - "candidate_version": str(candidate_version), - }, + annotations=annotations, ) @@ -313,12 +312,74 @@ def publish_event( event_name: str, payload: Dict[str, Any], ) -> None: + """Publish lifecycle event.""" + prj.publish_event(event_name, payload=payload) + + +# ============================================================================= +# Champion Management (via Metaflow run tags) +# ============================================================================= + +def get_champion_run_id(flow_name: str = "TrainDetectorFlow") -> Optional[str]: """ - Publish lifecycle event. + Get the run ID of the current champion model. + + Uses Metaflow's native tagging to find runs tagged 'champion'. Args: - prj: ProjectFlow.prj - event_name: Event name (e.g., "approval_requested", "model_promoted") - payload: Event payload + flow_name: The training flow name + + Returns: + The run ID of the champion, or None if no champion set """ - prj.publish_event(event_name, payload=payload) + from metaflow import Flow + + try: + flow = Flow(flow_name) + # Query runs with the 'champion' tag + for run in flow.runs("champion"): + return run.id + except Exception: + pass + + return None + + +def set_champion_run( + run_id: str, + flow_name: str = "TrainDetectorFlow", +) -> Optional[str]: + """ + Set a training run as the champion by tagging it. + + Removes 'champion' tag from any previous champion and adds it to the new one. + + Args: + run_id: The run ID to promote + flow_name: The training flow name + + Returns: + The previous champion's run ID, or None if no previous champion + """ + from metaflow import Flow, Run + + previous_champion = None + + # Find and untag current champion + try: + flow = Flow(flow_name) + for run in flow.runs("champion"): + previous_champion = run.id + run.remove_tag("champion") + break # Only one champion at a time + except Exception: + pass + + # Tag the new champion + try: + run = Run(f"{flow_name}/{run_id}") + run.add_tag("champion") + except Exception as e: + raise RuntimeError(f"Failed to tag run {run_id} as champion: {e}") + + return previous_champion diff --git a/src/storage.py b/src/storage.py index 6fb6cad..d698143 100644 --- a/src/storage.py +++ b/src/storage.py @@ -411,17 +411,19 @@ def load_dataset(self, name: str, version: str = "latest", num_threads: int = 8) Returns: Tuple of (pyarrow.Table, DatasetMetadata) + Note: metadata.builder_run_id contains the resolved version when 'latest' is used """ import pyarrow as pa import pyarrow.parquet as pq from metaflow import S3 + resolved_version = version if version == "latest": - version = self._get_latest_version(name) - if not version: + resolved_version = self._get_latest_version(name) + if not resolved_version: return None, None - dataset_path = self._get_dataset_path(name, version) + dataset_path = self._get_dataset_path(name, resolved_version) meta_path = f"{dataset_path}/_metadata.json" # Load metadata From 156b4e55184549b39f1c9bd927ee7f90913d803a Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Mon, 1 Dec 2025 21:48:27 -0800 Subject: [PATCH 09/26] update flows to use new registry API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - TrainDetectorFlow: add data lineage tracking (snapshot range, count) - EvaluateDetectorFlow: flexible model version selection via parameters - PromoteModelFlow: use Metaflow run tags instead of Asset status 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- flows/evaluate/flow.py | 179 +++++++++++++++++++----------- flows/promote/flow.py | 239 +++++++++++++++++++++++++---------------- flows/train/flow.py | 27 ++++- 3 files changed, 287 insertions(+), 158 deletions(-) diff --git a/flows/evaluate/flow.py b/flows/evaluate/flow.py index b9d6956..6072163 100644 --- a/flows/evaluate/flow.py +++ b/flows/evaluate/flow.py @@ -2,15 +2,20 @@ Evaluate anomaly detection model on fresh market data. This flow: -1. Loads candidate model from asset registry -2. Fetches fresh market data (not from asset - point is to test on new data) -3. Runs inference with same model config -4. Applies quality gates -5. Updates status to 'evaluated' if passed +1. Loads the candidate model (configurable version, default=latest) +2. Optionally loads a comparison model (e.g., current champion) +3. Fetches fresh market data for evaluation +4. Runs inference and applies quality gates +5. Registers evaluation results linked to the model version + +Model Loading: +- By default, evaluates "latest" (the candidate) +- Can specify any version or alias via parameters +- Comparison model is optional (for first model, or A/B tests) Triggering: - Auto-triggers when TrainDetectorFlow completes (in Argo) -- Publishes 'approval_requested' event on success +- Publishes 'evaluation_passed' event on success """ from metaflow import step, card, Parameter, Config, trigger_on_finish @@ -26,45 +31,68 @@ class EvaluateDetectorFlow(ProjectFlow): """ Evaluate anomaly detector on fresh market data. - Fetches new data (not from asset) to test model generalization. + Loads models by version or alias - no hardcoded assumptions. """ # Centralized configs training_config = Config("training_config", default="configs/training.json") eval_config = Config("eval_config", default="configs/evaluation.json") + # Model selection parameters + candidate_version = Parameter( + "candidate_version", + default="latest", + help="Version of model to evaluate (e.g., 'latest', 'v5', or version ID)" + ) + + compare_to = Parameter( + "compare_to", + default="latest-1", + help="Version to compare against (e.g., 'latest-1', 'champion', or version ID). Use 'none' to skip comparison." + ) + @step def start(self): - """Load candidate model from registry.""" + """Load models from registry.""" from src import registry print(f"Project: {self.prj.project}, Branch: {self.prj.branch}") - # Load candidate (latest) + # Load candidate model + print(f"\nLoading candidate model (version={self.candidate_version})...") try: self.candidate = registry.load_model( - self.prj.asset, "anomaly_detector", instance="latest" + self.prj.asset, + "anomaly_detector", + version=self.candidate_version ) except Exception as e: - raise RuntimeError(f"No anomaly_detector found. Run TrainDetectorFlow first: {e}") + raise RuntimeError(f"Could not load anomaly_detector version '{self.candidate_version}'. " + f"Run TrainDetectorFlow first: {e}") - print(f"\nCandidate (v{self.candidate.version}):") - print(f" Status: {self.candidate.status.value}") - print(f" Algorithm: {self.candidate.annotations.get('algorithm')}") + print(f"Candidate (v{self.candidate.version}):") + print(f" Algorithm: {self.candidate.algorithm}") + print(f" Alias: {self.candidate.alias or 'none'}") print(f" Training anomaly rate: {float(self.candidate.annotations.get('anomaly_rate', 0)):.1%}") - - # Load previous version for comparison - try: - self.champion = registry.load_model( - self.prj.asset, "anomaly_detector", instance="latest-1" - ) - self.has_champion = True - print(f"\nPrevious (v{self.champion.version}):") - print(f" Training anomaly rate: {float(self.champion.annotations.get('anomaly_rate', 0)):.1%}") - except Exception: - print("\nNo previous version (first model)") - self.has_champion = False - self.champion = None + print(f" Training run: {self.candidate.training_run_id}") + + # Optionally load comparison model + self.comparison = None + if self.compare_to and self.compare_to.lower() != "none": + print(f"\nLoading comparison model (version={self.compare_to})...") + try: + self.comparison = registry.load_model( + self.prj.asset, + "anomaly_detector", + version=self.compare_to + ) + print(f"Comparison (v{self.comparison.version}):") + print(f" Algorithm: {self.comparison.algorithm}") + print(f" Alias: {self.comparison.alias or 'none'}") + print(f" Training anomaly rate: {float(self.comparison.annotations.get('anomaly_rate', 0)):.1%}") + except Exception as e: + print(f"[INFO] No comparison model found ({self.compare_to}): {e}") + print("Proceeding without comparison (first model or invalid version)") self.next(self.fetch_eval_data) @@ -143,7 +171,11 @@ def evaluate(self): ) current.card.append(Markdown("# Anomaly Detector Evaluation")) - current.card.append(Markdown(f"**Model:** v{self.candidate.version}")) + current.card.append(Markdown(f"**Candidate:** v{self.candidate.version}")) + if self.candidate.alias: + current.card.append(Markdown(f"**Alias:** {self.candidate.alias}")) + if self.comparison: + current.card.append(Markdown(f"**Comparing to:** v{self.comparison.version} ({self.comparison.alias or 'no alias'})")) current.card.append(Markdown(f"**Evaluation Time:** {self.feature_set.timestamp}")) # Quality gates with visual indicators @@ -157,7 +189,7 @@ def evaluate(self): title="Evaluation Score Distribution" )) - # Model comparison (candidate vs previous) + # Model comparison (if we have a comparison model) candidate_metrics = { "version": f"v{self.candidate.version}", "anomaly_rate": self.prediction.anomaly_rate, @@ -165,55 +197,76 @@ def evaluate(self): "score_gap": self.eval_result.metrics.score_gap, "training_samples": int(self.candidate.annotations.get("training_samples", 0)), } - champion_metrics = None - if self.has_champion: - champion_metrics = { - "version": f"v{self.champion.version}", - "anomaly_rate": float(self.champion.annotations.get("anomaly_rate", 0)), - "silhouette_score": float(self.champion.annotations.get("silhouette_score", 0)) if self.champion.annotations.get("silhouette_score") else None, - "score_gap": float(self.champion.annotations.get("score_gap", 0)) if self.champion.annotations.get("score_gap") else None, - "training_samples": int(self.champion.annotations.get("training_samples", 0)), + comparison_metrics = None + if self.comparison: + comparison_metrics = { + "version": f"v{self.comparison.version}", + "anomaly_rate": float(self.comparison.annotations.get("anomaly_rate", 0)), + "silhouette_score": float(self.comparison.annotations.get("silhouette_score", 0)) if self.comparison.annotations.get("silhouette_score") else None, + "score_gap": float(self.comparison.annotations.get("score_gap", 0)) if self.comparison.annotations.get("score_gap") else None, + "training_samples": int(self.comparison.annotations.get("training_samples", 0)), } - current.card.append(model_comparison_table(candidate_metrics, champion_metrics)) + current.card.append(model_comparison_table(candidate_metrics, comparison_metrics)) # Detected anomalies table current.card.append(Markdown("## Detected Anomalies")) if self.anomalies: current.card.append(top_anomalies_table(self.anomalies, limit=10)) - self.next(self.update_status) + self.next(self.register_evaluation) @step - def update_status(self): - """Update model status if gates passed.""" + def register_evaluation(self): + """Register evaluation results linked to model version.""" from src import registry from metaflow import current + # Create evaluation record artifact (required by prj.register_data) + self.evaluation_record = { + "model_name": "anomaly_detector", + "model_version": self.candidate.version, + "passed": self.eval_result.all_passed, + "compared_to_version": self.comparison.version if self.comparison else None, + "eval_anomaly_rate": float(self.prediction.anomaly_rate), + "silhouette_score": self.eval_result.metrics.silhouette_score, + "score_gap": self.eval_result.metrics.score_gap, + "eval_timestamp": self.feature_set.timestamp, + } + + # Register evaluation results linked to this specific model version + registry.register_evaluation( + self.prj, + model_name="anomaly_detector", + model_version=self.candidate.version, + passed=self.eval_result.all_passed, + metrics={ + "eval_anomaly_rate": float(self.prediction.anomaly_rate), + "silhouette_score": self.eval_result.metrics.silhouette_score, + "score_gap": self.eval_result.metrics.score_gap, + "eval_timestamp": self.feature_set.timestamp, + }, + eval_dataset=f"coingecko_live:{self.feature_set.timestamp}", + compared_to_version=self.comparison.version if self.comparison else None, + run_metadata={ + "evaluated_by_flow": current.flow_name, + "evaluated_by_run_id": current.run_id, + }, + ) + if self.eval_result.all_passed: - registry.update_status( - self.prj.asset, - "anomaly_detector", - new_status=registry.ModelStatus.EVALUATED, - current_annotations=self.candidate.annotations, - additional_annotations={ - "evaluated_by_flow": current.flow_name, - "evaluated_by_run_id": current.run_id, - "eval_anomaly_rate": float(self.prediction.anomaly_rate), - "eval_timestamp": self.feature_set.timestamp, - }, - ) - print(f"\nModel v{self.candidate.version}: candidate -> evaluated") + print(f"\nEvaluation PASSED for v{self.candidate.version}") - # Publish approval request event - registry.publish_event(self.prj, "approval_requested", payload={ - "model_asset": "anomaly_detector", - "candidate_version": self.candidate.version, + # Publish event for downstream (e.g., notifications, promotion workflows) + registry.publish_event(self.prj, "evaluation_passed", payload={ + "model_name": "anomaly_detector", + "model_version": self.candidate.version, + "compared_to_version": self.comparison.version if self.comparison else None, "anomaly_rate": float(self.prediction.anomaly_rate), "evaluation_run_id": current.run_id, }) - print("Published 'approval_requested' event") + print("Published 'evaluation_passed' event") else: - print(f"\nModel v{self.candidate.version} failed gates - status unchanged") + print(f"\nEvaluation FAILED for v{self.candidate.version}") self.next(self.end) @@ -225,10 +278,14 @@ def end(self): print("Evaluation Complete") print(f"{'='*50}") print(f"Model: anomaly_detector v{self.candidate.version}") + if self.comparison: + print(f"Compared to: v{self.comparison.version}") print(f"Result: {result}") print(f"Anomaly rate: {self.prediction.anomaly_rate:.1%}") + if self.eval_result.all_passed: - print(f"\nNext: Run PromoteDetectorFlow to promote to champion") + print(f"\nModel passed quality gates!") + print(f"To promote to champion, run PromoteFlow or use the dashboard.") if __name__ == "__main__": diff --git a/flows/promote/flow.py b/flows/promote/flow.py index 65baeb8..d40ee19 100644 --- a/flows/promote/flow.py +++ b/flows/promote/flow.py @@ -1,124 +1,177 @@ """ -Promote anomaly detector to champion status. - -This flow orchestrates: -1. Load model to promote -2. Assign champion status -3. Publish promotion event - -Triggering (OPTIONAL - human-in-the-loop): -- Default: Listens for 'model_approved' event via @project_trigger -- A human publishes this event after reviewing the evaluated model -- To make fully automated: replace @project_trigger with @trigger_on_finish -- To make manual-only: remove the @project_trigger decorator entirely - -Model Lifecycle: -- candidate: Newly trained, not yet evaluated -- evaluated: Passed point-in-time quality gates -- challenger: Running in parallel with champion for trial period -- champion: Primary model, blessed for serving -- retired: Previous champion, replaced by newer version - -Promotion Paths: -1. Direct: evaluated -> champion (skip challenger phase) - Use when: offline evaluation is sufficient, low-risk domain - -2. With trial: evaluated -> challenger -> champion - Use when: need online/batch comparison over time +Promote a model version to champion (or other alias). + +This flow demonstrates: +- Assigning aliases to model versions +- Requesting approval for production changes +- Demoting previous champion automatically + +Usage: + # Promote latest to champion + python flow.py run --version latest + + # Promote specific version + python flow.py run --version v5 + + # Promote to custom alias + python flow.py run --version latest --alias production """ from metaflow import step, Parameter from obproject import ProjectFlow -from obproject.project_events import project_trigger -# Import src at module level so Metaflow detects METAFLOW_PACKAGE_POLICY -# and includes it in the code package for remote execution import src -# OPTIONAL: Remove this decorator for manual-only promotion -# Or replace with @trigger_on_finish(flow='EvaluateDetectorFlow') for full automation -@project_trigger(event="model_approved") -class PromoteDetectorFlow(ProjectFlow): +class PromoteModelFlow(ProjectFlow): """ - Promote anomaly detector to champion status. - - Human-in-the-loop: waits for 'model_approved' event. + Promote a model version to champion or other alias. - Usage: - # Manual run (always works) - python flow.py run - python flow.py run --version v5 - - # Deploy to Argo (waits for event) - python flow.py --with retry argo-workflows create - - # Trigger via event (after human approval) - python -c "from obproject.project_events import ProjectEvent; \\ - ProjectEvent('model_approved', 'crypto_anomaly', 'main').publish()" + This assigns the specified alias to a model version, + automatically clearing it from any previous version. """ version = Parameter( "version", - default="latest", - help="Model version to promote (e.g., 'latest', 'v5')" + required=True, + help="Model version to promote (e.g., 'latest', 'v5', or version ID)" + ) + + alias = Parameter( + "alias", + default="champion", + help="Alias to assign (e.g., 'champion', 'production', 'staging')" + ) + + model_name = Parameter( + "model_name", + default="anomaly_detector", + help="Name of model asset" ) @step def start(self): - """Load model to promote.""" + """Load the model version to promote.""" from src import registry + from metaflow import Flow print(f"Project: {self.prj.project}, Branch: {self.prj.branch}") + print(f"\nPromoting {self.model_name} version '{self.version}' to '{self.alias}'") - # Load model to promote + # Load the model to verify it exists + # Handle different version formats: + # - "latest", "latest-N" -> pass directly to Asset API + # - bare run ID like "186088" -> load via Metaflow Client API + # - full version pathspec -> pass directly to Asset API try: - self.model_ref = registry.load_model( - self.prj.asset, "anomaly_detector", instance=self.version - ) + if self.version.startswith("latest"): + # Use Asset API for "latest" or "latest-N" formats + self.model = registry.load_model( + self.prj.asset, + self.model_name, + version=self.version + ) + elif self.version.isdigit() or len(self.version) < 20: + # Looks like a bare run ID - load via Metaflow Client API + print(f"Loading model from TrainDetectorFlow run: {self.version}") + flow = Flow('TrainDetectorFlow') + run = flow[self.version] + + if not run.successful: + raise ValueError(f"Run {self.version} was not successful") + + # Get model info from run artifacts + train_step = run['train'] + for task in train_step: + config = dict(task.data.model_config) + prediction = task.data.prediction + feature_set = task.data.feature_set + data_source = task.data.data_source + + # Create a simple namespace object to hold model info + from types import SimpleNamespace + self.model = SimpleNamespace( + version=run.id, + alias=None, + algorithm=config.get("algorithm", "isolation_forest"), + training_run_id=run.id, + annotations={ + "algorithm": config.get("algorithm", "isolation_forest"), + "training_run_id": run.id, + "anomaly_rate": float(prediction.anomaly_rate) if hasattr(prediction, 'anomaly_rate') else 0, + "training_samples": int(feature_set.n_samples) if hasattr(feature_set, 'n_samples') else 0, + "data_source": data_source, + } + ) + print(f"Loaded model info from run {run.id}") + break + else: + # Full version pathspec - use Asset API + self.model = registry.load_model( + self.prj.asset, + self.model_name, + version=self.version + ) except Exception as e: - print(f"\n[ERROR] No anomaly_detector found") - raise RuntimeError(f"Model not found: {e}") + raise RuntimeError(f"Model version '{self.version}' not found: {e}") print(f"\nModel to promote:") - print(f" Version: v{self.model_ref.version}") - print(f" Current status: {self.model_ref.status.value}") - print(f" Algorithm: {self.model_ref.annotations.get('algorithm')}") - - anomaly_rate = float(self.model_ref.annotations.get("anomaly_rate", 0)) - print(f" Anomaly rate: {anomaly_rate:.1%}") - - self.next(self.assign_champion) + print(f" Version: {self.model.version}") + print(f" Current alias: {self.model.alias or 'none'}") + print(f" Algorithm: {self.model.algorithm}") + print(f" Training run: {self.model.training_run_id}") + print(f" Anomaly rate: {float(self.model.annotations.get('anomaly_rate', 0)):.1%}") + + # Find the current champion via Metaflow run tags + self.previous_holder = None + current_champion = registry.get_champion_run_id(flow_name="TrainDetectorFlow") + if current_champion and current_champion != self.model.training_run_id: + self.previous_holder = current_champion + print(f"\nCurrent {self.alias}: run {current_champion}") + elif not current_champion: + print(f"\nNo current {self.alias} (first assignment)") + + self.next(self.promote) @step - def assign_champion(self): - """Assign champion status to model.""" + def promote(self): + """Promote a model by tagging its training run.""" from src import registry from metaflow import current - print(f"\nPromoting to champion...") + print(f"\nPromoting v{self.model.version} to '{self.alias}'...") + + # Use Metaflow's native tagging to mark the training run as champion + # This removes the tag from any previous champion and adds it to this run + try: + previous = registry.set_champion_run( + run_id=self.model.training_run_id, + flow_name="TrainDetectorFlow", + ) - registry.promote_to_champion( - self.prj.asset, - "anomaly_detector", - source_version=self.model_ref.version, - source_annotations=self.model_ref.annotations, - promotion_metadata={ - "promotion_flow": current.flow_name, - "promotion_run_id": current.run_id, - }, - ) + self.promotion_success = True + print(f"Successfully promoted run {self.model.training_run_id} to {self.alias}") + print(f" Tagged TrainDetectorFlow/{self.model.training_run_id} as '{self.alias}'") - print(f"\nPromoted anomaly_detector v{self.model_ref.version} to CHAMPION") + if previous: + print(f" Previous {self.alias} (run {previous}) has been untagged") - # Publish event + except Exception as e: + self.promotion_success = False + self.promotion_error = str(e) + print(f"[ERROR] Promotion failed: {e}") + import traceback + traceback.print_exc() + + # Publish event for notification/audit registry.publish_event(self.prj, "model_promoted", payload={ - "model_asset": "anomaly_detector", - "version": self.model_ref.version, - "status": "champion", - "anomaly_rate": self.model_ref.annotations.get("anomaly_rate"), - "algorithm": self.model_ref.annotations.get("algorithm"), + "model_name": self.model_name, + "model_version": self.model.version, + "alias": self.alias, + "previous_holder": self.previous_holder, + "promoted_by_run_id": current.run_id, }) + print(f"Published 'model_promoted' event") self.next(self.end) @@ -128,14 +181,16 @@ def end(self): print(f"\n{'='*50}") print("Promotion Complete") print(f"{'='*50}") - print(f"Model: anomaly_detector v{self.model_ref.version}") - print(f"Status: CHAMPION") - print(f"\nDeployment configs referencing 'champion' will now use this version.") - print(f"\nNext steps:") - print(" 1. Deployments using champion alias auto-update") - print(" 2. Or: Update pinned version in deployment config") - print(" 3. Monitor: GET /model/info") + print(f"Model: {self.model_name}") + print(f"Version: {self.model.version}") + print(f"Alias: {self.alias}") + + if hasattr(self, 'promotion_success') and self.promotion_success: + print(f"\nThe model is now the {self.alias}!") + else: + print(f"\nNote: Alias tag update may have failed.") + print("The 'model_promoted' event was still published for tracking.") if __name__ == "__main__": - PromoteDetectorFlow() + PromoteModelFlow() diff --git a/flows/train/flow.py b/flows/train/flow.py index 200a311..7a09d16 100644 --- a/flows/train/flow.py +++ b/flows/train/flow.py @@ -72,6 +72,8 @@ def _fetch_fresh_data(self): snapshot = data.fetch_market_data(num_coins=num_coins) self.feature_set = data.extract_features(snapshot) self.data_source = "coingecko_live" + self.data_snapshot_range = None + self.data_snapshot_count = None print(f"Fetched {self.feature_set.n_samples} coins") @@ -90,9 +92,16 @@ def _load_parquet_dataset(self, asset_name: str, version: str): # Convert to FeatureSet self.feature_set = table_to_feature_set(table, FEATURE_COLS) - self.data_source = f"{asset_name}:{version}" + + # Capture the actual resolved version from metadata (not the alias) + # This traces back to BuildDatasetFlow run that created this dataset + resolved_version = metadata.builder_run_id + self.data_source = f"{asset_name}:v{resolved_version}" + self.data_snapshot_range = metadata.snapshot_range + self.data_snapshot_count = metadata.snapshot_count print(f"Loaded {self.feature_set.n_samples} samples from {metadata.snapshot_count} snapshots") + print(f"Resolved dataset version: BuildDatasetFlow/{resolved_version}") print(f"Snapshot range: {metadata.snapshot_range[0][:19]} to {metadata.snapshot_range[1][:19]}") @card @@ -117,6 +126,8 @@ def train(self): # Load single snapshot artifact (backwards compatible) self.feature_set = self.prj.get_data(data_asset, instance=data_version) self.data_source = f"{data_asset}:{data_version}" + self.data_snapshot_range = (self.feature_set.timestamp, self.feature_set.timestamp) + self.data_snapshot_count = 1 print(f"Loaded FeatureSet: {self.feature_set.n_samples} samples x {self.feature_set.n_features} features") print(f"Timestamp: {self.feature_set.timestamp}") except Exception as e: @@ -182,7 +193,7 @@ def train(self): @step def register(self): - """Register trained model as candidate and log predictions.""" + """Register trained model and log predictions.""" from src import registry from src.predictions import PredictionRecord, PredictionStore from metaflow import current @@ -202,15 +213,21 @@ def register(self): "data_source": self.data_source, } + # Add data snapshot lineage if available + if self.data_snapshot_range: + annotations["data_snapshot_start"] = self.data_snapshot_range[0] + annotations["data_snapshot_end"] = self.data_snapshot_range[1] + if self.data_snapshot_count: + annotations["data_snapshot_count"] = self.data_snapshot_count + registry.register_model( self.prj, "anomaly_detector", - status=registry.ModelStatus.CANDIDATE, annotations=annotations, description=f"Anomaly detector: {self.prediction.anomaly_rate:.1%} rate on {self.feature_set.n_samples} coins", ) - print(f"\nRegistered anomaly_detector as CANDIDATE") + print(f"\nRegistered anomaly_detector (new version)") print(f" Algorithm: {self.trained_model.algorithm}") print(f" Data: {self.data_source}") print(f" Run ID: {current.run_id}") @@ -247,7 +264,7 @@ def end(self): print(f"\n{'='*50}") print("Training Complete") print(f"{'='*50}") - print(f"Model: anomaly_detector (status: candidate)") + print(f"Model: anomaly_detector") print(f"Algorithm: {self.trained_model.algorithm}") print(f"Data source: {self.data_source}") print(f"Anomaly rate: {self.prediction.anomaly_rate:.1%}") From 9f6dc4bded02b9a008263902591a4bb88486a958 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Mon, 1 Dec 2025 21:48:32 -0800 Subject: [PATCH 10/26] enhance dashboard with improved model and data views MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Models page: show champion status, version comparison - Data page: improved snapshot visualization - Overview: updated metrics display 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- deployments/dashboard/app.py | 563 ++++++++++++------ deployments/dashboard/templates/data.html | 44 +- deployments/dashboard/templates/models.html | 338 ++++++++--- deployments/dashboard/templates/overview.html | 21 +- 4 files changed, 660 insertions(+), 306 deletions(-) diff --git a/deployments/dashboard/app.py b/deployments/dashboard/app.py index 2c73f6a..cba173a 100644 --- a/deployments/dashboard/app.py +++ b/deployments/dashboard/app.py @@ -4,10 +4,14 @@ A web dashboard for monitoring the ML pipeline and model registry. Pages: -- / (Overview) - Pipeline health, champion model, latest evaluation +- / (Overview) - Pipeline health, latest model, latest evaluation - /data - Data explorer (snapshots, datasets) -- /models - Model registry (versions, compare, promote) -- /scanner - Live anomaly scanner +- /models - Model registry (versions, compare) +- /cards - Flow cards viewer + +Simplified model lifecycle: +- latest = candidate (just trained, under evaluation) +- latest-1 = champion (previous version, proven) """ from pathlib import Path @@ -41,23 +45,19 @@ def format_version(version_str): - """Format long version strings to a readable short form. - - Asset versions in obproject are typically task pathspecs: - - Format: v{timestamp}_task_{FlowName}_{run_id}_{step}_{task_id} - - Example: v098235478858_task_TrainDetectorFlow_185193_train_12345 - - Argo format: v{ts}_task_{FlowName}_argo_cryptoanomaly_prod_{wf}_{uuid} + """Format long version strings to a truncated form. - We extract meaningful identifiers for display. + Shows a shortened version for display, with full version available via tooltip. """ if not version_str: return "-" v = str(version_str) + # If it's a simple version like "v5" or "5", return as-is - if len(v) < 20: + if len(v) < 25: return f"v{v}" if not v.startswith("v") else v - # For task pathspec format: v{ts}_task_{Flow}_{run}_{step}_{task} + # For task pathspec format, extract Flow/run_id if "_task_" in v: parts = v.split("_") try: @@ -65,40 +65,20 @@ def format_version(version_str): flow_name = parts[task_idx + 1] if len(parts) > task_idx + 1 else "" run_id = parts[task_idx + 2] if len(parts) > task_idx + 2 else "" - # Clean up flow name (remove common suffixes for brevity) - short_flow = flow_name.replace("DetectorFlow", "").replace("AnomalyFlow", "").replace("MarketDataFlow", "Ingest").replace("DatasetFlow", "Dataset").replace("Flow", "") - if not short_flow: - short_flow = flow_name[:10] - - # For Argo runs, extract a more useful identifier + # For Argo runs, find a unique identifier if run_id == "argo": - # Argo format: argo_cryptoanomaly_prod_traindetectorflow_xyz123 - # Find the UUID at the end (last part that looks like a hash) - remaining = "_".join(parts[task_idx + 3:]) - # Try to find a short unique ID - look for UUID-like part or last segment + # Look for UUID-like part uuid_parts = [p for p in parts if len(p) >= 8 and "-" in p] if uuid_parts: - # Use first 8 chars of UUID - return f"{short_flow}/{uuid_parts[0][:8]}" - # Fallback: use timestamp from version - ts = parts[0][1:] if parts[0].startswith("v") else parts[0] - if len(ts) >= 12: - # Convert epoch-like timestamp to readable format - return f"{short_flow}/argo-{ts[-6:]}" - return f"{short_flow}/argo" + return f"{flow_name[:15]}/{uuid_parts[0][:8]}" + return f"{flow_name[:15]}/argo" else: - return f"{short_flow}/{run_id}" + return f"{flow_name[:15]}/{run_id}" except (ValueError, IndexError): pass - # Fallback for other formats - if "_" in v: - parts = v.split("_") - if len(parts) >= 4: - flow_name = parts[2] if len(parts) > 2 else "" - run_id = parts[3] if len(parts) > 3 else "" - return f"{flow_name[:12]}.../{run_id}" - return v[:16] + "..." + # Fallback: just truncate + return v[:25] + "..." def format_version_tooltip(version_str): @@ -179,27 +159,10 @@ def _int(val, default=0): return default -def find_champion_model(asset: Asset): - """Find champion model by checking if latest model has champion status.""" - try: - # Since there's only one version at a time (each registration replaces), - # just check if the latest model has champion status - from src import registry - ref = registry.load_model(asset, "anomaly_detector", instance="latest") - if ref.status == registry.ModelStatus.CHAMPION: - return { - "version": ref.version, - "status": ref.status.value, - "annotations": ref.annotations, - "tags": ref.tags, - } - except Exception: - pass - return None - - def get_pipeline_status(asset: Asset, branch: str): """Get pipeline health status for display.""" + from src import registry + status = { "project": PROJECT, "branch": branch, @@ -209,14 +172,12 @@ def get_pipeline_status(asset: Asset, branch: str): "overall": "unknown", } - # Check model + # Check model (latest = candidate) try: - from src import registry ref = registry.load_model(asset, "anomaly_detector", instance="latest") status["model"] = { "status": "healthy", "version": ref.version, - "model_status": ref.status.value, } except Exception: status["model"] = {"status": "no_model"} @@ -234,14 +195,14 @@ def get_pipeline_status(asset: Asset, branch: str): except Exception: status["data"] = {"status": "no_data"} - # Check champion - use tag-based lookup instead of instance="champion" - champion = find_champion_model(asset) - if champion: + # Check champion (latest-1) + try: + ref = registry.load_model(asset, "anomaly_detector", instance="latest-1") status["champion"] = { "status": "available", - "version": champion["version"], + "version": ref.version, } - else: + except Exception: status["champion"] = {"status": "none"} # Overall @@ -257,66 +218,250 @@ def get_pipeline_status(asset: Asset, branch: str): def get_model_versions(asset: Asset, limit=20): - """Get list of model versions.""" - from src import registry + """Get list of model versions from Metaflow training runs. + + Uses Metaflow Client API to fetch run artifacts directly. + This provides complete version history with all training metadata. + """ + from metaflow import Flow versions = [] - seen_versions = set() - # Get versions via latest-N iteration - for i in range(limit): - try: - instance = "latest" if i == 0 else f"latest-{i}" - ref = registry.load_model(asset, "anomaly_detector", instance=instance) + # Look up champion by alias tag + current_champion_run = get_champion_run_id() - if ref.version not in seen_versions: - ann = ref.annotations + # Get version history from Metaflow runs + try: + flow = Flow('TrainDetectorFlow') + for i, run in enumerate(flow.runs()): + if i >= limit: + break + + # Only include successful runs + if not run.successful: + continue + + run_id = run.id + + # Get run artifacts from the train step + try: + train_step = run['train'] + for task in train_step: + # Access artifacts + config = dict(task.data.model_config) + prediction = task.data.prediction + feature_set = task.data.feature_set + data_source = task.data.data_source + + hp = config.get('hyperparameters', {}) + + versions.append({ + "version": f"Train/{run_id}", + "alias": "champion" if str(run_id) == str(current_champion_run) else None, + "algorithm": config.get("algorithm", "isolation_forest"), + "anomaly_rate": _float(prediction.anomaly_rate) if hasattr(prediction, 'anomaly_rate') else None, + "training_samples": _int(feature_set.n_samples) if hasattr(feature_set, 'n_samples') else None, + "silhouette_score": None, # Computed in evaluate flow + "score_gap": None, + "training_timestamp": run.created_at.isoformat() if run.created_at else None, + "training_run_id": run_id, + "data_source": data_source, + "contamination": _float(hp.get("contamination")), + "n_estimators": _int(hp.get("n_estimators")), + }) + break # Only need first task + except Exception as e: + print(f"[DEBUG] Failed to get artifacts for run {run_id}: {e}") + # Add minimal entry for run without detailed artifacts versions.append({ - "version": ref.version, - "status": ref.status.value, - "algorithm": ann.get("algorithm"), - "anomaly_rate": _float(ann.get("anomaly_rate")), - "training_samples": _int(ann.get("training_samples")), - "silhouette_score": _float(ann.get("silhouette_score")) if ann.get("silhouette_score") else None, - "score_gap": _float(ann.get("score_gap")) if ann.get("score_gap") else None, - "training_timestamp": ann.get("training_timestamp"), + "version": f"Train/{run_id}", + "alias": "champion" if str(run_id) == str(current_champion_run) else None, + "algorithm": "isolation_forest", + "anomaly_rate": None, + "training_samples": None, + "silhouette_score": None, + "score_gap": None, + "training_timestamp": run.created_at.isoformat() if run.created_at else None, + "training_run_id": run_id, + "data_source": None, + "contamination": None, + "n_estimators": None, }) - seen_versions.add(ref.version) + + except Exception as e: + print(f"[DEBUG] Failed to get Metaflow runs: {e}") + # Fall back to just the latest model from asset + try: + ref = registry.load_model(asset, "anomaly_detector", version="latest") + ann = ref.annotations + versions.append({ + "version": ref.version, + "alias": ref.alias, + "algorithm": ann.get("algorithm"), + "anomaly_rate": _float(ann.get("anomaly_rate")), + "training_samples": _int(ann.get("training_samples")), + "silhouette_score": _float(ann.get("silhouette_score")) if ann.get("silhouette_score") else None, + "score_gap": _float(ann.get("score_gap")) if ann.get("score_gap") else None, + "training_timestamp": ann.get("training_timestamp"), + "training_run_id": ann.get("training_run_id"), + "data_source": ann.get("data_source"), + }) except Exception: - break + pass return versions def get_latest_evaluation(asset: Asset): - """Get latest evaluation result from model annotations.""" + """Get latest evaluation result from evaluation_results data asset.""" try: - # Evaluation metrics are stored in model annotations when model is evaluated - from src import registry - ref = registry.load_model(asset, "anomaly_detector", instance="latest") - ann = ref.annotations - - # Check if model has been evaluated (has eval_anomaly_rate annotation) - if not ann.get("eval_anomaly_rate"): - return None + ref = asset.consume_data_asset("evaluation_results", instance="latest") + props = ref.get("data_properties", {}) + ann = props.get("annotations", {}) - # Model was evaluated - determine if it passed based on status - status = ref.status.value - passed = status in ("evaluated", "champion") + passed = ann.get("passed", False) + if isinstance(passed, str): + passed = passed.lower() == "true" return { - "decision": "approve" if passed else "reject", - "candidate_version": ref.version, + "passed": passed, + "model_name": ann.get("model_name"), + "model_version": ann.get("model_version"), + "compared_to_version": ann.get("compared_to_version"), + "eval_dataset": ann.get("eval_dataset"), "anomaly_rate": _float(ann.get("eval_anomaly_rate")), - "silhouette_score": _float(ann.get("silhouette_score")) if ann.get("silhouette_score") else 0.0, - "score_gap": _float(ann.get("score_gap")) if ann.get("score_gap") else 0.0, - "gates_passed": 5 if passed else 0, # Approximate - actual gate count not stored - "gates_failed": 0 if passed else 1, + "silhouette_score": _float(ann.get("silhouette_score")), + "score_gap": _float(ann.get("score_gap")), + "evaluated_by_run_id": ann.get("evaluated_by_run_id"), } except Exception: return None +def get_outcome_metrics_for_model(model_version: str): + """Get outcome validation metrics for a model from ValidateOutcomesFlow runs. + + These are the ground-truth metrics that answer: "Did this model's predictions + actually predict crashes?" + + Returns dict with crash_recall, anomaly_precision, false_alarm_rate, etc. + """ + try: + from metaflow import Flow + + flow = Flow('ValidateOutcomesFlow') + + # Search for validation runs that validated this model version + for run in flow.runs(): + if not run.successful: + continue + + try: + # Check if this run validated the model we're interested in + end_step = run['end'] + for task in end_step: + if not hasattr(task.data, 'metrics') or task.data.metrics is None: + continue + + metrics = task.data.metrics + + # Check if this validation included our model version + # (ValidateOutcomesFlow validates predictions from multiple models) + if hasattr(task.data, 'outcomes'): + outcomes = task.data.outcomes + model_versions = set(o.model_version for o in outcomes if hasattr(o, 'model_version')) + + # Look for matching model version (e.g., "TrainDetectorFlow/186089") + version_match = any(model_version in v or v in model_version for v in model_versions) + if not version_match: + continue + + return { + "crash_recall": _float(metrics.crash_recall) if hasattr(metrics, 'crash_recall') else None, + "anomaly_precision": _float(metrics.anomaly_precision) if hasattr(metrics, 'anomaly_precision') else None, + "false_alarm_rate": _float(metrics.false_alarm_rate) if hasattr(metrics, 'false_alarm_rate') else None, + "total_predictions": _int(metrics.total_predictions) if hasattr(metrics, 'total_predictions') else None, + "total_crashes": _int(metrics.total_crashes) if hasattr(metrics, 'total_crashes') else None, + "validated_at": run.created_at.isoformat() if run.created_at else None, + "validation_run_id": run.id, + } + except Exception: + continue + + return None + except Exception as e: + print(f"[DEBUG] get_outcome_metrics error: {e}") + return None + + +def get_model_details(run_id: str, alias: str = None): + """Get detailed model info from a training run. + + Args: + run_id: The Metaflow run ID (e.g., "186088") + alias: Optional alias label for display (e.g., "champion", "candidate") + + Returns dict with: + - Model version and hyperparameters + - Training data source (resolved version, not alias) + - Data snapshot range and count + - Outcome validation metrics if available + """ + from metaflow import Flow + + try: + flow = Flow('TrainDetectorFlow') + run = flow[run_id] + + if not run.successful: + return None + + train_step = run['train'] + for task in train_step: + config = dict(task.data.model_config) + prediction = task.data.prediction + feature_set = task.data.feature_set + data_source = task.data.data_source + + hp = config.get('hyperparameters', {}) + + details = { + "version": f"Train/{run_id}", + "alias": alias, + "algorithm": config.get("algorithm"), + "n_estimators": _int(hp.get("n_estimators")), + "contamination": _float(hp.get("contamination")), + "anomaly_rate": _float(prediction.anomaly_rate) if hasattr(prediction, 'anomaly_rate') else None, + "training_samples": _int(feature_set.n_samples) if hasattr(feature_set, 'n_samples') else None, + "training_run_id": run_id, + "data_source": data_source, + "training_timestamp": run.created_at.isoformat() if run.created_at else None, + } + + # Get outcome validation metrics if available + outcome_metrics = get_outcome_metrics_for_model(f"TrainDetectorFlow/{run_id}") + if outcome_metrics: + details["outcome_metrics"] = outcome_metrics + + return details + + except Exception as e: + print(f"[DEBUG] get_model_details error for run {run_id}: {e}") + return None + + +def get_champion_run_id() -> str: + """Get the training run ID of the current champion model. + + Uses Metaflow's native tagging to find the run tagged 'champion'. + + Returns the run ID string, or None if no champion exists. + """ + from src import registry + + return registry.get_champion_run_id(flow_name="TrainDetectorFlow") + + # ============================================================================= # Common template context # ============================================================================= @@ -339,25 +484,28 @@ def get_template_context(request: Request, branch: str, **kwargs): @app.get("/", response_class=HTMLResponse) async def overview(request: Request): - """Overview page - pipeline health, champion model, latest evaluation.""" + """Overview page - pipeline health, latest model, latest evaluation.""" branch = get_branch_from_request(request) asset = get_asset_client(branch) status = get_pipeline_status(asset, branch) evaluation = get_latest_evaluation(asset) - # Get champion details using tag-based lookup + # Get champion details (latest-1) champion_details = None - champion = find_champion_model(asset) - if champion: - ann = champion.get("annotations", {}) + try: + from src import registry + ref = registry.load_model(asset, "anomaly_detector", instance="latest-1") + ann = ref.annotations champion_details = { - "version": champion["version"], + "version": ref.version, "algorithm": ann.get("algorithm"), "anomaly_rate": _float(ann.get("anomaly_rate")), "training_samples": _int(ann.get("training_samples")), "silhouette_score": _float(ann.get("silhouette_score")), } + except Exception: + pass return templates.TemplateResponse("overview.html", get_template_context( request, branch, @@ -394,34 +542,15 @@ async def data_explorer(request: Request): "message": "No data yet", }) - # Get snapshots from storage - snapshots = [] - try: - from src.storage import SnapshotStore - store = SnapshotStore(PROJECT, branch) - paths = store.list_snapshots(limit=24) - for path in paths: - parts = path.split("/") - date_part = next((p for p in parts if p.startswith("dt=")), None) - hour_part = next((p for p in parts if p.startswith("hour=")), None) - snapshots.append({ - "date": date_part[3:] if date_part else "unknown", - "hour": hour_part[5:] if hour_part else "unknown", - "filename": parts[-1], - }) - except Exception: - pass - return templates.TemplateResponse("data.html", get_template_context( request, branch, data_assets=data_assets, - snapshots=snapshots, )) @app.get("/models", response_class=HTMLResponse) async def model_registry(request: Request): - """Model registry page - versions, compare, promote.""" + """Model registry page - versions, compare.""" branch = get_branch_from_request(request) asset = get_asset_client(branch) @@ -434,20 +563,19 @@ async def model_registry(request: Request): error_message = f"Failed to load model versions: {str(e)}" print(f"[ERROR] {error_message}") - # Get champion version using tag-based lookup - champion_version = None - try: - champion = find_champion_model(asset) - if champion: - champion_version = champion["version"] - except Exception as e: - print(f"[ERROR] Failed to find champion: {e}") + # Get champion details with full lineage + champion_run_id = get_champion_run_id() + champion = get_model_details(champion_run_id, alias="champion") if champion_run_id else None + + # Count champions to validate uniqueness + champion_count = sum(1 for v in versions if v.get("alias") == "champion") return templates.TemplateResponse("models.html", get_template_context( request, branch, versions=versions, - champion_version=champion_version, error_message=error_message, + champion=champion, + champion_count=champion_count, )) @@ -461,9 +589,9 @@ async def model_registry(request: Request): FLOW_NAME_MAP = { "train": "TrainDetectorFlow", "evaluate": "EvaluateDetectorFlow", - "promote": "PromoteDetectorFlow", "ingest": "IngestMarketDataFlow", "build_dataset": "BuildDatasetFlow", + "validate_outcomes": "ValidateOutcomesFlow", } @@ -507,7 +635,7 @@ def get_card_html(flow_name: str, run_id: str = "latest", step_name: str = None, return None # Find a step with a card (typically 'train', 'evaluate', 'register', etc.) - card_steps = ['train', 'evaluate', 'register', 'build_and_write'] + card_steps = ['train', 'evaluate', 'register', 'build_and_write', 'compute_metrics'] for step in card_steps: try: step_obj = run[step] @@ -566,7 +694,7 @@ async def card_latest(flow_alias: str): Get the card HTML for the latest run of a flow. Args: - flow_alias: Short flow name (train, evaluate, promote, ingest, build_dataset) + flow_alias: Short flow name (train, evaluate, ingest, build_dataset, validate_outcomes) Returns: HTML content of the card, or 404 if not found @@ -588,7 +716,7 @@ async def card_by_run(flow_alias: str, run_id: str): Get the card HTML for a specific run of a flow. Args: - flow_alias: Short flow name (train, evaluate, promote, ingest, build_dataset) + flow_alias: Short flow name (train, evaluate, ingest, build_dataset) run_id: Metaflow run ID Returns: @@ -647,54 +775,133 @@ async def cards_page(request: Request): @app.get("/api/models/compare") async def api_compare(request: Request, v1: str, v2: str): - """Compare two model versions.""" - from src import registry + """Compare two model versions using Metaflow run data.""" + from metaflow import Flow + import numpy as np + + def get_version_data(version_str: str) -> dict: + """Extract full model data from a version string like 'vTrain/186089'. + + Returns all lineage info useful for comparison: + - Model hyperparameters + - Training data source (resolved version) + - Data snapshot range and count + - Quality metrics (score gap, anomaly rate) + - Outcome validation metrics (if available) + """ + # Parse version string (e.g., "vTrain/186089" -> run_id=186089) + v = version_str.lstrip('v') + if '/' in v: + _, run_id = v.split('/', 1) + else: + run_id = v - branch = get_branch_from_request(request) - asset = get_asset_client(branch) + flow = Flow('TrainDetectorFlow') + run = flow[run_id] - try: - ref1 = registry.load_model(asset, "anomaly_detector", instance=v1) - ref2 = registry.load_model(asset, "anomaly_detector", instance=v2) + train_step = run['train'] + for task in train_step: + config = dict(task.data.model_config) + prediction = task.data.prediction + feature_set = task.data.feature_set + data_source = task.data.data_source - def extract(ref): - ann = ref.annotations - return { - "version": ref.version, - "status": ref.status.value, - "algorithm": ann.get("algorithm"), - "anomaly_rate": _float(ann.get("anomaly_rate")), - "silhouette_score": _float(ann.get("silhouette_score")), - "score_gap": _float(ann.get("score_gap")), - "training_samples": _int(ann.get("training_samples")), + hp = config.get('hyperparameters', {}) + + # Compute quality metrics from scores + scores = prediction.anomaly_scores + preds = prediction.predictions + normal_scores = scores[preds == 1] + anomaly_scores = scores[preds == -1] + score_gap = float(normal_scores.mean() - anomaly_scores.mean()) if len(anomaly_scores) > 0 else None + + # Get training timestamp + created_at = run.created_at.isoformat() if run.created_at else None + + # Get data timestamp (when the training data was created) + data_timestamp = feature_set.timestamp if hasattr(feature_set, 'timestamp') else None + + # Get data snapshot range/count if available (from register step artifacts) + data_snapshot_start = None + data_snapshot_end = None + data_snapshot_count = None + try: + register_step = run['register'] + for reg_task in register_step: + if hasattr(reg_task.data, 'data_snapshot_range'): + snapshot_range = reg_task.data.data_snapshot_range + if snapshot_range: + data_snapshot_start = snapshot_range[0] + data_snapshot_end = snapshot_range[1] + if hasattr(reg_task.data, 'data_snapshot_count'): + data_snapshot_count = reg_task.data.data_snapshot_count + break + except Exception: + pass + + result = { + "version": f"Train/{run_id}", + "alias": None, + "algorithm": config.get("algorithm", "isolation_forest"), + "training_run_id": run_id, + "data_source": data_source, + "data_timestamp": data_timestamp, + "data_snapshot_start": data_snapshot_start, + "data_snapshot_end": data_snapshot_end, + "data_snapshot_count": data_snapshot_count, + "training_samples": feature_set.n_samples if hasattr(feature_set, 'n_samples') else None, + "anomaly_rate": float(prediction.anomaly_rate) if hasattr(prediction, 'anomaly_rate') else None, + "score_gap": score_gap, + "score_std": float(scores.std()) if len(scores) > 0 else None, + "contamination": float(hp.get("contamination")) if hp.get("contamination") else None, + "n_estimators": int(hp.get("n_estimators")) if hp.get("n_estimators") else None, + "created_at": created_at, } + # Get outcome validation metrics if available + outcome_metrics = get_outcome_metrics_for_model(f"TrainDetectorFlow/{run_id}") + if outcome_metrics: + result["outcome_metrics"] = outcome_metrics + + return result + + raise ValueError(f"No data found for run {run_id}") + + try: + data1 = get_version_data(v1) + data2 = get_version_data(v2) + return { - "v1": extract(ref1), - "v2": extract(ref2), + "v1": data1, + "v2": data2, } except Exception as e: raise HTTPException(status_code=404, detail=str(e)) @app.post("/api/models/promote") -async def api_promote(request: Request, version: str = Form(...)): - """Promote a model version to champion.""" - from obproject import ProjectEvent - - branch = get_branch_from_request(request) +async def api_promote(request: Request, version: str, alias: str = "champion"): + """Promote a model version to an alias (e.g., champion). - try: - event = ProjectEvent("model_approved", project=PROJECT, branch=branch) - event.publish(payload={ - "model_name": "anomaly_detector", - "version": version, - "triggered_by": "dashboard", - "timestamp": datetime.now(timezone.utc).isoformat(), - }) - return {"status": "success", "message": f"Promotion event published for v{version}"} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + NOTE: The Outerbounds Asset SDK does not currently support updating tags + on existing asset versions. In production, this would trigger the PromoteFlow + which re-registers the model with the alias tag set. + """ + # The Asset SDK doesn't support tag updates after registration. + # To promote a model, use the PromoteFlow: + # python flows/promote/flow.py run --version= --alias=champion + run_id = version.split('/')[-1] + command = f"python flows/promote/flow.py run --version={run_id} --alias={alias}" + return { + "success": False, + "message": ( + f"Tag updates not supported via API. " + f"To promote Train/{run_id} to {alias}, run:\n\n" + f"{command}" + ), + "action": "run_flow", + "command": command, + } # ============================================================================= diff --git a/deployments/dashboard/templates/data.html b/deployments/dashboard/templates/data.html index 94b1a92..9ddb1b3 100644 --- a/deployments/dashboard/templates/data.html +++ b/deployments/dashboard/templates/data.html @@ -144,46 +144,16 @@

Data Explorer

- +
-
Raw Parquet Snapshots
- {{ snapshots|length }} snapshots +
Data Lineage
- - {% if snapshots %} -
- - - - - - - - - - {% for snap in snapshots[:24] %} - - - - - - {% endfor %} - -
DateHourFilename
{{ snap.date }}{{ snap.hour }}:00{{ snap.filename }}
-
- {% if snapshots|length > 24 %} -
- Showing 24 of {{ snapshots|length }} snapshots -
- {% endif %} - {% else %} -
-
📁
-

No snapshots in storage yet

-

Run IngestFlow to create snapshots

-
- {% endif %} +

+ Data assets are registered by flows and stored in cloud object storage. + Each training run captures the exact dataset version used via the data_source annotation. + View the Model Registry to see which data version each model was trained on. +

diff --git a/deployments/dashboard/templates/models.html b/deployments/dashboard/templates/models.html index a909e58..f2fb71d 100644 --- a/deployments/dashboard/templates/models.html +++ b/deployments/dashboard/templates/models.html @@ -11,22 +11,92 @@

Model Registry

{% endif %} - -
+ +{% if champion %} +
-
Champion Model
- {% if champion_version %} - v{{ champion_version }} - {% else %} - None - {% endif %} +
Production Model (Champion)
+ {{ champion.alias or 'latest' }}
- {% if not champion_version %} -
- No champion model has been promoted yet. Select an evaluated model below and click "Promote" to set it as champion. + +
+ +
+

Model

+
+
Version: {{ champion.version|format_version }}
+
Algorithm: {{ champion.algorithm or 'isolation_forest' }}
+
Estimators: {{ champion.n_estimators or '-' }}
+
Contamination: {{ "%.0f"|format(champion.contamination * 100) if champion.contamination else '-' }}%
+
Anomaly Rate: {{ "%.1f"|format(champion.anomaly_rate * 100) if champion.anomaly_rate else '-' }}%
+
+
+ + +
+

Training Data

+
+
Dataset: {{ champion.data_source or '-' }}
+ {% if champion.data_snapshot_count %} +
Snapshots: {{ champion.data_snapshot_count }}
+ {% endif %} + {% if champion.data_snapshot_start and champion.data_snapshot_end %} +
Range: {{ champion.data_snapshot_start|format_timestamp }} to {{ champion.data_snapshot_end|format_timestamp }}
+ {% endif %} +
Samples: {{ champion.training_samples or '-' }}
+
+
+ + +
+

Outcome Validation

+ {% if champion.outcome_metrics %} +
+
Crash Recall: + {% if champion.outcome_metrics.crash_recall >= 0.5 %} + {{ "%.0f"|format(champion.outcome_metrics.crash_recall * 100) }}% + {% else %} + {{ "%.0f"|format(champion.outcome_metrics.crash_recall * 100) }}% + {% endif %} +
+
Precision: + {% if champion.outcome_metrics.anomaly_precision >= 0.3 %} + {{ "%.0f"|format(champion.outcome_metrics.anomaly_precision * 100) }}% + {% else %} + {{ "%.0f"|format(champion.outcome_metrics.anomaly_precision * 100) }}% + {% endif %} +
+
False Alarm Rate: {{ "%.0f"|format(champion.outcome_metrics.false_alarm_rate * 100) }}%
+
+ Validated: {{ champion.outcome_metrics.validated_at|format_timestamp }} +
+
+ {% else %} +
+ No outcome validation yet.
+ Run ValidateOutcomesFlow to measure real-world performance. +
+ {% endif %} +
- {% endif %}
+{% elif champion_count == 0 %} +
+
+
No Champion Model
+ Action Required +
+

+ No model is currently designated as champion. Promote a model version below to set it as the production model. +

+
+{% endif %} + +{% if champion_count > 1 %} +
+ Warning: {{ champion_count }} models have the "champion" alias. There should only be one champion per branch. +
+{% endif %}
@@ -38,7 +108,7 @@

Model Registry

@@ -47,7 +117,7 @@

Model Registry

@@ -72,14 +142,14 @@

Model Registry

- - - - + + + + - - - + + + @@ -87,52 +157,75 @@

Model Registry

+ - - - + + {% endfor %} @@ -154,20 +247,23 @@

Model Registry

- Candidate -
Just trained
+ Train +
New version created
→
- Evaluated -
Passed quality gates
+ Evaluate +
Quality gates
→
- Champion -
Production serving
+ Promote +
Assign alias
+

+ Aliases like "champion" or "production" are assigned via promotion. Any version can be loaded by version ID or alias. +

{% endblock %} @@ -190,52 +286,109 @@

Model Registry

if (response.ok) { const resultsDiv = document.getElementById('comparison-results'); resultsDiv.style.display = 'block'; + // Format timestamp for display + const formatTime = (ts) => ts ? ts.replace('T', ' ').substring(0, 19) : '-'; + resultsDiv.innerHTML = `
VersionStatusAlgorithmTraining SamplesVersionAliasEstimatorsContamination Anomaly RateSilhouetteScore GapActionsTraining SamplesData SourceActions
{{ v.version|format_version }} - {% if v.version == champion_version %} - Champion + + {% if v.alias == 'champion' %} + {{ v.alias }} + {% elif v.alias == 'candidate' %} + {{ v.alias }} + {% elif v.alias == 'staging' %} + {{ v.alias }} + {% elif v.alias %} + {{ v.alias }} + {% else %} + - {% endif %} - {% if v.status == 'champion' %} - {{ v.status }} - {% elif v.status == 'evaluated' %} - {{ v.status }} - {% elif v.status == 'candidate' %} - {{ v.status }} + {% if v.n_estimators %} + {{ v.n_estimators }} {% else %} - {{ v.status }} + - {% endif %} {{ v.algorithm or '-' }}{{ v.training_samples or '-' }}{{ "%.1f"|format(v.anomaly_rate * 100) }}% - {% if v.silhouette_score is not none %} - {{ "%.3f"|format(v.silhouette_score) }} + {% if v.contamination %} + {{ "%.0f"|format(v.contamination * 100) }}% {% else %} - {% endif %} - {% if v.score_gap is not none %} - {{ "%.3f"|format(v.score_gap) }} + {% if v.anomaly_rate is not none %} + {{ "%.1f"|format(v.anomaly_rate * 100) }}% {% else %} - {% endif %} {{ v.training_samples or '-' }} - {% if v.status == 'evaluated' and v.version != champion_version %} -
- - -
- {% elif v.version == champion_version %} - Current champion - {% elif v.status == 'candidate' %} - Awaiting evaluation + {% if v.data_source %} + {% if ':v' in v.data_source %} + {# Show asset:version format nicely - extract just the version part for display #} + {% set parts = v.data_source.split(':v') %} + + {{ parts[0][:12] }}:v{{ parts[1] }} + + {% elif ':' in v.data_source and 'latest' in v.data_source %} + {# Unresolved alias like "training_dataset:latest" - show warning #} + {% set parts = v.data_source.split(':') %} + + {{ parts[0][:8] }}:{{ parts[1] }} + + {% else %} + {# Simple value like "coingecko_live" #} + + {{ v.data_source }} + + {% endif %} {% else %} - {% endif %} 		+ {% if v.alias != 'champion' %} + + {% else %} + Current + {% endif %} +
- - + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - + + + + + ${data.v1.data_snapshot_count || data.v2.data_snapshot_count ? ` - - - - + + + + + + ` : ''} + + + + + + + + + - - - - + + + + - - - - + + + +
Metric${data.v1.version}${data.v2.version}${data.v1.version}${data.v1.alias ? ' (' + data.v1.alias + ')' : ''}${data.v2.version}${data.v2.alias ? ' (' + data.v2.alias + ')' : ''} Difference
Status${data.v1.status}${data.v2.status}Hyperparameters
n_estimators${data.v1.n_estimators || '-'}${data.v2.n_estimators || '-'}${data.v1.n_estimators && data.v2.n_estimators ? (data.v1.n_estimators - data.v2.n_estimators > 0 ? '+' : '') + (data.v1.n_estimators - data.v2.n_estimators) : '-'}
contamination${data.v1.contamination ? (data.v1.contamination * 100).toFixed(0) + '%' : '-'}${data.v2.contamination ? (data.v2.contamination * 100).toFixed(0) + '%' : '-'}${data.v1.contamination && data.v2.contamination ? ((data.v1.contamination - data.v2.contamination) * 100).toFixed(0) + '%' : '-'}
Quality Metrics
Score Gap (separation)${data.v1.score_gap ? data.v1.score_gap.toFixed(3) : '-'}${data.v2.score_gap ? data.v2.score_gap.toFixed(3) : '-'}${data.v1.score_gap && data.v2.score_gap ? (data.v1.score_gap - data.v2.score_gap > 0 ? '+' : '') + (data.v1.score_gap - data.v2.score_gap).toFixed(3) : '-'}
Anomaly Rate${data.v1.anomaly_rate ? (data.v1.anomaly_rate * 100).toFixed(1) + '%' : '-'}${data.v2.anomaly_rate ? (data.v2.anomaly_rate * 100).toFixed(1) + '%' : '-'}(= contamination)
Training Samples${data.v1.training_samples || '-'}${data.v2.training_samples || '-'}${data.v1.training_samples && data.v2.training_samples ? data.v1.training_samples - data.v2.training_samples : '-'}
Lineage
Training Run${data.v1.training_run_id || '-'}${data.v2.training_run_id || '-'} -
Algorithm${data.v1.algorithm || '-'}${data.v2.algorithm || '-'}Trained At${formatTime(data.v1.created_at)}${formatTime(data.v2.created_at)} -
Training Samples${data.v1.training_samples}${data.v2.training_samples}${data.v1.training_samples - data.v2.training_samples}Data Snapshot${formatTime(data.v1.data_timestamp)}${formatTime(data.v2.data_timestamp)}${data.v1.data_timestamp === data.v2.data_timestamp ? 'Same data' : 'Different'}
Anomaly Rate${(data.v1.anomaly_rate * 100).toFixed(1)}%${(data.v2.anomaly_rate * 100).toFixed(1)}%${((data.v1.anomaly_rate - data.v2.anomaly_rate) * 100).toFixed(1)}%Snapshot Count${data.v1.data_snapshot_count || '-'}${data.v2.data_snapshot_count || '-'}-
Outcome Validation
Crash Recall${data.v1.outcome_metrics ? (data.v1.outcome_metrics.crash_recall * 100).toFixed(0) + '%' : 'Not validated'}${data.v2.outcome_metrics ? (data.v2.outcome_metrics.crash_recall * 100).toFixed(0) + '%' : 'Not validated'}${data.v1.outcome_metrics && data.v2.outcome_metrics ? ((data.v1.outcome_metrics.crash_recall - data.v2.outcome_metrics.crash_recall) * 100).toFixed(0) + '%' : '-'}
Silhouette Score${data.v1.silhouette_score?.toFixed(3) || '-'}${data.v2.silhouette_score?.toFixed(3) || '-'}${data.v1.silhouette_score && data.v2.silhouette_score ? (data.v1.silhouette_score - data.v2.silhouette_score).toFixed(3) : '-'}Anomaly Precision${data.v1.outcome_metrics ? (data.v1.outcome_metrics.anomaly_precision * 100).toFixed(0) + '%' : '-'}${data.v2.outcome_metrics ? (data.v2.outcome_metrics.anomaly_precision * 100).toFixed(0) + '%' : '-'}${data.v1.outcome_metrics && data.v2.outcome_metrics ? ((data.v1.outcome_metrics.anomaly_precision - data.v2.outcome_metrics.anomaly_precision) * 100).toFixed(0) + '%' : '-'}
Score Gap${data.v1.score_gap?.toFixed(3) || '-'}${data.v2.score_gap?.toFixed(3) || '-'}${data.v1.score_gap && data.v2.score_gap ? (data.v1.score_gap - data.v2.score_gap).toFixed(3) : '-'}False Alarm Rate${data.v1.outcome_metrics ? (data.v1.outcome_metrics.false_alarm_rate * 100).toFixed(0) + '%' : '-'}${data.v2.outcome_metrics ? (data.v2.outcome_metrics.false_alarm_rate * 100).toFixed(0) + '%' : '-'}${data.v1.outcome_metrics && data.v2.outcome_metrics ? ((data.v1.outcome_metrics.false_alarm_rate - data.v2.outcome_metrics.false_alarm_rate) * 100).toFixed(0) + '%' : '-'}
@@ -247,5 +400,38 @@

Model Registry

alert('Error: ' + err.message); } } + +async function promoteModel(version) { + const branch = new URLSearchParams(window.location.search).get('branch') || '{{ branch }}'; + + if (!confirm(`Promote version ${version} to champion?`)) { + return; + } + + try { + const response = await fetch(`/api/models/promote?version=${encodeURIComponent(version)}&alias=champion&branch=${branch}`, { + method: 'POST', + }); + const data = await response.json(); + + if (data.success) { + alert('Model promoted to champion! Refreshing...'); + window.location.reload(); + } else if (data.message) { + // Show the message with command to run + const msg = data.message + (data.command ? '\n\nCommand copied to clipboard.' : ''); + alert(msg); + + // Copy command to clipboard if available + if (data.command && navigator.clipboard) { + navigator.clipboard.writeText(data.command).catch(() => {}); + } + } else { + alert('Promotion failed: ' + (data.detail || 'Unknown error')); + } + } catch (err) { + alert('Error: ' + err.message); + } +} {% endblock %} diff --git a/deployments/dashboard/templates/overview.html b/deployments/dashboard/templates/overview.html index 42431bd..c800d73 100644 --- a/deployments/dashboard/templates/overview.html +++ b/deployments/dashboard/templates/overview.html @@ -109,18 +109,18 @@

Project Overview

Latest Evaluation
{% if evaluation %} - {% if evaluation.decision == 'approve' %} - Approved + {% if evaluation.passed %} + Passed {% else %} - Rejected + Failed {% endif %} {% endif %}
{% if evaluation %}
- Candidate Version - {{ evaluation.candidate_version|format_version }} + Model Version + {{ (evaluation.model_version or '-')|format_version }}
Anomaly Rate @@ -132,16 +132,7 @@

Project Overview

Score Gap - {{ "%.3f"|format(evaluation.score_gap) }} -
-
- Quality Gates - - {{ evaluation.gates_passed }} passed - {% if evaluation.gates_failed > 0 %} - / {{ evaluation.gates_failed }} failed - {% endif %} - + {{ "%.3f"|format(evaluation.score_gap) if evaluation.score_gap else '-' }}
{% else %}
From 8023813598a9a5d446c9cad24ab0d9b939d1c2d0 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Mon, 1 Dec 2025 21:48:36 -0800 Subject: [PATCH 11/26] add aggressive and conservative model config presets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- configs/model_aggressive.json | 9 +++++++++ configs/model_conservative.json | 9 +++++++++ 2 files changed, 18 insertions(+) create mode 100644 configs/model_aggressive.json create mode 100644 configs/model_conservative.json diff --git a/configs/model_aggressive.json b/configs/model_aggressive.json new file mode 100644 index 0000000..2a0abbd --- /dev/null +++ b/configs/model_aggressive.json @@ -0,0 +1,9 @@ +{ + "algorithm": "isolation_forest", + "hyperparameters": { + "n_estimators": 200, + "contamination": 0.15, + "random_state": 42, + "n_jobs": -1 + } +} diff --git a/configs/model_conservative.json b/configs/model_conservative.json new file mode 100644 index 0000000..1252422 --- /dev/null +++ b/configs/model_conservative.json @@ -0,0 +1,9 @@ +{ + "algorithm": "isolation_forest", + "hyperparameters": { + "n_estimators": 50, + "contamination": 0.05, + "random_state": 42, + "n_jobs": -1 + } +} From f86baaab767d0fffdd4d80f48b23bdb4ae96cca8 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Mon, 1 Dec 2025 21:48:40 -0800 Subject: [PATCH 12/26] docs: simplify README for demo users MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Streamline documentation with clearer quick start, project structure, and champion management explanation. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- README.md | 344 +++++++++++++++++------------------------------------- 1 file changed, 104 insertions(+), 240 deletions(-) diff --git a/README.md b/README.md index 0fe2796..92806e9 100644 --- a/README.md +++ b/README.md @@ -1,311 +1,175 @@ # Crypto Market Anomaly Detection -Real-time anomaly detection on live cryptocurrency market data from CoinGecko. +An end-to-end ML project demonstrating **model registry patterns** with Outerbounds Projects. Detects anomalies in live cryptocurrency market data from CoinGecko. -## Model Lifecycle - -This project uses the **champion/challenger** pattern for model lifecycle: - -| Status | Meaning | How it's set | -|--------|---------|--------------| -| `candidate` | Newly trained, not yet evaluated | TrainDetectorFlow | -| `evaluated` | Passed point-in-time quality gates | EvaluateDetectorFlow | -| `challenger` | Running in parallel with champion | Deployment config | -| `champion` | Primary model, blessed for serving | PromoteDetectorFlow | -| `retired` | Previous champion, replaced | (when new champion is promoted) | +## Quick Start -The idea that will become important is to view these states as ML Asset quality designations, and not confuse such status tags with deployment environments. -- A `champion` model can be deployed to dev, staging, OR prod -- Deployment configs can then reference models by alias (`champion`) or version (`v5`) -- The intent is clean separation of "is this model performant enough to serve production traffic?" from "which code/project branch is this model on?" - -### Evaluated vs Challenger +```bash +# Setup +cd model-registry-project-v1 +export PYTHONPATH="$PWD" +export METAFLOW_PROFILE=yellow # or your profile -The distinction matters: +# Train a model (fetches fresh data) +python flows/train/flow.py run -- **`evaluated`**: Point-in-time assessment. A single evaluation run passed quality gates. This is necessary but may not be sufficient in most real-world systems. -- **`challenger`**: Trial period. The model runs in parallel with the champion over time, allowing comparison of Evals.log streams. This applies to BOTH: - - **Batch workflows**: Parallel daily/hourly runs with different model refs - - **API deployments**: Traffic split between deployment variants +# Evaluate the model +python flows/evaluate/flow.py run +# Promote to champion +python flows/promote/flow.py run --run_id ``` -candidate ──► evaluated ──► challenger ──► champion - │ │ - Point-in-time Trial period - (single run) (parallel runs) -``` + ## Project Structure ``` -├── src/ # Core operational logic (reusable) -│ ├── data.py # Data fetching, feature engineering -│ ├── model.py # Model training, inference -│ ├── registry.py # Asset registration, versioning -│ └── eval.py # Evaluation, quality gates +├── src/ # Reusable ML logic +│ ├── data.py # Data fetching from CoinGecko +│ ├── features.py # Feature engineering +│ ├── model.py # Isolation Forest training/inference +│ ├── registry.py # Asset registration, champion management +│ └── eval.py # Evaluation metrics, quality gates │ -├── flows/ # Thin orchestration layers -│ ├── ingest/flow.py # Orchestrates: fetch → extract → register DataAsset -│ ├── train/flow.py # Orchestrates: load data → train → register ModelAsset -│ ├── evaluate/flow.py # Orchestrates: load → predict → gates → update -│ └── promote/flow.py # Orchestrates: load → promote → publish +├── flows/ # Metaflow orchestration +│ ├── ingest/flow.py # Fetch → register DataAsset +│ ├── build_dataset/flow.py # Accumulate snapshots → train/eval split +│ ├── train/flow.py # Train → register ModelAsset +│ ├── evaluate/flow.py # Evaluate → quality gates +│ ├── promote/flow.py # Promote → champion tag +│ └── validate_outcomes/ # Production validation │ ├── deployments/ -│ └── api/app.py # FastAPI service (uses src modules) +│ └── dashboard/ # FastAPI dashboard for monitoring │ -├── data/ # DataAsset configs -│ └── market_snapshot/ -└── models/ # ModelAsset configs - └── anomaly_detector/ -``` - -### Why `/src` at project root? - -Separating operational logic from orchestration enables: - -1. **Dependency injection** - Config files can specify which code paths to use -2. **Dynamic loading** - Different asset states trigger different behaviors -3. **Testability** - Test workflows independently of flow orchestration -4. **Reusability** - Same logic in flows, API, and notebooks - -```python -# In a flow -from src import data, model, registry -snapshot = data.fetch_market_data() -feature_set = data.extract_features(snapshot) -trained, result = model.train(feature_set) -registry.register_model(prj, "anomaly_detector", ...) - -# In API -from src import data, model -snapshot = data.fetch_market_data() -prediction = model.predict_fresh(config, data.extract_features(snapshot)) - -# In notebook -from src import data -snapshot = data.fetch_market_data() -data.get_top_movers(snapshot) -``` - -## ML Lifecycle - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ ML LIFECYCLE │ -│ │ -│ TrainFlow ──────► EvaluateFlow ──────► PromoteFlow │ -│ │ │ │ │ -│ ▼ ▼ ▼ │ -│ candidate evaluated champion │ -│ (new model) (gates passed) (blessed for │ -│ serving) │ -│ │ -│ Optional: Deploy as 'challenger' for trial period │ -│ before promoting to champion │ -└─────────────────────────────────────────────────────────────────┘ - -┌─────────────────────────────────────────────────────────────────┐ -│ DEPLOYMENT CONFIGS │ -│ (in git branches) │ -│ │ -│ git:main → model_ref: "anomaly_detector:latest" │ -│ git:staging → model_ref: "anomaly_detector:champion" │ -│ git:prod → model_ref: "anomaly_detector:v5" (pinned) │ -└─────────────────────────────────────────────────────────────────┘ +├── configs/ # Environment-specific configs +│ ├── model.json +│ ├── training.json +│ └── evaluation.json +│ +└── obproject.toml # Project configuration ``` ## Flows ### IngestMarketDataFlow -Fetches live data, registers versioned `market_snapshot` DataAsset: +Fetches live crypto prices and registers a `market_snapshot` DataAsset. ```bash python flows/ingest/flow.py run -python flows/ingest/flow.py run --num_coins 200 ``` -### TrainDetectorFlow +### BuildDatasetFlow -Trains model, registers as `candidate`. Can use registered data or fetch fresh: +Accumulates snapshots into train/eval datasets with time-based splits. ```bash -# Quick testing (fetch fresh data) -python flows/train/flow.py run --contamination 0.10 +# Default: 168hr history, 24hr holdout +python flows/build_dataset/flow.py run -# Production (use registered data asset for lineage) -python flows/train/flow.py run --data_version latest +# Quick test (smaller holdout) +python flows/build_dataset/flow.py run --holdout_hours 1 ``` -### EvaluateDetectorFlow +### TrainDetectorFlow -Evaluates candidate, applies quality gates, updates status to `evaluated`: +Trains an Isolation Forest model and registers it as a ModelAsset. ```bash -python flows/evaluate/flow.py run --max_anomaly_rate 0.20 +# Use accumulated dataset +python flows/train/flow.py run + +# Quick test with fresh data (no dataset lineage) +python flows/train/flow.py run --fresh_data ``` -### PromoteDetectorFlow +### EvaluateDetectorFlow -Promotes model to `champion` status: +Evaluates the latest model against quality gates. ```bash -python flows/promote/flow.py run --version latest +python flows/evaluate/flow.py run ``` -## API - -The API tries to load `champion` first, falls back to `latest`: +### PromoteModelFlow -| Endpoint | Description | -|----------|-------------| -| `POST /scan` | Scan market for anomalies | -| `GET /anomalies` | Get detected anomalies | -| `GET /market` | Market overview | -| `GET /model/info` | Model info (status: candidate/evaluated/champion) | -| `POST /model/reload` | Hot-reload model | -| `GET /versions` | List all versions | - -## Data Source - -**CoinGecko API** - Free, no API key, updates every few minutes. - -## Scheduling & Event Triggering - -When deployed to Argo Workflows, flows trigger automatically: - -``` -@schedule(hourly) @trigger_on_finish @trigger_on_finish - │ │ │ - ▼ ▼ ▼ - IngestFlow ──────────────► TrainFlow ──────────────► EvaluateFlow - │ - publish_event("approval_requested") - │ - [HUMAN REVIEW] - │ - @project_trigger("model_approved") - ▼ - PromoteFlow -``` - -### Deploy to Argo +Promotes a model to champion using Metaflow run tags. ```bash -# Deploy all flows (one-time) -python flows/ingest/flow.py --with retry argo-workflows create -python flows/train/flow.py --with retry argo-workflows create -python flows/evaluate/flow.py --with retry argo-workflows create -python flows/promote/flow.py --with retry argo-workflows create +python flows/promote/flow.py run --run_id ``` -### Trigger Promotion (after human review) +## Model Lifecycle -```bash -python -c "from obproject.project_events import ProjectEvent; \ - ProjectEvent('model_approved', 'crypto_anomaly', 'main').publish()" +``` +TrainFlow ──────► EvaluateFlow ──────► PromoteFlow + │ │ │ + ▼ ▼ ▼ +candidate evaluated champion +(new model) (gates passed) (tagged for serving) ``` -### Making Promotion Fully Automated +### Champion Management -To skip human approval, edit `flows/promote/flow.py`: +Champions are tracked via **Metaflow run tags**, not Asset API tags: ```python -# Replace: -@project_trigger(event="model_approved") +from metaflow import Flow, Run -# With: -@trigger_on_finish(flow='EvaluateDetectorFlow') -``` - -### Making Promotion Manual-Only - -Remove the `@project_trigger` decorator entirely from PromoteFlow. +# Find current champion +for run in Flow("TrainDetectorFlow").runs("champion"): + print(f"Champion: {run.id}") + break -## Human-in-the-Loop Stages - -Two stages require human judgment (not automated by default): +# Promote a new champion +Run(f"TrainDetectorFlow/{run_id}").add_tag("champion") +``` -| Stage | Who | Decision | -|-------|-----|----------| -| **Promote to Champion** | ML Engineer | "Is this model ready?" (quality/business judgment) | -| **Challenger Deployment** | Platform/MLOps | "Deploy for A/B testing?" (infrastructure concern) | +This enables efficient server-side queries to find the current champion model. -These are intentionally separate from flow automation because: -- Champion promotion is a **business decision** requiring domain context -- Challenger deployment is an **infrastructure task** (traffic splitting, monitoring)—not a flow +## Configuration -## System Extensions (Quick-Start Notes) +### obproject.toml -For teams looking to extend this system: +```toml +project = "crypto_anomaly" +title = "Crypto Market Anomaly Detection" -### Approval Queue/Dashboard +# Map branches to environments +[branch_to_environment] +"main" = "production" +"*" = "dev" -**Simplest approach:** Use Slack workflow or a shared spreadsheet. +# Environment-specific configs +[environments.production.flow_configs] +model_config = "configs/model.json" +training_config = "configs/training.json" +``` -1. EvaluateFlow's `approval_requested` event triggers a Slack message (via Argo notification) -2. Message includes: model version, anomaly rate, link to UI -3. Reviewer clicks "Approve" button → runs a Slack workflow that calls: - ```bash - curl -X POST your-webhook/approve?version=v5 - ``` -4. Webhook publishes `model_approved` event +### Local Development -**Why not build a queue?** Events are fire-and-forget. Building state tracking adds complexity. Slack/JIRA already handle approval workflows well. +For local testing, read and write use your user branch. For production-like behavior (read from main, write to your branch): -### Challenger Traffic Splitting +```toml +[dev-assets] +branch = "main" +``` -**Simplest approach:** Two deployment configs, manual traffic routing. +## Deployment -1. Create `deployments/api-challenger/` with `model_ref: evaluated` -2. Deploy both APIs to separate endpoints -3. Use nginx/envoy/cloud load balancer to split traffic (e.g., 90/10) -4. Monitor both via existing `/model/info` endpoint -5. After trial period, promote challenger and remove the deployment +### Deploy Flows to Argo -**Why not automate?** Traffic splitting is infrastructure-specific (K8s Istio, AWS ALB, etc.). Keep flows focused on ML logic. +```bash +python flows/ingest/flow.py argo-workflows create +python flows/train/flow.py argo-workflows create +python flows/evaluate/flow.py argo-workflows create +python flows/promote/flow.py argo-workflows create +``` -### Rollback Automation +### Dashboard -**Simplest approach:** Manual PromoteFlow with previous version. +The FastAPI dashboard in `deployments/dashboard/` displays model performance and anomaly predictions. -```bash -# Rollback to previous champion -python flows/promote/flow.py run --version v4 -``` +## Data Source -For automated rollback on drift detection: -1. Create `MonitorFlow` that runs hourly, checks model performance -2. If degraded, publish `rollback_requested` event with `--version` in payload -3. PromoteFlow listens for this event (add second trigger) - -**Why start manual?** Automated rollback requires clear rollback criteria. Start with manual, learn what triggers rollback, then automate. - -## Customer Requirements Mapping - -| Requirement | Implementation | -|-------------|----------------| -| Create model | `models/anomaly_detector/asset_config.toml` | -| Create version | `prj.register_model()` in TrainFlow | -| Link to training job | Annotations: `training_run_id`, `training_flow` | -| Link to training metrics | Annotations: `anomaly_rate`, etc. | -| Link to training dataset | Annotations: `data_source`, `training_timestamp` | -| Link to evaluation job | `Evals.log()` or evaluation_results asset | -| Link to evaluation dataset | Evaluation annotations | -| **Assign status** | `tags: {status: "candidate/evaluated/champion"}` | -| Download by alias | `consume_model_asset(instance="champion")` | -| Request approval | `publish_event("approval_requested")` | -| Compare metrics | Evals comparison / EvaluateFlow card | -| Trial period | Deploy as challenger with parallel execution | - -## Why Champion, Not Production? - -The word "production" conflates two concepts: - -1. **ML Quality**: "Is this model good enough?" -2. **Deployment Environment**: "Where is this code running?" - -By using `champion`: -- It's clear this is an ML quality designation -- Deployment configs can reference `champion` from any environment -- No confusion between model status and deploy environment -- Clean mental model for MLOps professionals +**CoinGecko API** - Free tier, no API key required, updates every few minutes. From ed61be5ffc1048cecfed971d446c9037d213950a Mon Sep 17 00:00:00 2001 From: Eddie Mattia <40632488+emattia@users.noreply.github.com> Date: Mon, 1 Dec 2025 21:50:19 -0800 Subject: [PATCH 13/26] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 92806e9..0945d4a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Crypto Market Anomaly Detection +System designs (1) An end-to-end ML project demonstrating **model registry patterns** with Outerbounds Projects. Detects anomalies in live cryptocurrency market data from CoinGecko. From aa6cdb8c1eae6c52cb007f172a8c8442a65db3e0 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Mon, 1 Dec 2025 21:53:48 -0800 Subject: [PATCH 14/26] fix: add alias property to ModelRef MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add missing alias property that evaluate flow references. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/registry.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/registry.py b/src/registry.py index 0d799d5..aacf11b 100644 --- a/src/registry.py +++ b/src/registry.py @@ -48,6 +48,11 @@ def training_flow(self) -> Optional[str]: def data_source(self) -> Optional[str]: return self.annotations.get("data_source") + @property + def alias(self) -> Optional[str]: + """Alias is determined by Metaflow run tags, not stored in Asset.""" + return None # Use get_champion_run_id() to check if this is champion + def _get_asset(prj_or_asset): """Get Asset from ProjectContext or return Asset directly.""" From 8a3c011b45299a20dc248b634e9e9abfef6c582d Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Tue, 2 Dec 2025 10:25:22 -0800 Subject: [PATCH 15/26] feat: use holdout dataset for reproducible evaluation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EvaluateDetectorFlow now loads eval_holdout asset by default instead of fetching fresh data. This provides: - Reproducible evaluations (same holdout set across runs) - Proper train/eval separation - Full lineage from eval results back to BuildDatasetFlow Config vs Parameter override pattern: - eval_config sets defaults at deployment time - Parameters (eval_data_source, eval_data_version) can override at runtime Usage: # Default: use holdout set python flow.py run # Override to fresh data at runtime python flow.py run --eval_data_source fresh 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- configs/evaluation.json | 2 + flows/evaluate/flow.py | 102 +++++++++++++++++++++++++++++++++------- 2 files changed, 88 insertions(+), 16 deletions(-) diff --git a/configs/evaluation.json b/configs/evaluation.json index 826bebf..dd8f76c 100644 --- a/configs/evaluation.json +++ b/configs/evaluation.json @@ -1,4 +1,6 @@ { + "eval_data_source": "eval_holdout", + "eval_data_version": "latest", "max_anomaly_rate": 0.20, "min_anomaly_rate": 0.02, "max_rate_diff": 0.25, diff --git a/flows/evaluate/flow.py b/flows/evaluate/flow.py index 6072163..fa757c5 100644 --- a/flows/evaluate/flow.py +++ b/flows/evaluate/flow.py @@ -1,17 +1,30 @@ """ -Evaluate anomaly detection model on fresh market data. +Evaluate anomaly detection model on holdout data. This flow: 1. Loads the candidate model (configurable version, default=latest) 2. Optionally loads a comparison model (e.g., current champion) -3. Fetches fresh market data for evaluation +3. Loads evaluation data from eval_holdout asset (default) or fetches fresh 4. Runs inference and applies quality gates 5. Registers evaluation results linked to the model version -Model Loading: -- By default, evaluates "latest" (the candidate) -- Can specify any version or alias via parameters -- Comparison model is optional (for first model, or A/B tests) +Data Sources (via eval_config.eval_data_source): +- "eval_holdout" (default) - Use the holdout set from BuildDatasetFlow for reproducible evaluation +- "fresh" - Fetch live data from CoinGecko for testing on truly unseen data + +Config vs Parameter Override: +- eval_config sets defaults at deployment time +- Parameters can override config values at trigger time + +Usage: + # Use holdout set (default, reproducible) + python flow.py run + + # Override to use fresh data at runtime + python flow.py run --eval_data_source fresh + + # Evaluate specific model version + python flow.py run --candidate_version v5 Triggering: - Auto-triggers when TrainDetectorFlow completes (in Argo) @@ -29,9 +42,9 @@ @trigger_on_finish(flow='TrainDetectorFlow') class EvaluateDetectorFlow(ProjectFlow): """ - Evaluate anomaly detector on fresh market data. + Evaluate anomaly detector on holdout or fresh data. - Loads models by version or alias - no hardcoded assumptions. + Config-driven with parameter overrides for runtime flexibility. """ # Centralized configs @@ -51,6 +64,19 @@ class EvaluateDetectorFlow(ProjectFlow): help="Version to compare against (e.g., 'latest-1', 'champion', or version ID). Use 'none' to skip comparison." ) + # Data source override - Parameter overrides config at runtime + eval_data_source = Parameter( + "eval_data_source", + default=None, + help="Override eval data source: 'eval_holdout' (default) or 'fresh'. If not set, uses eval_config." + ) + + eval_data_version = Parameter( + "eval_data_version", + default=None, + help="Override eval data version (e.g., 'latest', 'latest-1'). If not set, uses eval_config." + ) + @step def start(self): """Load models from registry.""" @@ -94,35 +120,75 @@ def start(self): print(f"[INFO] No comparison model found ({self.compare_to}): {e}") print("Proceeding without comparison (first model or invalid version)") - self.next(self.fetch_eval_data) + self.next(self.load_eval_data) @step - def fetch_eval_data(self): - """Fetch fresh market data for evaluation.""" + def load_eval_data(self): + """Load evaluation data from holdout set or fetch fresh.""" + # Resolve data source: Parameter override > Config > Default + data_source = self.eval_data_source or self.eval_config.get("eval_data_source", "eval_holdout") + data_version = self.eval_data_version or self.eval_config.get("eval_data_version", "latest") + + print(f"\nEvaluation data source: {data_source}") + print(f"Evaluation data version: {data_version}") + + if data_source == "fresh": + self._fetch_fresh_data() + else: + self._load_holdout_data(data_source, data_version) + + self.next(self.evaluate) + + def _fetch_fresh_data(self): + """Fetch fresh data from CoinGecko.""" from src import data - # Read num_coins from centralized config num_coins = self.training_config.get("num_coins", 100) print(f"\nFetching fresh data for top {num_coins} coins...") snapshot = data.fetch_market_data(num_coins=num_coins) self.feature_set = data.extract_features(snapshot) + self.eval_data_ref = f"coingecko_live:{self.feature_set.timestamp}" print(f"Prepared {self.feature_set.n_samples} samples x {self.feature_set.n_features} features") print(f"Timestamp: {self.feature_set.timestamp}") - self.next(self.evaluate) + def _load_holdout_data(self, asset_name: str, version: str): + """Load holdout dataset from storage.""" + from src.storage import DatasetStore, table_to_feature_set + from src.data import FEATURE_COLS + + print(f"\nLoading holdout dataset: {asset_name} (version: {version})...") + + store = DatasetStore(self.prj.project, self.prj.branch) + table, metadata = store.load_dataset(asset_name, version=version) + + if table is None: + print(f"[WARN] Holdout dataset {asset_name}:{version} not found, falling back to fresh data") + self._fetch_fresh_data() + return + + # Convert to FeatureSet + self.feature_set = table_to_feature_set(table, FEATURE_COLS) + + # Capture resolved version for lineage + resolved_version = metadata.builder_run_id + self.eval_data_ref = f"{asset_name}:v{resolved_version}" + + print(f"Loaded {self.feature_set.n_samples} samples from {metadata.snapshot_count} snapshots") + print(f"Resolved version: BuildDatasetFlow/{resolved_version}") + print(f"Snapshot range: {metadata.snapshot_range[0][:19]} to {metadata.snapshot_range[1][:19]}") @card @step def evaluate(self): - """Run model on fresh data and apply quality gates.""" + """Run model on evaluation data and apply quality gates.""" from src.model import Model, get_anomalies from src import eval as evaluation from metaflow import current from metaflow.cards import Markdown, Table - print("\nEvaluating candidate on fresh data...") + print("\nEvaluating candidate on evaluation data...") # Recreate model from annotations model_config = { @@ -172,6 +238,7 @@ def evaluate(self): current.card.append(Markdown("# Anomaly Detector Evaluation")) current.card.append(Markdown(f"**Candidate:** v{self.candidate.version}")) + current.card.append(Markdown(f"**Evaluation Data:** {self.eval_data_ref}")) if self.candidate.alias: current.card.append(Markdown(f"**Alias:** {self.candidate.alias}")) if self.comparison: @@ -231,6 +298,7 @@ def register_evaluation(self): "silhouette_score": self.eval_result.metrics.silhouette_score, "score_gap": self.eval_result.metrics.score_gap, "eval_timestamp": self.feature_set.timestamp, + "eval_data_ref": self.eval_data_ref, } # Register evaluation results linked to this specific model version @@ -245,7 +313,7 @@ def register_evaluation(self): "score_gap": self.eval_result.metrics.score_gap, "eval_timestamp": self.feature_set.timestamp, }, - eval_dataset=f"coingecko_live:{self.feature_set.timestamp}", + eval_dataset=self.eval_data_ref, compared_to_version=self.comparison.version if self.comparison else None, run_metadata={ "evaluated_by_flow": current.flow_name, @@ -263,6 +331,7 @@ def register_evaluation(self): "compared_to_version": self.comparison.version if self.comparison else None, "anomaly_rate": float(self.prediction.anomaly_rate), "evaluation_run_id": current.run_id, + "eval_data_ref": self.eval_data_ref, }) print("Published 'evaluation_passed' event") else: @@ -278,6 +347,7 @@ def end(self): print("Evaluation Complete") print(f"{'='*50}") print(f"Model: anomaly_detector v{self.candidate.version}") + print(f"Eval data: {self.eval_data_ref}") if self.comparison: print(f"Compared to: v{self.comparison.version}") print(f"Result: {result}") From 1a807f1d7bed8c412235765f160f0b8981d5c72b Mon Sep 17 00:00:00 2001 From: Eddie Mattia <40632488+emattia@users.noreply.github.com> Date: Tue, 2 Dec 2025 10:30:52 -0800 Subject: [PATCH 16/26] Improve image display in README.md Updated image tag for better responsiveness in README. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0945d4a..f97c24b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -System designs (1) +system + An end-to-end ML project demonstrating **model registry patterns** with Outerbounds Projects. Detects anomalies in live cryptocurrency market data from CoinGecko. From c682b25f325b0d1dd934836db0ee03c0bfb4f897 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Tue, 2 Dec 2025 10:40:28 -0800 Subject: [PATCH 17/26] docs: add trigger form parameter tables to README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document how to fill trigger forms for: - BuildDatasetFlow (holdout_hours, max_history_hours, etc.) - EvaluateDetectorFlow (candidate_version, compare_to, eval_data_source) - PromoteModelFlow (version, alias, model_name) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- README.md | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f97c24b..21b6f02 100644 --- a/README.md +++ b/README.md @@ -65,13 +65,22 @@ python flows/ingest/flow.py run Accumulates snapshots into train/eval datasets with time-based splits. ```bash -# Default: 168hr history, 24hr holdout +# Default: 96hr history, 7hr holdout python flows/build_dataset/flow.py run # Quick test (smaller holdout) python flows/build_dataset/flow.py run --holdout_hours 1 ``` +**Trigger Form Parameters:** + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `holdout_hours` | `7` | Hours of recent data reserved for evaluation holdout | +| `max_history_hours` | `96` | Maximum history window to include | +| `min_snapshots_per_coin` | `3` | Minimum snapshots required per coin (filters incomplete data) | +| `add_targets` | `false` | Add target columns for supervised learning | + ### TrainDetectorFlow Trains an Isolation Forest model and registers it as a ModelAsset. @@ -86,20 +95,52 @@ python flows/train/flow.py run --fresh_data ### EvaluateDetectorFlow -Evaluates the latest model against quality gates. +Evaluates a model against quality gates using the holdout dataset. ```bash +# Default: evaluate latest model on holdout set python flows/evaluate/flow.py run + +# Use fresh data instead of holdout +python flows/evaluate/flow.py run --eval_data_source fresh + +# Evaluate specific model version +python flows/evaluate/flow.py run --candidate_version v5 ``` +**Trigger Form Parameters:** + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `candidate_version` | `latest` | Model to evaluate. Use `latest`, `latest-N`, or version ID | +| `compare_to` | `latest-1` | Model to compare against. Use `none` to skip comparison | +| `eval_data_source` | *(empty)* | Leave empty for `eval_holdout` (from config). Set `fresh` for live data | +| `eval_data_version` | *(empty)* | Leave empty for `latest` (from config). Set `latest-1` for previous holdout | + +Empty fields use defaults from `configs/evaluation.json`. Parameters override config at runtime. + +**Version specifiers:** `latest`, `latest-1`, `latest-2`, etc. resolve to actual version IDs at runtime. + ### PromoteModelFlow Promotes a model to champion using Metaflow run tags. ```bash -python flows/promote/flow.py run --run_id +# Promote by training run ID +python flows/promote/flow.py run --version 186362 + +# Promote latest model +python flows/promote/flow.py run --version latest ``` +**Trigger Form Parameters:** + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `version` | *(required)* | Training run ID to promote (e.g., `186362`) or `latest` | +| `alias` | `champion` | Tag to apply (typically `champion`) | +| `model_name` | `anomaly_detector` | Model asset name | + ## Model Lifecycle ``` From f4cf77e76cfced91d1ef2214cf7fe26915eaff60 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Tue, 2 Dec 2025 10:47:33 -0800 Subject: [PATCH 18/26] fix: dashboard evaluation status using wrong field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The header was checking evaluation.decision but get_latest_evaluation() returns evaluation.passed. Changed to use correct field. Also add trigger form parameter tables to README. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- deployments/dashboard/templates/overview.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deployments/dashboard/templates/overview.html b/deployments/dashboard/templates/overview.html index c800d73..49a7fc4 100644 --- a/deployments/dashboard/templates/overview.html +++ b/deployments/dashboard/templates/overview.html @@ -50,10 +50,10 @@

Project Overview

{% if evaluation %} - {% if evaluation.decision == 'approve' %} - Approved + {% if evaluation.passed %} + Passed {% else %} - Rejected + Failed {% endif %} {% else %} - From 154af83848cb56dd752e365bc1c2e5a3f1d3df3a Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Tue, 2 Dec 2025 11:04:46 -0800 Subject: [PATCH 19/26] fix: scope dashboard model queries by project_branch tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Metaflow runs are namespaced by project_branch tag, not global. The dashboard was querying Flow('TrainDetectorFlow').runs() without filtering, showing models from all branches. Changes: - get_model_versions() now filters by project_branch:{branch} tag - get_champion_run_id() now accepts branch param for scoped lookup - model_registry() passes branch to both functions - overview.html: fix evaluation.decision -> evaluation.passed 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- deployments/dashboard/app.py | 53 +++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/deployments/dashboard/app.py b/deployments/dashboard/app.py index cba173a..f3af96e 100644 --- a/deployments/dashboard/app.py +++ b/deployments/dashboard/app.py @@ -217,23 +217,29 @@ def get_pipeline_status(asset: Asset, branch: str): return status -def get_model_versions(asset: Asset, limit=20): +def get_model_versions(asset: Asset, branch: str, limit=20): """Get list of model versions from Metaflow training runs. Uses Metaflow Client API to fetch run artifacts directly. This provides complete version history with all training metadata. + + Args: + asset: Asset client (for fallback) + branch: Project branch to filter runs by (e.g., 'test.feature_prediction') + limit: Maximum number of versions to return """ from metaflow import Flow versions = [] - # Look up champion by alias tag - current_champion_run = get_champion_run_id() + # Look up champion by alias tag (scoped to this branch) + current_champion_run = get_champion_run_id(branch=branch) - # Get version history from Metaflow runs + # Get version history from Metaflow runs, filtered by project_branch tag try: flow = Flow('TrainDetectorFlow') - for i, run in enumerate(flow.runs()): + branch_tag = f"project_branch:{branch}" + for i, run in enumerate(flow.runs(branch_tag)): if i >= limit: break @@ -450,16 +456,37 @@ def get_model_details(run_id: str, alias: str = None): return None -def get_champion_run_id() -> str: +def get_champion_run_id(branch: str = None) -> str: """Get the training run ID of the current champion model. - Uses Metaflow's native tagging to find the run tagged 'champion'. + Uses Metaflow's native tagging to find the run tagged 'champion', + optionally scoped to a specific project branch. + + Args: + branch: Project branch to filter by (e.g., 'test.feature_prediction'). + If None, returns champion across all branches. Returns the run ID string, or None if no champion exists. """ - from src import registry + from metaflow import Flow + + try: + flow = Flow("TrainDetectorFlow") + # Build tag filter: always require 'champion', optionally filter by branch + if branch: + # Find champion within this branch + branch_tag = f"project_branch:{branch}" + for run in flow.runs(branch_tag): + if "champion" in run.tags: + return run.id + else: + # Find champion across all branches + for run in flow.runs("champion"): + return run.id + except Exception: + pass - return registry.get_champion_run_id(flow_name="TrainDetectorFlow") + return None # ============================================================================= @@ -554,17 +581,17 @@ async def model_registry(request: Request): branch = get_branch_from_request(request) asset = get_asset_client(branch) - # Get versions with error handling + # Get versions with error handling (filtered by branch) versions = [] error_message = None try: - versions = get_model_versions(asset, limit=20) + versions = get_model_versions(asset, branch=branch, limit=20) except Exception as e: error_message = f"Failed to load model versions: {str(e)}" print(f"[ERROR] {error_message}") - # Get champion details with full lineage - champion_run_id = get_champion_run_id() + # Get champion details with full lineage (scoped to branch) + champion_run_id = get_champion_run_id(branch=branch) champion = get_model_details(champion_run_id, alias="champion") if champion_run_id else None # Count champions to validate uniqueness From fa88aa270ccd85c41bf4049d8d7550ad478b91f5 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Tue, 2 Dec 2025 11:17:03 -0800 Subject: [PATCH 20/26] fix: use project namespace for cross-runtime run visibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dashboard was only seeing runs in the default user namespace. Argo runs exist in a different namespace but share the same project. Fix: Use namespace(f"project:{PROJECT}") to see all runs (dev + argo) for the project, then filter by project_branch tag for branch-specific queries. Key insight from Metaflow docs: - namespace('user:X') = only local dev runs by that user - namespace('project:X') = all runs for project (dev + argo + prod) - Filter by project_branch:{branch} tag for branch-specific runs 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- deployments/dashboard/app.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/deployments/dashboard/app.py b/deployments/dashboard/app.py index f3af96e..fffded5 100644 --- a/deployments/dashboard/app.py +++ b/deployments/dashboard/app.py @@ -228,13 +228,17 @@ def get_model_versions(asset: Asset, branch: str, limit=20): branch: Project branch to filter runs by (e.g., 'test.feature_prediction') limit: Maximum number of versions to return """ - from metaflow import Flow + from metaflow import Flow, namespace versions = [] # Look up champion by alias tag (scoped to this branch) current_champion_run = get_champion_run_id(branch=branch) + # Set project namespace to see all runs (dev + argo) for this project, + # then filter by project_branch tag to get branch-specific runs + namespace(f"project:{PROJECT}") + # Get version history from Metaflow runs, filtered by project_branch tag try: flow = Flow('TrainDetectorFlow') @@ -468,10 +472,13 @@ def get_champion_run_id(branch: str = None) -> str: Returns the run ID string, or None if no champion exists. """ - from metaflow import Flow + from metaflow import Flow, namespace try: + # Set project namespace to see all runs (dev + argo) + namespace(f"project:{PROJECT}") flow = Flow("TrainDetectorFlow") + # Build tag filter: always require 'champion', optionally filter by branch if branch: # Find champion within this branch From 52d261c6815087b2ba1bb3b113c9398084d8c769 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Tue, 2 Dec 2025 11:35:48 -0800 Subject: [PATCH 21/26] fix: dashboard UX improvements for branch scoping and loading states MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cards page: - Fix branch scoping using namespace(f"project:{PROJECT}") - Filter runs by project_branch tag to show only relevant cards - Improve dropdown display with run IDs Models page: - Add loading spinner for Compare button - Fix lineage comparison to show Dataset Version (not timestamp) - Add hover tooltip for full IDs - Improve Outcome Validation explanation (3-step process) Compare API: - Add project namespace for cross-runtime run access Base styles: - Add .spinner and .spinner-lg CSS animations - Add .loading-overlay for async operations - Add .btn.loading state with inline spinner 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- deployments/dashboard/app.py | 29 ++++++++++--- deployments/dashboard/templates/base.html | 47 +++++++++++++++++++++ deployments/dashboard/templates/cards.html | 17 +++++--- deployments/dashboard/templates/models.html | 37 ++++++++++++---- 4 files changed, 109 insertions(+), 21 deletions(-) diff --git a/deployments/dashboard/app.py b/deployments/dashboard/app.py index fffded5..0787b9c 100644 --- a/deployments/dashboard/app.py +++ b/deployments/dashboard/app.py @@ -772,24 +772,36 @@ async def cards_page(request: Request): """ Cards overview page - list available cards with iframe previews. """ + from metaflow import Flow, namespace + branch = get_branch_from_request(request) - # Get recent runs with cards for each flow + # Set project namespace to see all runs (dev + argo) for this project + namespace(f"project:{PROJECT}") + + # Get recent runs with cards for each flow, filtered by branch cards_info = [] + branch_tag = f"project_branch:{branch}" + for alias, flow_name in FLOW_NAME_MAP.items(): try: - from metaflow import Flow flow = Flow(flow_name) - run = flow.latest_successful_run + # Find latest successful run for this branch + run = None + for r in flow.runs(branch_tag): + if r.successful: + run = r + break + if run: # Get detailed card info for this run - step_cards = get_flow_cards_info(flow_name, "latest") + step_cards = get_flow_cards_info(flow_name, run.id) cards_info.append({ "alias": alias, "flow_name": flow_name, "run_id": run.id, "created_at": run.created_at.isoformat() if run.created_at else None, - "url": f"/cards/{alias}/latest", + "url": f"/cards/{alias}/{run.id}?branch={branch}", "step_cards": step_cards, # List of {step_name, card_index} "card_count": len(step_cards), }) @@ -808,11 +820,14 @@ async def cards_page(request: Request): # ============================================================================= @app.get("/api/models/compare") -async def api_compare(request: Request, v1: str, v2: str): +async def api_compare(request: Request, v1: str, v2: str, branch: str = None): """Compare two model versions using Metaflow run data.""" - from metaflow import Flow + from metaflow import Flow, namespace import numpy as np + # Set project namespace to see all runs + namespace(f"project:{PROJECT}") + def get_version_data(version_str: str) -> dict: """Extract full model data from a version string like 'vTrain/186089'. diff --git a/deployments/dashboard/templates/base.html b/deployments/dashboard/templates/base.html index 3d2ade1..170ebad 100644 --- a/deployments/dashboard/templates/base.html +++ b/deployments/dashboard/templates/base.html @@ -581,6 +581,53 @@ .items-center { align-items: center; } .gap-1 { gap: 0.5rem; } .gap-2 { gap: 1rem; } + + /* Loading spinner */ + .spinner { + display: inline-block; + width: 1rem; + height: 1rem; + border: 2px solid var(--sand-300); + border-top-color: var(--primary); + border-radius: 50%; + animation: spin 0.8s linear infinite; + } + + .spinner-lg { + width: 2rem; + height: 2rem; + border-width: 3px; + } + + @keyframes spin { + to { transform: rotate(360deg); } + } + + .loading-overlay { + display: flex; + align-items: center; + justify-content: center; + gap: 0.75rem; + padding: 2rem; + color: var(--gray-300); + } + + .btn.loading { + pointer-events: none; + opacity: 0.7; + } + + .btn.loading::after { + content: ''; + display: inline-block; + width: 0.875rem; + height: 0.875rem; + margin-left: 0.5rem; + border: 2px solid currentColor; + border-top-color: transparent; + border-radius: 50%; + animation: spin 0.8s linear infinite; + } diff --git a/deployments/dashboard/templates/cards.html b/deployments/dashboard/templates/cards.html index 1626b57..3cf7729 100644 --- a/deployments/dashboard/templates/cards.html +++ b/deployments/dashboard/templates/cards.html @@ -14,14 +14,13 @@

Metaflow Cards

Compare Cards
+ {% if cards %}
@@ -29,9 +28,7 @@

Metaflow Cards

@@ -39,6 +36,14 @@

Metaflow Cards

+

+ Compare training cards across runs to see how score distributions, anomaly rates, and detected coins change over time. +

+ {% else %} +
+ No cards available for this branch. Run flows with @card decorator first. +
+ {% endif %}
{% if cards %} diff --git a/deployments/dashboard/templates/models.html b/deployments/dashboard/templates/models.html index f2fb71d..15bc034 100644 --- a/deployments/dashboard/templates/models.html +++ b/deployments/dashboard/templates/models.html @@ -73,8 +73,15 @@

O

{% else %}
- No outcome validation yet.
- Run ValidateOutcomesFlow to measure real-world performance. + Not yet validated.
+ + Outcome validation requires: +
    +
  1. Model makes predictions (logged by TrainFlow)
    2. +
  2. Time passes (24-48h) to observe actual outcomes
    3. +
  3. Run ValidateOutcomesFlow to compare predictions vs reality
    4. +
+
{% endif %} @@ -273,16 +280,28 @@

O const v1 = document.getElementById('compare-v1').value; const v2 = document.getElementById('compare-v2').value; const branch = new URLSearchParams(window.location.search).get('branch') || '{{ branch }}'; + const btn = event.target; + const resultsDiv = document.getElementById('comparison-results'); if (v1 === v2) { alert('Please select two different versions to compare'); return; } + // Show loading state + btn.classList.add('loading'); + btn.disabled = true; + resultsDiv.style.display = 'block'; + resultsDiv.innerHTML = '
Loading comparison...
'; + try { const response = await fetch(`/api/models/compare?v1=${v1}&v2=${v2}&branch=${branch}`); const data = await response.json(); + // Remove loading state + btn.classList.remove('loading'); + btn.disabled = false; + if (response.ok) { const resultsDiv = document.getElementById('comparison-results'); resultsDiv.style.display = 'block'; @@ -355,10 +374,10 @@

O - - Data Snapshot - ${formatTime(data.v1.data_timestamp)} - ${formatTime(data.v2.data_timestamp)} - ${data.v1.data_timestamp === data.v2.data_timestamp ? 'Same data' : 'Different'} + Dataset Version + ${data.v1.data_source ? data.v1.data_source.substring(0, 30) + (data.v1.data_source.length > 30 ? '...' : '') : '-'} + ${data.v2.data_source ? data.v2.data_source.substring(0, 30) + (data.v2.data_source.length > 30 ? '...' : '') : '-'} + ${data.v1.data_source === data.v2.data_source ? 'Same data' : '-'} ${data.v1.data_snapshot_count || data.v2.data_snapshot_count ? ` @@ -394,10 +413,12 @@

O `; } else { - alert('Comparison failed: ' + (data.detail || 'Unknown error')); + resultsDiv.innerHTML = `
Comparison failed: ${data.detail || 'Unknown error'}
`; } } catch (err) { - alert('Error: ' + err.message); + btn.classList.remove('loading'); + btn.disabled = false; + resultsDiv.innerHTML = `
Error: ${err.message}
`; } } From 54165290bd502f02fa80689cb3e7a665731f20cf Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Tue, 2 Dec 2025 12:15:10 -0800 Subject: [PATCH 22/26] fix: PromoteModelFlow support for Argo run IDs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The promote flow failed when trying to promote Argo runs because: 1. Argo run IDs (like argo-foo-bar-xyz) weren't recognized as run IDs 2. Metaflow Client API needed project namespace to find Argo runs Changes: - flows/promote/flow.py: Detect argo- prefix as run ID format - flows/promote/flow.py: Use project namespace when loading runs - src/registry.py: set_champion_run() now accepts project param and uses project namespace to find runs across all runtimes 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- flows/promote/flow.py | 10 +++++++--- src/registry.py | 7 ++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/flows/promote/flow.py b/flows/promote/flow.py index d40ee19..683cdb3 100644 --- a/flows/promote/flow.py +++ b/flows/promote/flow.py @@ -61,7 +61,7 @@ def start(self): # Load the model to verify it exists # Handle different version formats: # - "latest", "latest-N" -> pass directly to Asset API - # - bare run ID like "186088" -> load via Metaflow Client API + # - run ID (numeric like "186088" or argo-style like "argo-foo-bar") -> load via Metaflow Client API # - full version pathspec -> pass directly to Asset API try: if self.version.startswith("latest"): @@ -71,9 +71,12 @@ def start(self): self.model_name, version=self.version ) - elif self.version.isdigit() or len(self.version) < 20: - # Looks like a bare run ID - load via Metaflow Client API + elif self.version.isdigit() or self.version.startswith("argo-"): + # Looks like a run ID (numeric or argo-style) - load via Metaflow Client API print(f"Loading model from TrainDetectorFlow run: {self.version}") + from metaflow import namespace + # Use project namespace to see all runs (dev + argo) + namespace(f"project:{self.prj.project}") flow = Flow('TrainDetectorFlow') run = flow[self.version] @@ -147,6 +150,7 @@ def promote(self): previous = registry.set_champion_run( run_id=self.model.training_run_id, flow_name="TrainDetectorFlow", + project=self.prj.project, ) self.promotion_success = True diff --git a/src/registry.py b/src/registry.py index aacf11b..4f1e7a0 100644 --- a/src/registry.py +++ b/src/registry.py @@ -353,6 +353,7 @@ def get_champion_run_id(flow_name: str = "TrainDetectorFlow") -> Optional[str]: def set_champion_run( run_id: str, flow_name: str = "TrainDetectorFlow", + project: str = "crypto_anomaly", ) -> Optional[str]: """ Set a training run as the champion by tagging it. @@ -362,11 +363,15 @@ def set_champion_run( Args: run_id: The run ID to promote flow_name: The training flow name + project: Project name for namespace scoping Returns: The previous champion's run ID, or None if no previous champion """ - from metaflow import Flow, Run + from metaflow import Flow, Run, namespace + + # Use project namespace to see all runs (dev + argo) + namespace(f"project:{project}") previous_champion = None From 5fcf4ecc64de68091d195df2fa4399291183fa3d Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Tue, 2 Dec 2025 12:34:23 -0800 Subject: [PATCH 23/26] feat: add intra-flow card comparison for comparing runs over time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow comparing multiple runs of the same flow (e.g., latest vs latest-1 training runs) in addition to cross-flow comparison. This helps track how score distributions, anomaly rates, and detected coins change over time. - Collect up to 5 recent runs per flow for intra-flow comparison - Add comparison mode toggle (cross-flow vs intra-flow) - Add flow selector and version dropdowns for intra-flow mode - Pass flow_runs data to template for JavaScript-driven UI 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- deployments/dashboard/app.py | 43 ++++++-- deployments/dashboard/templates/cards.html | 121 ++++++++++++++++++++- 2 files changed, 150 insertions(+), 14 deletions(-) diff --git a/deployments/dashboard/app.py b/deployments/dashboard/app.py index 0787b9c..32f3fb7 100644 --- a/deployments/dashboard/app.py +++ b/deployments/dashboard/app.py @@ -771,6 +771,8 @@ async def card_by_run(flow_alias: str, run_id: str): async def cards_page(request: Request): """ Cards overview page - list available cards with iframe previews. + Supports both cross-flow comparison (latest of each flow) and + intra-flow comparison (multiple runs of same flow). """ from metaflow import Flow, namespace @@ -780,29 +782,45 @@ async def cards_page(request: Request): namespace(f"project:{PROJECT}") # Get recent runs with cards for each flow, filtered by branch - cards_info = [] + # Collect up to MAX_RUNS_PER_FLOW for intra-flow comparison + MAX_RUNS_PER_FLOW = 5 + cards_info = [] # Latest run per flow (for display cards) + flow_runs = {} # All recent runs per flow (for intra-flow comparison) branch_tag = f"project_branch:{branch}" for alias, flow_name in FLOW_NAME_MAP.items(): try: flow = Flow(flow_name) - # Find latest successful run for this branch - run = None + # Collect recent successful runs for this branch + runs_for_flow = [] for r in flow.runs(branch_tag): if r.successful: - run = r - break + run_info = { + "run_id": r.id, + "created_at": r.created_at.isoformat() if r.created_at else None, + "url": f"/cards/{alias}/{r.id}?branch={branch}", + } + runs_for_flow.append(run_info) + if len(runs_for_flow) >= MAX_RUNS_PER_FLOW: + break + + if runs_for_flow: + # Store all runs for this flow + flow_runs[alias] = { + "flow_name": flow_name, + "runs": runs_for_flow, + } - if run: - # Get detailed card info for this run - step_cards = get_flow_cards_info(flow_name, run.id) + # Use latest run for the display card + latest = runs_for_flow[0] + step_cards = get_flow_cards_info(flow_name, latest["run_id"]) cards_info.append({ "alias": alias, "flow_name": flow_name, - "run_id": run.id, - "created_at": run.created_at.isoformat() if run.created_at else None, - "url": f"/cards/{alias}/{run.id}?branch={branch}", - "step_cards": step_cards, # List of {step_name, card_index} + "run_id": latest["run_id"], + "created_at": latest["created_at"], + "url": latest["url"], + "step_cards": step_cards, "card_count": len(step_cards), }) except Exception as e: @@ -812,6 +830,7 @@ async def cards_page(request: Request): return templates.TemplateResponse("cards.html", get_template_context( request, branch, cards=cards_info, + flow_runs=flow_runs, # For intra-flow comparison dropdowns )) diff --git a/deployments/dashboard/templates/cards.html b/deployments/dashboard/templates/cards.html index 3cf7729..8e8e2e0 100644 --- a/deployments/dashboard/templates/cards.html +++ b/deployments/dashboard/templates/cards.html @@ -15,7 +15,20 @@

Metaflow Cards

Compare Cards {% if cards %} -
+ +
+ + +
+ + +
+ {% for alias, flow_data in flow_runs.items() %} + + {% endfor %} + +
+
+ + +
+
+ + +
+
+ +
+
+

- Compare training cards across runs to see how score distributions, anomaly rates, and detected coins change over time. + Cross-flow: Compare cards from different flows (e.g., Train vs Evaluate).
+ Intra-flow: Compare runs of the same flow over time (e.g., latest vs latest-1 training runs).

{% else %}
@@ -118,6 +156,9 @@

Metaflow Cards

{% else %} From 61776d51c724d595d1c2e49d1345050a588093ad Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Tue, 2 Dec 2025 12:53:04 -0800 Subject: [PATCH 24/26] fix: Overview page champion detection using Metaflow run tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Overview page was using Asset API (latest-1) to find the champion, which doesn't reflect the actual promoted champion. Now uses the same get_champion_run_id() function as the Models page to correctly find runs tagged with 'champion'. - get_pipeline_status() now uses get_champion_run_id(branch) - overview() now fetches champion details from the tagged training run - Template handles missing silhouette_score gracefully (only in eval flow) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- deployments/dashboard/app.py | 55 ++++++++++++------- deployments/dashboard/templates/overview.html | 4 +- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/deployments/dashboard/app.py b/deployments/dashboard/app.py index 32f3fb7..f35cc32 100644 --- a/deployments/dashboard/app.py +++ b/deployments/dashboard/app.py @@ -195,14 +195,15 @@ def get_pipeline_status(asset: Asset, branch: str): except Exception: status["data"] = {"status": "no_data"} - # Check champion (latest-1) - try: - ref = registry.load_model(asset, "anomaly_detector", instance="latest-1") + # Check champion via Metaflow run tags + champion_run_id = get_champion_run_id(branch=branch) + if champion_run_id: status["champion"] = { "status": "available", - "version": ref.version, + "version": f"Train/{champion_run_id}", + "run_id": champion_run_id, } - except Exception: + else: status["champion"] = {"status": "none"} # Overall @@ -525,21 +526,37 @@ async def overview(request: Request): status = get_pipeline_status(asset, branch) evaluation = get_latest_evaluation(asset) - # Get champion details (latest-1) + # Get champion details via Metaflow run tags (same as Models page) champion_details = None - try: - from src import registry - ref = registry.load_model(asset, "anomaly_detector", instance="latest-1") - ann = ref.annotations - champion_details = { - "version": ref.version, - "algorithm": ann.get("algorithm"), - "anomaly_rate": _float(ann.get("anomaly_rate")), - "training_samples": _int(ann.get("training_samples")), - "silhouette_score": _float(ann.get("silhouette_score")), - } - except Exception: - pass + champion_run_id = get_champion_run_id(branch=branch) + if champion_run_id: + try: + from metaflow import Flow, namespace + namespace(f"project:{PROJECT}") + flow = Flow("TrainDetectorFlow") + run = flow[champion_run_id] + + # Get model details from the train step + train_step = run['train'] + for task in train_step: + config = dict(task.data.model_config) + prediction = task.data.prediction + feature_set = task.data.feature_set + hp = config.get('hyperparameters', {}) + + champion_details = { + "version": f"Train/{champion_run_id}", + "run_id": champion_run_id, + "algorithm": config.get("algorithm", "isolation_forest"), + "anomaly_rate": _float(prediction.anomaly_rate) if hasattr(prediction, 'anomaly_rate') else None, + "training_samples": _int(feature_set.n_samples) if hasattr(feature_set, 'n_samples') else None, + "silhouette_score": None, # Computed in evaluate flow, not available here + "n_estimators": _int(hp.get("n_estimators")), + "contamination": _float(hp.get("contamination")), + } + break + except Exception as e: + print(f"[DEBUG] Failed to load champion run {champion_run_id}: {e}") return templates.TemplateResponse("overview.html", get_template_context( request, branch, diff --git a/deployments/dashboard/templates/overview.html b/deployments/dashboard/templates/overview.html index 49a7fc4..dfeb4a9 100644 --- a/deployments/dashboard/templates/overview.html +++ b/deployments/dashboard/templates/overview.html @@ -89,12 +89,14 @@

Project Overview

Anomaly Rate - {{ "%.1f"|format(champion.anomaly_rate * 100) }}% + {% if champion.anomaly_rate %}{{ "%.1f"|format(champion.anomaly_rate * 100) }}%{% else %}-{% endif %}
+ {% if champion.silhouette_score %}
Silhouette Score {{ "%.3f"|format(champion.silhouette_score) }}
+ {% endif %} {% else %}
🏆
From f9baff512a4e7b61bd334b72535793b0d5f06863 Mon Sep 17 00:00:00 2001 From: Eddie Mattia <40632488+emattia@users.noreply.github.com> Date: Tue, 2 Dec 2025 13:44:12 -0800 Subject: [PATCH 25/26] Replace old image with new one in README Updated image source in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 21b6f02..64809ac 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -system +system An end-to-end ML project demonstrating **model registry patterns** with Outerbounds Projects. Detects anomalies in live cryptocurrency market data from CoinGecko. From 9642cec05c4bcaa8b2057076d4a735dbcbedb125 Mon Sep 17 00:00:00 2001 From: Eddie Mattia Date: Tue, 2 Dec 2025 13:55:47 -0800 Subject: [PATCH 26/26] delete dev artifacts --- .claude/agents/ds-strategist.md | 150 ----------------------------- .claude/agents/lifecycle-runner.md | 145 ---------------------------- .claude/agents/ui-improver.md | 79 --------------- .github/workflows/deploy.yml | 2 +- 4 files changed, 1 insertion(+), 375 deletions(-) delete mode 100644 .claude/agents/ds-strategist.md delete mode 100644 .claude/agents/lifecycle-runner.md delete mode 100644 .claude/agents/ui-improver.md diff --git a/.claude/agents/ds-strategist.md b/.claude/agents/ds-strategist.md deleted file mode 100644 index 4be35cd..0000000 --- a/.claude/agents/ds-strategist.md +++ /dev/null @@ -1,150 +0,0 @@ ---- -name: ds-strategist -description: Critiques the data science approach and proposes evolutions toward practical business value -tools: Read, Edit, Write, Bash, Glob, Grep, WebSearch, WebFetch -model: opus ---- - -You are a senior data science strategist who critiques ML reference architectures and proposes evolutions that increase practical business value. Your role is to challenge assumptions, identify gaps, and suggest concrete improvements. - -## Context: What This Project Is - -This is a **reference implementation** showing how to build production ML systems with Metaflow + Outerbounds. The current example is crypto anomaly detection, but the architecture patterns matter more than the specific use case. - -### Current Architecture (What's Been Built) - -``` -Data Pipeline: -IngestMarketDataFlow → BuildDatasetFlow → TrainDetectorFlow → EvaluateDetectorFlow → PromoteDetectorFlow - (hourly) (accumulates) (trains) (quality gates) (champion) - -Model Lifecycle: -candidate → evaluated → champion - ↑ ↑ ↑ - Training QA Gates Human/Auto -``` - -### Key Files to Understand - -| File | What It Does | -|------|--------------| -| `src/data.py` | Data fetching from CoinGecko, feature engineering | -| `src/model.py` | Isolation Forest / LOF anomaly detection | -| `src/eval.py` | Quality gates (rate bounds, silhouette, score gap) | -| `src/registry.py` | Model versioning, status management | -| `flows/*/flow.py` | Metaflow orchestration for each lifecycle stage | -| `deployments/dashboard/` | FastAPI dashboard showing model registry state | - -### What the Reference Architecture Demonstrates - -1. **Model Lifecycle Management** - Create, version, evaluate, promote models -2. **Data Lineage** - Track which dataset trained which model -3. **Quality Gates** - Automated checks before promotion -4. **Human-in-the-Loop** - Optional approval workflows -5. **Observability** - Dashboard showing registry state, Metaflow cards for viz - -## Your Task: Critique and Evolve - -When invoked, you should: - -### 1. Critique the Current Data Science Task - -The current task is **crypto anomaly detection** using unsupervised learning. Be honest: - -- **What's the actual business value?** Who would pay for this? What decisions does it enable? -- **Is the evaluation meaningful?** Without ground truth, are the quality gates actually measuring anything useful? -- **Does the model generalize?** Crypto markets are notoriously non-stationary. Does training on historical data predict future anomalies? -- **What's the feedback loop?** How would we know if the model is working in production? - -### 2. Propose Evolution Paths - -Suggest 2-3 concrete evolutions that would make this reference architecture more valuable to real companies. Consider: - -**Domain pivots** (keep the architecture, change the problem): -- Fraud detection (has ground truth labels eventually) -- Infrastructure monitoring (anomaly → incident correlation) -- Financial transaction monitoring (regulatory requirements) -- Manufacturing quality control (sensor data, defect labels) - -**Architecture evolutions** (keep the domain, add capabilities): -- A/B testing framework for challenger models -- Drift detection triggering automatic retraining -- Feature store integration -- Online learning / incremental updates -- Explainability (why is this point anomalous?) - -**Evaluation improvements**: -- Synthetic anomaly injection for testing -- Backtesting framework with historical "known bad" events -- Business metric correlation (did flagged anomalies lead to real events?) - -### 3. Provide Implementation Guidance - -For your top recommendation, provide: -- Which files need to change -- What new flows/modules to add -- Estimated complexity (small/medium/large) -- How it demonstrates additional reference architecture value - -## How to Work - -1. **Read the codebase first** - Use Glob/Grep/Read to understand what exists -2. **Search for context** - Use WebSearch to find industry examples, papers, or benchmarks -3. **Be specific** - Reference actual files, functions, and code patterns -4. **Be honest** - If something is demo-ware with limited practical value, say so -5. **Be constructive** - Critique should lead to actionable improvements - -## What Previous Contributors Have Built - -The conversation history shows iterative improvements: - -1. **Flow refactoring** - Renamed flows from `*AnomalyFlow` to `*DetectorFlow` -2. **Card visualizations** - Added `src/cards.py` with reusable Metaflow card components -3. **Dashboard improvements** - Card embedding, timestamp formatting, error handling -4. **Playwright testing** - Automated UI validation in `scripts/validate_dashboard.py` -5. **Claude agents** - `ui-improver` and `lifecycle-runner` for iterative development - -Build on this foundation. Don't reinvent what's already working. - -## Output Format - -Structure your response as: - -```markdown -## Critique: Current State - -### Business Value Assessment -[Honest assessment of who would use this and why] - -### Technical Gaps -[What's missing or broken] - -### Evaluation Quality -[Are we actually measuring anything meaningful?] - -## Recommended Evolution: [Name] - -### Why This Evolution -[Business case] - -### Implementation Plan -[Specific files, flows, changes] - -### Complexity & Dependencies -[What it takes to build] - -### Reference Architecture Value -[What new patterns does this demonstrate?] - -## Alternative Evolutions - -### Option B: [Name] -[Brief description] - -### Option C: [Name] -[Brief description] -``` - -## Remember - -You are not here to praise the existing work. You are here to make it better by identifying what's weak and proposing concrete improvements. The goal is a reference architecture that companies actually want to use as a starting point for their ML platforms. diff --git a/.claude/agents/lifecycle-runner.md b/.claude/agents/lifecycle-runner.md deleted file mode 100644 index ea97894..0000000 --- a/.claude/agents/lifecycle-runner.md +++ /dev/null @@ -1,145 +0,0 @@ ---- -name: lifecycle-runner -description: Orchestrates model registry scenarios by running flows and verifying UI state -tools: Read, Edit, Write, Bash, Glob, Grep -model: sonnet ---- - -You orchestrate end-to-end model registry scenarios to exercise the full lifecycle and verify the dashboard correctly reflects each stage. - -## Project Context - -This is a reference implementation showing how to build production ML systems with Metaflow + Outerbounds. You will run flows that move assets through lifecycle stages and verify the dashboard updates correctly. - -### Model Lifecycle States -``` -candidate → evaluated → champion - ↑ ↑ ↑ - TrainFlow EvaluateFlow PromoteFlow -``` - -### Data Pipeline -``` -IngestMarketDataFlow → BuildDatasetFlow → TrainDetectorFlow → ValidateOutcomesFlow - (hourly snapshots) (temporal features) (trains + logs) (24h later) - │ - ▼ - predictions/ - (logged for validation) -``` - -## How to Run Flows - -All flows are in `flows/{name}/flow.py`. Run them with: - -```bash -cd /Users/eddie/Dev/ado-projects-dev/model-registry-project-v1 -export PYTHONPATH="$PWD" -export METAFLOW_PROFILE=yellow - -# Data pipeline -python flows/ingest/flow.py run -python flows/build_dataset/flow.py run --n_snapshots 5 --holdout_snapshots 1 - -# Model lifecycle -python flows/train/flow.py run -python flows/evaluate/flow.py run -python flows/promote/flow.py run -``` - -### Config Overrides - -```bash -# Different model algorithm -python flows/train/flow.py --config model_config configs/model_lof.json run - -# Fresh data (skip asset registry) -python flows/train/flow.py --config-value training_config '{"data_version": null}' run - -# Evaluation thresholds -python flows/evaluate/flow.py run --max_anomaly_rate 0.25 --min_anomaly_rate 0.01 - -# Promote specific version -python flows/promote/flow.py run --version v3 -``` - -## Verify Dashboard State - -After each flow, verify the dashboard reflects the change: - -```bash -python scripts/validate_dashboard.py --branch prod -``` - -Or check specific pages manually: -- http://localhost:8001/?branch=prod - Overview -- http://localhost:8001/data?branch=prod - Data assets -- http://localhost:8001/models?branch=prod - Model versions -- http://localhost:8001/cards?branch=prod - Flow cards - -## Example Scenarios - -### Scenario 1: First Model (Cold Start) -1. `python flows/ingest/flow.py run` - Creates market_snapshot -2. Verify: Data page shows market_snapshot asset -3. `python flows/train/flow.py run` - Creates candidate model -4. Verify: Models page shows version with "candidate" status -5. `python flows/evaluate/flow.py run` - Evaluates and updates status -6. Verify: Models page shows "evaluated" status -7. `python flows/promote/flow.py run` - Promotes to champion -8. Verify: Models page shows "champion" badge - -### Scenario 2: Dataset Accumulation -1. Run `python flows/ingest/flow.py run` multiple times (3-5x) -2. `python flows/build_dataset/flow.py run --n_snapshots 5 --holdout_snapshots 1` -3. Verify: Data page shows training_dataset and eval_holdout -4. `python flows/train/flow.py run` - Uses accumulated dataset -5. Verify: Models page shows new version trained on accumulated data - -### Scenario 3: Model Comparison -1. Train first model: `python flows/train/flow.py run` -2. Promote it: `python flows/promote/flow.py run` -3. Train second model with different config: - ```bash - python flows/train/flow.py --config model_config configs/model_lof.json run - ``` -4. Evaluate: `python flows/evaluate/flow.py run` -5. Verify: Models page shows comparison between candidate and champion - -### Scenario 4: Rollback -1. Note current champion version -2. Promote a new model to champion -3. Rollback: `python flows/promote/flow.py run --version v{old}` -4. Verify: Models page shows old version as champion again - -## What to Verify After Each Flow - -| Flow | Expected Dashboard Changes | -|------|---------------------------| -| IngestMarketDataFlow | Data page: market_snapshot asset appears/updates | -| BuildDatasetFlow | Data page: training_dataset and eval_holdout appear | -| TrainDetectorFlow | Models page: new version with "candidate" status | -| EvaluateDetectorFlow | Models page: status changes to "evaluated" | -| PromoteDetectorFlow | Models page: "champion" badge on promoted version | - -## Key Files - -| File | Purpose | -|------|---------| -| `flows/ingest/flow.py` | IngestMarketDataFlow | -| `flows/build_dataset/flow.py` | BuildDatasetFlow | -| `flows/train/flow.py` | TrainDetectorFlow | -| `flows/evaluate/flow.py` | EvaluateDetectorFlow | -| `flows/promote/flow.py` | PromoteDetectorFlow | -| `configs/model.json` | Isolation Forest config | -| `configs/model_lof.json` | Local Outlier Factor config | -| `configs/training.json` | Data source config | -| `scripts/validate_dashboard.py` | Playwright UI validation | - -## Troubleshooting - -**Flow fails with "No data asset found"**: Run IngestMarketDataFlow first, or use `--config-value training_config '{"data_version": null}'` to fetch fresh data. - -**Dashboard shows stale data**: Refresh the page. The dashboard reads from the asset registry on each request. - -**Playwright validation fails**: Ensure dashboard is running at http://localhost:8001 diff --git a/.claude/agents/ui-improver.md b/.claude/agents/ui-improver.md deleted file mode 100644 index 15a5978..0000000 --- a/.claude/agents/ui-improver.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -name: ui-improver -description: Iteratively improves the model registry dashboard using playwright validation -tools: Read, Edit, Write, Bash, Glob, Grep -model: sonnet ---- - -You improve the dashboard UI for a model registry reference implementation on Outerbounds. - -## What This Project Demonstrates - -This is a reference implementation showing how to build production ML systems with Metaflow + Outerbounds. The dashboard must clearly demonstrate these model registry capabilities: - -### Model Lifecycle -- **Create models** - TrainDetectorFlow trains and registers new model versions -- **Version lineage** - Each version links to its training run, metrics, and dataset -- **Status progression** - candidate → evaluated → champion -- **Aliases** - Tag versions as "champion" or "production" for serving - -### What Users Need to See -- **Model versions table** - All versions with status, algorithm, metrics, training run link -- **Comparison view** - Side-by-side metrics between any two versions -- **Promotion workflow** - Evaluate → approve → promote to champion -- **Data lineage** - Which dataset trained which model version -- **Quality gates** - Pass/fail criteria that guard promotion - -### Key Metrics to Display -- Anomaly rate (training vs evaluation) -- Silhouette score (cluster separation quality) -- Score gap (normal vs anomaly separation) -- Training samples count -- Evaluation timestamp - -## Validation Workflow - -1. **Run tests** to see current state: - ```bash - python scripts/validate_dashboard.py --branch prod - ``` - -2. **Read screenshots** when tests fail - they show exactly what users see: - ``` - scripts/screenshots/*_01_overview.png - Pipeline status - scripts/screenshots/*_02_data.png - Dataset lineage - scripts/screenshots/*_03_models.png - Model versions & comparison - scripts/screenshots/*_04_cards.png - Metaflow card visualizations - ``` - -3. **Fix issues** in priority order: - - 500 errors → `deployments/dashboard/app.py` - - Missing content → `deployments/dashboard/templates/*.html` - - Bad formatting → `templates/base.html` CSS - -4. **Re-run validation** until all 4 tests pass - -## Key Files - -| File | Purpose | -|------|---------| -| `deployments/dashboard/app.py` | FastAPI routes, data loading from asset registry | -| `deployments/dashboard/templates/models.html` | Model versions table, comparison UI | -| `deployments/dashboard/templates/data.html` | Dataset lineage view | -| `scripts/validate_dashboard.py` | Playwright validation suite | - -## Adding Validation Checks - -When you notice a missing UI element, add a check: - -```python -# Example: Verify model comparison shows metric deltas -checks["shows_metric_delta"] = "Difference" in page_content or "Delta" in page_content -print(f" Metric comparison: {'PASS' if checks['shows_metric_delta'] else 'FAIL'}") -``` - -## Dashboard Must Be Running - -```bash -curl -s http://localhost:8001/health # Should return {"status":"healthy"} -``` diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a92fa3c..992624e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -39,7 +39,7 @@ jobs: run: | python3 -m pip install -U requests python3 -m pip install outerbounds pyyaml - python3 -m pip install -U ob-project-utils==0.2.4rc3 + python3 -m pip install -U ob-project-utils - name: Configure Outerbounds run: |