diff --git a/src/execution/broker.py b/src/execution/broker.py index 2d05a47..22bd08d 100644 --- a/src/execution/broker.py +++ b/src/execution/broker.py @@ -185,6 +185,48 @@ def get_account(self) -> dict: "last_equity": last_equity, } + def get_recent_daily_closes(self, lookback_days: int = 10) -> list[tuple[str, float]]: + """Official regular-session daily CLOSE equity for recent trading days. + + Source: Alpaca portfolio_history at 1D timeframe with + ``extended_hours=False`` — the broker-side source of truth for + end-of-regular-session equity. Crucially, unlike ``account.last_equity`` + (which is the PRIOR trading day's close, and so is one day stale at the + 20:00 ET evening run), the LAST point here is TODAY's 4pm close. That + lets the evening report show a true close-to-close ("4pm-to-4pm") P&L + instead of a close-to-8pm-after-hours broker diff. + + Returns ``[(et_date_str, close_equity), ...]`` oldest-first, or ``[]`` + on any failure (caller falls back to the real-time P&L). Best-effort — + never raises. ET-date mapping mirrors scripts/export_alpaca_trades.py. + """ + from datetime import datetime, timedelta, timezone + from src.util.time import ET + try: + from alpaca.trading.requests import GetPortfolioHistoryRequest + now = datetime.now(timezone.utc) + req = GetPortfolioHistoryRequest( + timeframe="1D", extended_hours=False, + start=now - timedelta(days=lookback_days * 2 + 10), end=now, + ) + history = self.client.get_portfolio_history(history_filter=req) + except Exception as exc: + logger.warning("get_recent_daily_closes: portfolio_history failed: %s", exc) + return [] + timestamps = getattr(history, "timestamp", None) or [] + equities = getattr(history, "equity", None) or [] + out: list[tuple[str, float]] = [] + for i, ts in enumerate(timestamps): + if i >= len(equities) or equities[i] is None: + continue + try: + d = datetime.fromtimestamp(int(ts), tz=timezone.utc).astimezone(ET).strftime("%Y-%m-%d") + eq = float(equities[i]) + except (TypeError, ValueError, OSError): + continue + out.append((d, eq)) + return out + def get_positions(self) -> list[Position]: raw_positions = self.client.get_all_positions() positions = [] diff --git a/src/notifier.py b/src/notifier.py index 717e055..468deb7 100644 --- a/src/notifier.py +++ b/src/notifier.py @@ -449,40 +449,45 @@ def _append_evening_body(lines: list[str], result: dict) -> None: f"≥80% of the {dl_limit:.0f}% circuit-breaker limit" ) - # Daily P&L summary — the headline of the evening push. Operator - # wants to know "did I make money today" without grepping logs. + # Daily P&L summary — the headline of the evening push. Operator wants to + # know "did I make money today" without grepping logs. + # + # Prefer the TRUE close-to-close ("4pm-to-4pm") P&L the pipeline computed + # from Alpaca portfolio_history (pnl_4pm / equity_close = today's official + # regular-session close). That's clean of after-hours drift AND free of the + # off-by-one trap of differencing account.last_equity (which is the PRIOR + # day's close). Fall back to the real-time prior-close→now diff when the + # 4pm figures aren't available (API gap / legacy result dicts). daily_pnl = result.get("daily_pnl") total_value = result.get("total_value") - daily_ret = result.get("daily_return_pct") - if daily_pnl is not None and total_value is not None: - # daily_return_pct from pipeline is in % already (e.g. -0.35 - # for a 0.35% loss), but historical rows pre-2026 may have had - # different scaling — compute fresh for the message either way. - # - # Daily return is P&L over PRIOR-day equity, not current. Using - # current understates losses (denominator includes today's draw). - # Prior equity = total_value − daily_pnl by definition. CLAUDE.md - # mandates broker.last_equity as the canonical baseline; this - # arithmetic reconstructs the same number from the result dict - # without plumbing last_equity through. Audit 2026-05-27: a -5% - # day was displayed as ~-4.76% under the old denominator. - prior_equity = total_value - daily_pnl - # Format signs as prefix (-$373.46 not $-373.46) — the latter - # reads as "dollar minus 373" which is awkward. - if daily_pnl >= 0: - pnl_str = f"+${daily_pnl:,.2f}" + pnl_4pm = result.get("pnl_4pm") + equity_close = result.get("equity_close") + + def _fmt_pnl(v: float) -> str: + return f"+${v:,.2f}" if v >= 0 else f"-${abs(v):,.2f}" + + if pnl_4pm is not None and equity_close: + # baseline = prior official close = equity_close - pnl_4pm. + baseline = equity_close - pnl_4pm + if baseline > 0: + r = pnl_4pm / baseline * 100 + ret_str = f"+{r:.2f}%" if pnl_4pm >= 0 else f"{r:.2f}%" else: - pnl_str = f"-${abs(daily_pnl):,.2f}" + ret_str = "n/a" + lines.append(f"💰 Daily P&L: {_fmt_pnl(pnl_4pm)} ({ret_str}) · 4pm close") + lines.append(f" Equity: ${equity_close:,.2f}") + elif daily_pnl is not None and total_value is not None: + # Fallback: real-time diff (prior close → 8pm, includes after-hours). + # Return is P&L over PRIOR-day equity (= total_value − daily_pnl); using + # current equity would understate losses (denominator includes the draw). + prior_equity = total_value - daily_pnl if prior_equity > 0: ret_pct = (daily_pnl / prior_equity) * 100 - # ret already carries its own leading minus when negative. ret_str = f"+{ret_pct:.2f}%" if daily_pnl >= 0 else f"{ret_pct:.2f}%" else: - # prior_equity <= 0 → return % is mathematically undefined. - # Rendering "0.00%" would read as a real flat day; surface the - # undefined state instead so the operator isn't misled. + # prior_equity <= 0 → return % undefined; "0.00%" would mislead. ret_str = "n/a" - lines.append(f"💰 Daily P&L: {pnl_str} ({ret_str})") + lines.append(f"💰 Daily P&L: {_fmt_pnl(daily_pnl)} ({ret_str})") lines.append(f" Equity: ${total_value:,.2f}") # Suggested actions — surfaced HIGH in the message (right after the diff --git a/src/pipeline.py b/src/pipeline.py index 834695e..fb18dd0 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -5774,6 +5774,33 @@ def run_evening(self) -> dict: cost_usd=ev_result.cost_usd, ) + # True close-to-close ("4pm-to-4pm") P&L. account.last_equity is the + # PRIOR day's close (stale at the 20:00 ET evening run), and + # total_value here is the 8pm after-hours value — neither gives today's + # official 4pm close. Alpaca portfolio_history (extended_hours=False) + # does: its latest 1D point is today's regular-session close. We report + # the clean close-to-close P&L when available and store today's close + # for the audit trail; on any gap we fall back to the real-time diff. + equity_close = None + pnl_4pm = None + pnl_4pm_pct = None + try: + closes = self.broker.get_recent_daily_closes(lookback_days=10) + if closes and closes[-1][0] == today_str: + equity_close = closes[-1][1] + prev_close = closes[-2][1] if len(closes) >= 2 else None + if prev_close: + pnl_4pm = equity_close - prev_close + pnl_4pm_pct = pnl_4pm / prev_close * 100 + elif closes: + logger.info( + "4pm snapshot: portfolio_history latest date %s != today %s " + "(API lag?) — evening uses the real-time P&L fallback", + closes[-1][0], today_str, + ) + except Exception as e: + logger.warning("4pm snapshot fetch failed: %s — using real-time P&L", e) + # Save daily_pnl + insights atomically (Phase 4 #5). If the LLM # failed (analysis is None), still record the P&L number so the # audit trail is complete — just with empty insights fields. @@ -5782,6 +5809,7 @@ def run_evening(self) -> dict: date=today_str, total_value=total_value, daily_pnl=daily_pnl, daily_return_pct=daily_return_pct, + equity_close=equity_close, tomorrow_outlook=analysis.tomorrow_outlook, lessons=analysis.lessons, suggested_actions=analysis.suggested_actions, @@ -5806,6 +5834,7 @@ def run_evening(self) -> dict: total_value=total_value, daily_pnl=daily_pnl, daily_return_pct=daily_return_pct, + equity_close=equity_close, ) # Housekeeping: drop agent_logs older than 2 years (full_response bloats the DB @@ -5881,6 +5910,11 @@ def run_evening(self) -> dict: getattr(self.config, "risk", None), "max_daily_loss_pct", None, ), "stop_coverage_gaps": coverage_gaps, + # True 4pm-to-4pm headline P&L (None → notifier falls back to the + # real-time total_value/daily_pnl figures). + "equity_close": equity_close, + "pnl_4pm": pnl_4pm, + "pnl_4pm_pct": pnl_4pm_pct, } def _expected_sessions_missing_today(self) -> list[str]: diff --git a/src/storage/db.py b/src/storage/db.py index 545378f..8129ff7 100644 --- a/src/storage/db.py +++ b/src/storage/db.py @@ -147,6 +147,7 @@ def _create_tables(self): total_value REAL NOT NULL, daily_pnl REAL NOT NULL, daily_return_pct REAL NOT NULL, + equity_close REAL, timestamp TEXT NOT NULL DEFAULT (datetime('now')) ); @@ -216,6 +217,10 @@ def _ensure_column(table: str, column: str, ddl: str) -> None: _log.error("Schema migration failed for %s.%s: %s", table, column, e) _ensure_column("agent_logs", "input_message", "input_message TEXT DEFAULT ''") + # Today's official regular-session (4pm) close equity, captured from + # Alpaca portfolio_history — enables true close-to-close evening P&L + # instead of the close-to-8pm-AH broker diff. NULL for legacy rows. + _ensure_column("daily_pnl", "equity_close", "equity_close REAL") _ensure_column("trades", "stop_loss", "stop_loss REAL DEFAULT 0") _ensure_column("trades", "take_profit", "take_profit REAL DEFAULT 0") _ensure_column("insights", "tomorrow_bias", "tomorrow_bias TEXT DEFAULT 'neutral'") @@ -303,6 +308,7 @@ def save_evening_snapshot( total_value: float, daily_pnl: float, daily_return_pct: float, + equity_close: float | None = None, tomorrow_outlook: str, lessons: str, suggested_actions, @@ -362,9 +368,9 @@ def _to_json_list(val) -> str: self.conn.execute("BEGIN") self.conn.execute( "INSERT OR REPLACE INTO daily_pnl " - "(date, total_value, daily_pnl, daily_return_pct) " - "VALUES (?, ?, ?, ?)", - (date, total_value, daily_pnl, daily_return_pct), + "(date, total_value, daily_pnl, daily_return_pct, equity_close) " + "VALUES (?, ?, ?, ?, ?)", + (date, total_value, daily_pnl, daily_return_pct, equity_close), ) self.conn.execute( "INSERT OR REPLACE INTO insights " @@ -940,12 +946,13 @@ def prune_agent_logs(self, keep_days: int = 730) -> int: return cursor.rowcount or 0 def insert_daily_pnl(self, date: str, total_value: float, daily_pnl: float, - daily_return_pct: float): + daily_return_pct: float, equity_close: float | None = None): with self._lock: self.conn.execute( - """INSERT OR REPLACE INTO daily_pnl (date, total_value, daily_pnl, daily_return_pct) - VALUES (?, ?, ?, ?)""", - (date, total_value, daily_pnl, daily_return_pct), + """INSERT OR REPLACE INTO daily_pnl + (date, total_value, daily_pnl, daily_return_pct, equity_close) + VALUES (?, ?, ?, ?, ?)""", + (date, total_value, daily_pnl, daily_return_pct, equity_close), ) self.conn.commit() diff --git a/tests/test_broker.py b/tests/test_broker.py index 0871b4d..e7bc3c3 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -1500,3 +1500,33 @@ def test_list_recent_orders_returns_none_on_api_failure(mock_tc_cls): mock_client.get_orders.side_effect = None mock_client.get_orders.return_value = [] assert broker.list_recent_orders("NVDA", "buy", after) == [] + + +@patch("src.execution.broker.TradingClient") +def test_get_recent_daily_closes_maps_et_dates_and_equity(mock_tc_cls): + """portfolio_history (1D, extended_hours=False) → [(ET-date, close_equity)]. + 20:00 UTC = 16:00 EDT, so each bar maps to that ET trading date; equity[i] + is that day's official regular-session close.""" + from datetime import datetime, timezone + from types import SimpleNamespace + + ts1 = int(datetime(2026, 5, 27, 20, 0, tzinfo=timezone.utc).timestamp()) + ts2 = int(datetime(2026, 5, 28, 20, 0, tzinfo=timezone.utc).timestamp()) + mock_client = MagicMock() + mock_client.get_portfolio_history.return_value = SimpleNamespace( + timestamp=[ts1, ts2], equity=[101000.0, 100500.0], + ) + mock_tc_cls.return_value = mock_client + + broker = AlpacaBroker(api_key="k", secret_key="s", paper=True) + closes = broker.get_recent_daily_closes(lookback_days=5) + assert closes == [("2026-05-27", 101000.0), ("2026-05-28", 100500.0)] + + +@patch("src.execution.broker.TradingClient") +def test_get_recent_daily_closes_swallows_errors(mock_tc_cls): + mock_client = MagicMock() + mock_client.get_portfolio_history.side_effect = RuntimeError("api down") + mock_tc_cls.return_value = mock_client + broker = AlpacaBroker(api_key="k", secret_key="s", paper=True) + assert broker.get_recent_daily_closes() == [] # best-effort, never raises diff --git a/tests/test_db.py b/tests/test_db.py index 45f2229..6f80e7b 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -677,3 +677,14 @@ def test_session_prefixes_logged_on_extracts_run_id_prefixes(db): assert "run" in prefixes # morning ran assert "midday" in prefixes # midday ran assert "close" not in prefixes # close did NOT run today + + +def test_daily_pnl_equity_close_roundtrips(db): + """equity_close (today's official 4pm close) persists + reads back.""" + db.insert_daily_pnl("2026-05-28", 100_400.0, -600.0, -0.59, equity_close=100_500.0) + rows = db.get_daily_pnl(limit=1) + assert rows[0]["equity_close"] == 100_500.0 + # legacy path (no equity_close) stores NULL, not a crash + db.insert_daily_pnl("2026-05-29", 100_000.0, 0.0, 0.0) + rows = db.get_daily_pnl(limit=1) + assert rows[0]["equity_close"] is None diff --git a/tests/test_notifier.py b/tests/test_notifier.py index 1bc530d..c2a3225 100644 --- a/tests/test_notifier.py +++ b/tests/test_notifier.py @@ -974,3 +974,36 @@ def test_format_evening_renders_stop_coverage_gap_banner(): msg = format_session_result("evening", result, 5.0) assert "🔴 STOP-COVERAGE GAP" in msg assert "AAPL" in msg + + +def test_format_evening_uses_true_4pm_pnl_not_offset_day(): + """The 4pm-to-4pm headline uses pnl_4pm/equity_close (today's OFFICIAL + close, computed by the pipeline) directly. Regression against the off-by- + one that differenced account.last_equity and showed the PRIOR day's P&L: + here the real-time figures say +$1,200 (incl. after-hours) but today + actually closed DOWN $500 — the headline must show -$500, not +$1,200.""" + result = { + "status": "analyzed", "run_id": "r", + "daily_pnl": 1200.0, "total_value": 101_200.0, # real-time (incl AH) — ignored + "pnl_4pm": -500.0, "equity_close": 100_500.0, # today's true close-to-close + "analysis": {"risk_rating": "low"}, + } + msg = format_session_result("evening", result, 10.0) + assert "💰 Daily P&L: -$500.00" in msg + assert "4pm close" in msg + assert "$100,500.00" in msg + assert "+$1,200" not in msg # must not leak the real-time figure + assert "(-0.50%)" in msg # -500 / (100500+500) = -0.495% → -0.50% + + +def test_format_evening_falls_back_to_realtime_when_no_4pm(): + """No pnl_4pm/equity_close (API gap / legacy) → real-time fallback, no + '4pm close' tag.""" + result = { + "status": "analyzed", "run_id": "r", + "daily_pnl": 1234.56, "total_value": 107_278.55, + "analysis": {"risk_rating": "low"}, + } + msg = format_session_result("evening", result, 10.0) + assert "💰 Daily P&L: +$1,234.56" in msg + assert "4pm close" not in msg