fix(notifier): evening 4pm-to-4pm P&L from portfolio_history (supersedes #95)

src/execution/broker.py

@@ -185,6 +185,48 @@ def get_account(self) -> dict:
             "last_equity": last_equity,
         }
 
+    def get_recent_daily_closes(self, lookback_days: int = 10) -> list[tuple[str, float]]:
+        """Official regular-session daily CLOSE equity for recent trading days.
+
+        Source: Alpaca portfolio_history at 1D timeframe with
+        ``extended_hours=False`` — the broker-side source of truth for
+        end-of-regular-session equity. Crucially, unlike ``account.last_equity``
+        (which is the PRIOR trading day's close, and so is one day stale at the
+        20:00 ET evening run), the LAST point here is TODAY's 4pm close. That
+        lets the evening report show a true close-to-close ("4pm-to-4pm") P&L
+        instead of a close-to-8pm-after-hours broker diff.
+
+        Returns ``[(et_date_str, close_equity), ...]`` oldest-first, or ``[]``
+        on any failure (caller falls back to the real-time P&L). Best-effort —
+        never raises. ET-date mapping mirrors scripts/export_alpaca_trades.py.
+        """
+        from datetime import datetime, timedelta, timezone
+        from src.util.time import ET
+        try:
+            from alpaca.trading.requests import GetPortfolioHistoryRequest
+            now = datetime.now(timezone.utc)
+            req = GetPortfolioHistoryRequest(
+                timeframe="1D", extended_hours=False,
+                start=now - timedelta(days=lookback_days * 2 + 10), end=now,
+            )
+            history = self.client.get_portfolio_history(history_filter=req)
+        except Exception as exc:
+            logger.warning("get_recent_daily_closes: portfolio_history failed: %s", exc)
+            return []
+        timestamps = getattr(history, "timestamp", None) or []
+        equities = getattr(history, "equity", None) or []
+        out: list[tuple[str, float]] = []
+        for i, ts in enumerate(timestamps):
+            if i >= len(equities) or equities[i] is None:
+                continue
+            try:
+                d = datetime.fromtimestamp(int(ts), tz=timezone.utc).astimezone(ET).strftime("%Y-%m-%d")
+                eq = float(equities[i])
+            except (TypeError, ValueError, OSError):
+                continue
+            out.append((d, eq))
+        return out
+
     def get_positions(self) -> list[Position]:
         raw_positions = self.client.get_all_positions()
         positions = []

src/notifier.py

@@ -449,40 +449,45 @@ def _append_evening_body(lines: list[str], result: dict) -> None:
                     f"≥80% of the {dl_limit:.0f}% circuit-breaker limit"
                 )
 
-    # Daily P&L summary — the headline of the evening push. Operator
-    # wants to know "did I make money today" without grepping logs.
+    # Daily P&L summary — the headline of the evening push. Operator wants to
+    # know "did I make money today" without grepping logs.
+    #
+    # Prefer the TRUE close-to-close ("4pm-to-4pm") P&L the pipeline computed
+    # from Alpaca portfolio_history (pnl_4pm / equity_close = today's official
+    # regular-session close). That's clean of after-hours drift AND free of the
+    # off-by-one trap of differencing account.last_equity (which is the PRIOR
+    # day's close). Fall back to the real-time prior-close→now diff when the
+    # 4pm figures aren't available (API gap / legacy result dicts).
     daily_pnl = result.get("daily_pnl")
     total_value = result.get("total_value")
-    daily_ret = result.get("daily_return_pct")
-    if daily_pnl is not None and total_value is not None:
-        # daily_return_pct from pipeline is in % already (e.g. -0.35
-        # for a 0.35% loss), but historical rows pre-2026 may have had
-        # different scaling — compute fresh for the message either way.
-        #
-        # Daily return is P&L over PRIOR-day equity, not current. Using
-        # current understates losses (denominator includes today's draw).
-        # Prior equity = total_value − daily_pnl by definition. CLAUDE.md
-        # mandates broker.last_equity as the canonical baseline; this
-        # arithmetic reconstructs the same number from the result dict
-        # without plumbing last_equity through. Audit 2026-05-27: a -5%
-        # day was displayed as ~-4.76% under the old denominator.
-        prior_equity = total_value - daily_pnl
-        # Format signs as prefix (-$373.46 not $-373.46) — the latter
-        # reads as "dollar minus 373" which is awkward.
-        if daily_pnl >= 0:
-            pnl_str = f"+${daily_pnl:,.2f}"
+    pnl_4pm = result.get("pnl_4pm")
+    equity_close = result.get("equity_close")
+
+    def _fmt_pnl(v: float) -> str:
+        return f"+${v:,.2f}" if v >= 0 else f"-${abs(v):,.2f}"
+
+    if pnl_4pm is not None and equity_close:
+        # baseline = prior official close = equity_close - pnl_4pm.
+        baseline = equity_close - pnl_4pm
+        if baseline > 0:
+            r = pnl_4pm / baseline * 100
+            ret_str = f"+{r:.2f}%" if pnl_4pm >= 0 else f"{r:.2f}%"
         else:
-            pnl_str = f"-${abs(daily_pnl):,.2f}"
+            ret_str = "n/a"
+        lines.append(f"💰 Daily P&L: {_fmt_pnl(pnl_4pm)} ({ret_str})  ·  4pm close")
+        lines.append(f"   Equity: ${equity_close:,.2f}")
+    elif daily_pnl is not None and total_value is not None:
+        # Fallback: real-time diff (prior close → 8pm, includes after-hours).
+        # Return is P&L over PRIOR-day equity (= total_value − daily_pnl); using
+        # current equity would understate losses (denominator includes the draw).
+        prior_equity = total_value - daily_pnl
         if prior_equity > 0:
             ret_pct = (daily_pnl / prior_equity) * 100
-            # ret already carries its own leading minus when negative.
             ret_str = f"+{ret_pct:.2f}%" if daily_pnl >= 0 else f"{ret_pct:.2f}%"
         else:
-            # prior_equity <= 0 → return % is mathematically undefined.
-            # Rendering "0.00%" would read as a real flat day; surface the
-            # undefined state instead so the operator isn't misled.
+            # prior_equity <= 0 → return % undefined; "0.00%" would mislead.
             ret_str = "n/a"
-        lines.append(f"💰 Daily P&L: {pnl_str} ({ret_str})")
+        lines.append(f"💰 Daily P&L: {_fmt_pnl(daily_pnl)} ({ret_str})")
         lines.append(f"   Equity: ${total_value:,.2f}")
 
     # Suggested actions — surfaced HIGH in the message (right after the

src/pipeline.py

@@ -5774,6 +5774,33 @@ def run_evening(self) -> dict:
             cost_usd=ev_result.cost_usd,
         )
 
+        # True close-to-close ("4pm-to-4pm") P&L. account.last_equity is the
+        # PRIOR day's close (stale at the 20:00 ET evening run), and
+        # total_value here is the 8pm after-hours value — neither gives today's
+        # official 4pm close. Alpaca portfolio_history (extended_hours=False)
+        # does: its latest 1D point is today's regular-session close. We report
+        # the clean close-to-close P&L when available and store today's close
+        # for the audit trail; on any gap we fall back to the real-time diff.
+        equity_close = None
+        pnl_4pm = None
+        pnl_4pm_pct = None
+        try:
+            closes = self.broker.get_recent_daily_closes(lookback_days=10)
+            if closes and closes[-1][0] == today_str:
+                equity_close = closes[-1][1]
+                prev_close = closes[-2][1] if len(closes) >= 2 else None
+                if prev_close:
+                    pnl_4pm = equity_close - prev_close
+                    pnl_4pm_pct = pnl_4pm / prev_close * 100
+            elif closes:
+                logger.info(
+                    "4pm snapshot: portfolio_history latest date %s != today %s "
+                    "(API lag?) — evening uses the real-time P&L fallback",
+                    closes[-1][0], today_str,
+                )
+        except Exception as e:
+            logger.warning("4pm snapshot fetch failed: %s — using real-time P&L", e)
+
         # Save daily_pnl + insights atomically (Phase 4 #5). If the LLM
         # failed (analysis is None), still record the P&L number so the
         # audit trail is complete — just with empty insights fields.
@@ -5782,6 +5809,7 @@ def run_evening(self) -> dict:
                 date=today_str,
                 total_value=total_value, daily_pnl=daily_pnl,
                 daily_return_pct=daily_return_pct,
+                equity_close=equity_close,
                 tomorrow_outlook=analysis.tomorrow_outlook,
                 lessons=analysis.lessons,
                 suggested_actions=analysis.suggested_actions,
@@ -5806,6 +5834,7 @@ def run_evening(self) -> dict:
                 total_value=total_value,
                 daily_pnl=daily_pnl,
                 daily_return_pct=daily_return_pct,
+                equity_close=equity_close,
             )
 
         # Housekeeping: drop agent_logs older than 2 years (full_response bloats the DB
@@ -5881,6 +5910,11 @@ def run_evening(self) -> dict:
                 getattr(self.config, "risk", None), "max_daily_loss_pct", None,
             ),
             "stop_coverage_gaps": coverage_gaps,
+            # True 4pm-to-4pm headline P&L (None → notifier falls back to the
+            # real-time total_value/daily_pnl figures).
+            "equity_close": equity_close,
+            "pnl_4pm": pnl_4pm,
+            "pnl_4pm_pct": pnl_4pm_pct,
         }
 
     def _expected_sessions_missing_today(self) -> list[str]:

src/storage/db.py

@@ -147,6 +147,7 @@ def _create_tables(self):
                 total_value REAL NOT NULL,
                 daily_pnl REAL NOT NULL,
                 daily_return_pct REAL NOT NULL,
+                equity_close REAL,
                 timestamp TEXT NOT NULL DEFAULT (datetime('now'))
             );
 
@@ -216,6 +217,10 @@ def _ensure_column(table: str, column: str, ddl: str) -> None:
                 _log.error("Schema migration failed for %s.%s: %s", table, column, e)
 
         _ensure_column("agent_logs", "input_message", "input_message TEXT DEFAULT ''")
+        # Today's official regular-session (4pm) close equity, captured from
+        # Alpaca portfolio_history — enables true close-to-close evening P&L
+        # instead of the close-to-8pm-AH broker diff. NULL for legacy rows.
+        _ensure_column("daily_pnl", "equity_close", "equity_close REAL")
         _ensure_column("trades", "stop_loss", "stop_loss REAL DEFAULT 0")
         _ensure_column("trades", "take_profit", "take_profit REAL DEFAULT 0")
         _ensure_column("insights", "tomorrow_bias", "tomorrow_bias TEXT DEFAULT 'neutral'")
@@ -303,6 +308,7 @@ def save_evening_snapshot(
         total_value: float,
         daily_pnl: float,
         daily_return_pct: float,
+        equity_close: float | None = None,
         tomorrow_outlook: str,
         lessons: str,
         suggested_actions,
@@ -362,9 +368,9 @@ def _to_json_list(val) -> str:
                 self.conn.execute("BEGIN")
                 self.conn.execute(
                     "INSERT OR REPLACE INTO daily_pnl "
-                    "(date, total_value, daily_pnl, daily_return_pct) "
-                    "VALUES (?, ?, ?, ?)",
-                    (date, total_value, daily_pnl, daily_return_pct),
+                    "(date, total_value, daily_pnl, daily_return_pct, equity_close) "
+                    "VALUES (?, ?, ?, ?, ?)",
+                    (date, total_value, daily_pnl, daily_return_pct, equity_close),
                 )
                 self.conn.execute(
                     "INSERT OR REPLACE INTO insights "
@@ -940,12 +946,13 @@ def prune_agent_logs(self, keep_days: int = 730) -> int:
             return cursor.rowcount or 0
 
     def insert_daily_pnl(self, date: str, total_value: float, daily_pnl: float,
-                         daily_return_pct: float):
+                         daily_return_pct: float, equity_close: float | None = None):
         with self._lock:
             self.conn.execute(
-                """INSERT OR REPLACE INTO daily_pnl (date, total_value, daily_pnl, daily_return_pct)
-                   VALUES (?, ?, ?, ?)""",
-                (date, total_value, daily_pnl, daily_return_pct),
+                """INSERT OR REPLACE INTO daily_pnl
+                   (date, total_value, daily_pnl, daily_return_pct, equity_close)
+                   VALUES (?, ?, ?, ?, ?)""",
+                (date, total_value, daily_pnl, daily_return_pct, equity_close),
             )
             self.conn.commit()
 

tests/test_broker.py

@@ -1500,3 +1500,33 @@ def test_list_recent_orders_returns_none_on_api_failure(mock_tc_cls):
     mock_client.get_orders.side_effect = None
     mock_client.get_orders.return_value = []
     assert broker.list_recent_orders("NVDA", "buy", after) == []
+
+
+@patch("src.execution.broker.TradingClient")
+def test_get_recent_daily_closes_maps_et_dates_and_equity(mock_tc_cls):
+    """portfolio_history (1D, extended_hours=False) → [(ET-date, close_equity)].
+    20:00 UTC = 16:00 EDT, so each bar maps to that ET trading date; equity[i]
+    is that day's official regular-session close."""
+    from datetime import datetime, timezone
+    from types import SimpleNamespace
+
+    ts1 = int(datetime(2026, 5, 27, 20, 0, tzinfo=timezone.utc).timestamp())
+    ts2 = int(datetime(2026, 5, 28, 20, 0, tzinfo=timezone.utc).timestamp())
+    mock_client = MagicMock()
+    mock_client.get_portfolio_history.return_value = SimpleNamespace(
+        timestamp=[ts1, ts2], equity=[101000.0, 100500.0],
+    )
+    mock_tc_cls.return_value = mock_client
+
+    broker = AlpacaBroker(api_key="k", secret_key="s", paper=True)
+    closes = broker.get_recent_daily_closes(lookback_days=5)
+    assert closes == [("2026-05-27", 101000.0), ("2026-05-28", 100500.0)]
+
+
+@patch("src.execution.broker.TradingClient")
+def test_get_recent_daily_closes_swallows_errors(mock_tc_cls):
+    mock_client = MagicMock()
+    mock_client.get_portfolio_history.side_effect = RuntimeError("api down")
+    mock_tc_cls.return_value = mock_client
+    broker = AlpacaBroker(api_key="k", secret_key="s", paper=True)
+    assert broker.get_recent_daily_closes() == []   # best-effort, never raises

tests/test_db.py

@@ -677,3 +677,14 @@ def test_session_prefixes_logged_on_extracts_run_id_prefixes(db):
     assert "run" in prefixes      # morning ran
     assert "midday" in prefixes   # midday ran
     assert "close" not in prefixes  # close did NOT run today
+
+
+def test_daily_pnl_equity_close_roundtrips(db):
+    """equity_close (today's official 4pm close) persists + reads back."""
+    db.insert_daily_pnl("2026-05-28", 100_400.0, -600.0, -0.59, equity_close=100_500.0)
+    rows = db.get_daily_pnl(limit=1)
+    assert rows[0]["equity_close"] == 100_500.0
+    # legacy path (no equity_close) stores NULL, not a crash
+    db.insert_daily_pnl("2026-05-29", 100_000.0, 0.0, 0.0)
+    rows = db.get_daily_pnl(limit=1)
+    assert rows[0]["equity_close"] is None

tests/test_notifier.py

@@ -974,3 +974,36 @@ def test_format_evening_renders_stop_coverage_gap_banner():
     msg = format_session_result("evening", result, 5.0)
     assert "🔴 STOP-COVERAGE GAP" in msg
     assert "AAPL" in msg
+
+
+def test_format_evening_uses_true_4pm_pnl_not_offset_day():
+    """The 4pm-to-4pm headline uses pnl_4pm/equity_close (today's OFFICIAL
+    close, computed by the pipeline) directly. Regression against the off-by-
+    one that differenced account.last_equity and showed the PRIOR day's P&L:
+    here the real-time figures say +$1,200 (incl. after-hours) but today
+    actually closed DOWN $500 — the headline must show -$500, not +$1,200."""
+    result = {
+        "status": "analyzed", "run_id": "r",
+        "daily_pnl": 1200.0, "total_value": 101_200.0,   # real-time (incl AH) — ignored
+        "pnl_4pm": -500.0, "equity_close": 100_500.0,     # today's true close-to-close
+        "analysis": {"risk_rating": "low"},
+    }
+    msg = format_session_result("evening", result, 10.0)
+    assert "💰 Daily P&L: -$500.00" in msg
+    assert "4pm close" in msg
+    assert "$100,500.00" in msg
+    assert "+$1,200" not in msg                 # must not leak the real-time figure
+    assert "(-0.50%)" in msg                    # -500 / (100500+500) = -0.495% → -0.50%
+
+
+def test_format_evening_falls_back_to_realtime_when_no_4pm():
+    """No pnl_4pm/equity_close (API gap / legacy) → real-time fallback, no
+    '4pm close' tag."""
+    result = {
+        "status": "analyzed", "run_id": "r",
+        "daily_pnl": 1234.56, "total_value": 107_278.55,
+        "analysis": {"risk_rating": "low"},
+    }
+    msg = format_session_result("evening", result, 10.0)
+    assert "💰 Daily P&L: +$1,234.56" in msg
+    assert "4pm close" not in msg