Fix 10%/90%/CL calibrated PV values being ~2× too low with Open-Meteo

apps/predbat/solcast.py

@@ -879,12 +879,23 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period):
                     },
                 )
 
-    def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10, divide_by, max_kwh, forecast_days):
+    def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10, divide_by, max_kwh, forecast_days, period=None):
         """
         Perform PV calibration based on historical data and forecast data.
         This will adjust the forecast data based on historical PV production and forecast data.
         It will also create pv_estimate10 and pv_estimate90 data if create_pv10 is True.
         """
+        # If no period is given, default to the plan interval (backward-compatible for unit tests).
+        if period is None:
+            period = self.plan_interval_minutes
+        # Number of plan-interval slots that span one forecast entry period.
+        # For 30-min plan slots with a 60-min forecast (Open-Meteo) this is 2,
+        # for 30-min plan slots with a 30-min forecast (Solcast) this is 1.
+        # Use ceiling so partial forecast periods are fully covered rather than rounded down.
+        if period % self.plan_interval_minutes != 0:
+            self.log("Warn: PV calibration forecast period {} does not divide evenly into plan interval {} - using ceiling slot coverage".format(period, self.plan_interval_minutes))
+        slots_per_period = max(1, int(math.ceil(period / self.plan_interval_minutes)))
+
         self.log("PV Calibration: Fetching PV data for calibration")
 
         days = 7
@@ -1075,16 +1086,38 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_d
             if period_start:
                 minutes_since_midnight = (datetime.strptime(period_start, TIME_FORMAT) - self.midnight_utc).total_seconds() / 60
                 slot = int(minutes_since_midnight / self.plan_interval_minutes) * self.plan_interval_minutes
-                calibrated = pv_estimateCL.get(slot, None)
-                calibrated10 = pv_estimate10.get(slot, None)
-                calibrated90 = pv_estimate90.get(slot, None)
+
+                # Sum all plan-interval slots that fall within this forecast entry's period.
+                # When the forecast resolution is coarser than plan_interval_minutes (e.g. 60-min
+                # Open-Meteo entries with 30-min plan slots) we must accumulate multiple slots so
+                # that the annotated value covers the full entry duration, not just the first half.
+                calibrated = 0
+                calibrated10 = 0
+                calibrated90 = 0
+                has_calibrated = False
+                has_calibrated10 = False
+                has_calibrated90 = False
+                for i in range(slots_per_period):
+                    s = slot + i * self.plan_interval_minutes
+                    v = pv_estimateCL.get(s, None)
+                    if v is not None:
+                        calibrated += v
+                        has_calibrated = True
+                    v10 = pv_estimate10.get(s, None)
+                    if v10 is not None:
+                        calibrated10 += v10
+                        has_calibrated10 = True
+                    v90 = pv_estimate90.get(s, None)
+                    if v90 is not None:
+                        calibrated90 += v90
+                        has_calibrated90 = True
 
                 # When we store the data we have to reverse the divide_by factor
-                if calibrated is not None:
+                if has_calibrated:
                     entry["pv_estimateCL"] = calibrated * divide_by
-                if create_pv10 and (calibrated10 is not None):
+                if create_pv10 and has_calibrated10:
                     entry["pv_estimate10"] = calibrated10 * divide_by
-                if create_pv10 and (calibrated90 is not None):
+                if create_pv10 and has_calibrated90:
                     entry["pv_estimate90"] = calibrated90 * divide_by
 
         # Creation of PV10 data using worst day scaling factor
@@ -1245,7 +1278,7 @@ async def fetch_pv_forecast(self):
             )
 
             # Run calibration on the data
-            pv_forecast_minute, pv_forecast_minute10, pv_forecast_data = self.pv_calibration(pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10, divide_by / period, max_kwh, self.forecast_days)
+            pv_forecast_minute, pv_forecast_minute10, pv_forecast_data = self.pv_calibration(pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10, divide_by / period, max_kwh, self.forecast_days, period)
             self.publish_pv_stats(pv_forecast_data, divide_by / period, period)
             self.pack_and_store_forecast(pv_forecast_minute, pv_forecast_minute10)
             self.update_success_timestamp()

apps/predbat/tests/test_solcast.py

@@ -2414,6 +2414,263 @@ def mock_minute_import_export(max_days_prev, now_utc, key, scale=1.0, required_u
     return failed
 
 
+def test_pv_calibration_60min_period(my_predbat):
+    """
+    Test that pv_calibration correctly annotates pv_forecast_data entries when the
+    forecast period (60 min, as used by Open-Meteo) is coarser than the plan interval
+    (30 min by default).
+
+    Before the fix, each 60-min entry was annotated with only the first 30-min plan
+    slot's calibrated value, producing values that were approximately half the correct
+    amount.  After the fix, both 30-min slots within the 60-min window are summed.
+    """
+    print("  - test_pv_calibration_60min_period")
+    failed = False
+
+    GEN_START = 480  # 8:00 UTC in minutes since midnight
+    GEN_END = 600  # 10:00 UTC (2 hours = 2 × 60-min entries; safely before noon)
+    FORECAST_KW = 2.0  # kW during generation window
+    PLAN_INTERVAL = 30  # minutes
+    FORECAST_PERIOD = 60  # minutes (Open-Meteo resolution)
+    TOL = 0.15  # 15% tolerance (matches other calibration tests; allows for minor slot-boundary effects)
+
+    test_api = create_test_solar_api()
+    solar = test_api.solar
+    base = test_api.mock_base
+    base.plan_interval_minutes = PLAN_INTERVAL
+
+    # Flat historical production matching the forecast → calibration ratio ~1.0
+    # so the calibrated values should be very close to the raw forecast values.
+    hist = {}
+    days = 5
+    minutes_now = base.minutes_now  # 720 (noon)
+    for day_idx in range(days):
+        day = day_idx + 1
+        midnight_ago = day * 1440 + minutes_now
+        for step in range(0, 24 * 60, 5):
+            minute_ago = midnight_ago - step
+            if minute_ago < 0:
+                continue
+            actual_min = step
+            if actual_min < GEN_START:
+                cumulative = 0.0
+            elif actual_min < GEN_END:
+                cumulative = FORECAST_KW * (actual_min - GEN_START) / 60.0
+            else:
+                cumulative = FORECAST_KW * (GEN_END - GEN_START) / 60.0
+            hist[minute_ago] = cumulative
+
+    def mock_minute_import_export(max_days_prev, now_utc, key, scale=1.0, required_unit=None, increment=True, smoothing=True, pad=True, _hist=hist):
+        """Mock historical PV data."""
+        return dict(_hist) if key == "pv_today" else {}
+
+    base.minute_data_import_export = mock_minute_import_export
+    solar.get_history_wrapper = lambda entity_id, days, required=False: []
+
+    # Build per-minute forecast arrays (as minute_data() would produce)
+    # Each minute in the gen window has FORECAST_KW / 60 kWh/min
+    total_minutes = 4 * 24 * 60
+    pv_m = {}
+    pv_m10 = {}
+    for m in range(total_minutes):
+        val = (FORECAST_KW / 60.0) if GEN_START <= m < GEN_END else 0.0
+        pv_m[m] = val
+        pv_m10[m] = val
+
+    # Build 60-min forecast data entries (like Open-Meteo would produce).
+    # Each entry covers FORECAST_PERIOD minutes and holds FORECAST_KW * (FORECAST_PERIOD/60) kWh.
+    midnight = datetime(2025, 6, 15, 0, 0, 0, tzinfo=pytz.utc)
+    pv_data = []
+    for slot in range(GEN_START, GEN_END, FORECAST_PERIOD):
+        ts = midnight + timedelta(minutes=slot)
+        kwh_per_entry = FORECAST_KW * FORECAST_PERIOD / 60.0
+        pv_data.append({"period_start": ts.strftime("%Y-%m-%dT%H:%M:%S+0000"), "pv_estimate": kwh_per_entry, "pv_estimate10": kwh_per_entry, "pv_estimate90": kwh_per_entry})
+
+    # divide_by passed to pv_calibration = divide_by_full / period = factor.
+    # For kWh entries factor = 1.0.
+    divide_by_factor = 1.0
+
+    # Build historic forecast dict (per-minute power in kW, keyed by minutes-ago)
+    pv_forecast_hist = {}
+    for day_num in range(1, days + 1):
+        for m_of_day in range(GEN_START, GEN_END):
+            minutes_ago = day_num * 1440 + (minutes_now - m_of_day)
+            pv_forecast_hist[minutes_ago] = float(FORECAST_KW)
+
+    with patch("solcast.history_attribute_to_minute_data", return_value=(pv_forecast_hist, days)):
+        adj_m, adj_m10, adj_data = solar.pv_calibration(pv_m, pv_m10, pv_data, create_pv10=True, divide_by=divide_by_factor, max_kwh=10.0, forecast_days=solar.forecast_days, period=FORECAST_PERIOD)
+
+    # Each annotated entry should cover the full FORECAST_PERIOD minutes.
+    # Expected calibrated kWh per entry ≈ FORECAST_KW * FORECAST_PERIOD / 60 = 2.0 kWh.
+    # With the pre-fix bug, only the first 30-min plan slot was summed, giving ~1.0 kWh (half).
+    expected_kwh = FORECAST_KW * FORECAST_PERIOD / 60.0
+    half_expected = expected_kwh / 2.0
+
+    entries_validated = 0
+    for entry in adj_data:
+        cl = entry.get("pv_estimateCL")
+        e10 = entry.get("pv_estimate10")
+        e90 = entry.get("pv_estimate90")
+
+        if cl is None or cl == 0:
+            continue
+
+        entries_validated += 1
+
+        # The calibrated value must be close to the full 60-min kWh (not the buggy half-period value).
+        if abs(cl - expected_kwh) > TOL * expected_kwh:
+            print("ERROR: pv_estimateCL ({:.4f}) should be ~{:.4f} (full 60-min period kWh); half-period bug would give ~{:.4f}".format(cl, expected_kwh, half_expected))
+            failed = True
+            break
+
+        if e10 is not None and e10 > 0:
+            # e10 is worst-day scaling × CL; must be > 0 and plausibly related to CL
+            if e10 > cl * 2.0:
+                print("ERROR: pv_estimate10 ({:.4f}) is unexpectedly much larger than pv_estimateCL ({:.4f})".format(e10, cl))
+                failed = True
+                break
+
+        if e90 is not None and e90 < cl * (1.0 - TOL):
+            print("ERROR: pv_estimate90 ({:.4f}) should be >= pv_estimateCL ({:.4f})".format(e90, cl))
+            failed = True
+            break
+
+    if entries_validated == 0:
+        print("ERROR: pv_calibration() annotated no entries with pv_estimateCL; annotation step may have regressed")
+        failed = True
+
+    test_api.cleanup()
+    return failed
+
+
+def test_pv_calibration_15min_period(my_predbat):
+    """
+    Test that pv_calibration correctly annotates pv_forecast_data entries when the
+    forecast period (15 min) is finer than the plan interval (30 min by default).
+
+    Each 15-min entry covers half a 30-min plan slot, so slots_per_period=1 and only
+    the single plan slot that starts at the entry's timestamp is used.  The annotated
+    pv_estimateCL should therefore be close to FORECAST_KW * 15/60 kWh (not double).
+    """
+    print("  - test_pv_calibration_15min_period")
+    failed = False
+
+    GEN_START = 480  # 8:00 UTC in minutes since midnight
+    GEN_END = 600  # 10:00 UTC (2 hours = 8 × 15-min entries)
+    FORECAST_KW = 2.0  # kW during generation window
+    PLAN_INTERVAL = 30  # minutes (production default)
+    FORECAST_PERIOD = 15  # minutes (forecast.solar / fine-resolution Solcast)
+    TOL = 0.15  # 15% tolerance
+
+    test_api = create_test_solar_api()
+    solar = test_api.solar
+    base = test_api.mock_base
+    base.plan_interval_minutes = PLAN_INTERVAL
+
+    # Flat historical production matching the forecast → calibration ratio ~1.0
+    hist = {}
+    days = 5
+    minutes_now = base.minutes_now  # 720 (noon)
+    for day_idx in range(days):
+        day = day_idx + 1
+        midnight_ago = day * 1440 + minutes_now
+        for step in range(0, 24 * 60, 5):
+            minute_ago = midnight_ago - step
+            if minute_ago < 0:
+                continue
+            actual_min = step
+            if actual_min < GEN_START:
+                cumulative = 0.0
+            elif actual_min < GEN_END:
+                cumulative = FORECAST_KW * (actual_min - GEN_START) / 60.0
+            else:
+                cumulative = FORECAST_KW * (GEN_END - GEN_START) / 60.0
+            hist[minute_ago] = cumulative
+
+    def mock_minute_import_export(max_days_prev, now_utc, key, scale=1.0, required_unit=None, increment=True, smoothing=True, pad=True, _hist=hist):
+        """Mock historical PV data."""
+        return dict(_hist) if key == "pv_today" else {}
+
+    base.minute_data_import_export = mock_minute_import_export
+    solar.get_history_wrapper = lambda entity_id, days, required=False: []
+
+    # Build per-minute forecast arrays
+    total_minutes = 4 * 24 * 60
+    pv_m = {}
+    pv_m10 = {}
+    for m in range(total_minutes):
+        val = (FORECAST_KW / 60.0) if GEN_START <= m < GEN_END else 0.0
+        pv_m[m] = val
+        pv_m10[m] = val
+
+    # Build 15-min forecast data entries.
+    # Each entry covers FORECAST_PERIOD minutes and holds FORECAST_KW * (FORECAST_PERIOD/60) kWh.
+    midnight = datetime(2025, 6, 15, 0, 0, 0, tzinfo=pytz.utc)
+    pv_data = []
+    for slot in range(GEN_START, GEN_END, FORECAST_PERIOD):
+        ts = midnight + timedelta(minutes=slot)
+        kwh_per_entry = FORECAST_KW * FORECAST_PERIOD / 60.0
+        pv_data.append({"period_start": ts.strftime("%Y-%m-%dT%H:%M:%S+0000"), "pv_estimate": kwh_per_entry, "pv_estimate10": kwh_per_entry, "pv_estimate90": kwh_per_entry})
+
+    divide_by_factor = 1.0
+
+    pv_forecast_hist = {}
+    for day_num in range(1, days + 1):
+        for m_of_day in range(GEN_START, GEN_END):
+            minutes_ago = day_num * 1440 + (minutes_now - m_of_day)
+            pv_forecast_hist[minutes_ago] = float(FORECAST_KW)
+
+    with patch("solcast.history_attribute_to_minute_data", return_value=(pv_forecast_hist, days)):
+        adj_m, adj_m10, adj_data = solar.pv_calibration(pv_m, pv_m10, pv_data, create_pv10=True, divide_by=divide_by_factor, max_kwh=10.0, forecast_days=solar.forecast_days, period=FORECAST_PERIOD)
+
+    # Each 15-min entry should be annotated with the single 30-min plan slot that
+    # starts at the entry timestamp.  slots_per_period=max(1,round(15/30))=1, so
+    # the value should be one plan-slot's worth of calibrated kWh ≈ FORECAST_KW*30/60=1.0.
+    # (The plan slot is 30 min wide; each 15-min entry maps to one such slot.)
+    expected_kwh_per_slot = FORECAST_KW * PLAN_INTERVAL / 60.0  # 1.0 kWh
+
+    entries_validated = 0
+    for entry in adj_data:
+        cl = entry.get("pv_estimateCL")
+        e10 = entry.get("pv_estimate10")
+        e90 = entry.get("pv_estimate90")
+
+        if cl is None or cl == 0:
+            continue
+
+        entries_validated += 1
+
+        # pv_estimateCL must not be doubled (which would happen if slots_per_period were
+        # incorrectly set to 2 for a 15-min forecast with a 30-min plan interval).
+        double_expected = expected_kwh_per_slot * 2.0
+        if cl > double_expected * (1.0 + TOL):
+            print("ERROR: pv_estimateCL ({:.4f}) is larger than double the expected slot kWh ({:.4f}); slots may be over-accumulated".format(cl, expected_kwh_per_slot))
+            failed = True
+            break
+
+        if cl < expected_kwh_per_slot * (1.0 - TOL):
+            print("ERROR: pv_estimateCL ({:.4f}) is less than expected slot kWh ({:.4f})".format(cl, expected_kwh_per_slot))
+            failed = True
+            break
+
+        if e10 is not None and e10 > cl * 2.0:
+            print("ERROR: pv_estimate10 ({:.4f}) is unexpectedly much larger than pv_estimateCL ({:.4f})".format(e10, cl))
+            failed = True
+            break
+
+        if e90 is not None and e90 < cl * (1.0 - TOL):
+            print("ERROR: pv_estimate90 ({:.4f}) should be >= pv_estimateCL ({:.4f})".format(e90, cl))
+            failed = True
+            break
+
+    if entries_validated == 0:
+        print("ERROR: pv_calibration() annotated no entries with pv_estimateCL; annotation step may have regressed")
+        failed = True
+
+    test_api.cleanup()
+    return failed
+
+
 # ============================================================================
 # Main Test Runner
 # ============================================================================
@@ -2481,5 +2738,7 @@ def run_solcast_tests(my_predbat):
     failed |= test_pv_calibration_capped_data_clamp(my_predbat)
     failed |= test_pv_calibration_partial_history(my_predbat)
     failed |= test_pv_calibration_synthetic_values(my_predbat)
+    failed |= test_pv_calibration_60min_period(my_predbat)
+    failed |= test_pv_calibration_15min_period(my_predbat)
 
     return failed