diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index ae2615a49..ab2609506 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -879,12 +879,23 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period): }, ) - def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10, divide_by, max_kwh, forecast_days): + def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10, divide_by, max_kwh, forecast_days, period=None): """ Perform PV calibration based on historical data and forecast data. This will adjust the forecast data based on historical PV production and forecast data. It will also create pv_estimate10 and pv_estimate90 data if create_pv10 is True. """ + # If no period is given, default to the plan interval (backward-compatible for unit tests). + if period is None: + period = self.plan_interval_minutes + # Number of plan-interval slots that span one forecast entry period. + # For 30-min plan slots with a 60-min forecast (Open-Meteo) this is 2, + # for 30-min plan slots with a 30-min forecast (Solcast) this is 1. + # Use ceiling so partial forecast periods are fully covered rather than rounded down. + if period % self.plan_interval_minutes != 0: + self.log("Warn: PV calibration forecast period {} does not divide evenly into plan interval {} - using ceiling slot coverage".format(period, self.plan_interval_minutes)) + slots_per_period = max(1, int(math.ceil(period / self.plan_interval_minutes))) + self.log("PV Calibration: Fetching PV data for calibration") days = 7 @@ -1075,16 +1086,38 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_d if period_start: minutes_since_midnight = (datetime.strptime(period_start, TIME_FORMAT) - self.midnight_utc).total_seconds() / 60 slot = int(minutes_since_midnight / self.plan_interval_minutes) * self.plan_interval_minutes - calibrated = pv_estimateCL.get(slot, None) - calibrated10 = pv_estimate10.get(slot, None) - calibrated90 = pv_estimate90.get(slot, None) + + # Sum all plan-interval slots that fall within this forecast entry's period. + # When the forecast resolution is coarser than plan_interval_minutes (e.g. 60-min + # Open-Meteo entries with 30-min plan slots) we must accumulate multiple slots so + # that the annotated value covers the full entry duration, not just the first half. + calibrated = 0 + calibrated10 = 0 + calibrated90 = 0 + has_calibrated = False + has_calibrated10 = False + has_calibrated90 = False + for i in range(slots_per_period): + s = slot + i * self.plan_interval_minutes + v = pv_estimateCL.get(s, None) + if v is not None: + calibrated += v + has_calibrated = True + v10 = pv_estimate10.get(s, None) + if v10 is not None: + calibrated10 += v10 + has_calibrated10 = True + v90 = pv_estimate90.get(s, None) + if v90 is not None: + calibrated90 += v90 + has_calibrated90 = True # When we store the data we have to reverse the divide_by factor - if calibrated is not None: + if has_calibrated: entry["pv_estimateCL"] = calibrated * divide_by - if create_pv10 and (calibrated10 is not None): + if create_pv10 and has_calibrated10: entry["pv_estimate10"] = calibrated10 * divide_by - if create_pv10 and (calibrated90 is not None): + if create_pv10 and has_calibrated90: entry["pv_estimate90"] = calibrated90 * divide_by # Creation of PV10 data using worst day scaling factor @@ -1245,7 +1278,7 @@ async def fetch_pv_forecast(self): ) # Run calibration on the data - pv_forecast_minute, pv_forecast_minute10, pv_forecast_data = self.pv_calibration(pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10, divide_by / period, max_kwh, self.forecast_days) + pv_forecast_minute, pv_forecast_minute10, pv_forecast_data = self.pv_calibration(pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10, divide_by / period, max_kwh, self.forecast_days, period) self.publish_pv_stats(pv_forecast_data, divide_by / period, period) self.pack_and_store_forecast(pv_forecast_minute, pv_forecast_minute10) self.update_success_timestamp() diff --git a/apps/predbat/tests/test_solcast.py b/apps/predbat/tests/test_solcast.py index 519ade9f5..5dc85fb7e 100644 --- a/apps/predbat/tests/test_solcast.py +++ b/apps/predbat/tests/test_solcast.py @@ -2414,6 +2414,263 @@ def mock_minute_import_export(max_days_prev, now_utc, key, scale=1.0, required_u return failed +def test_pv_calibration_60min_period(my_predbat): + """ + Test that pv_calibration correctly annotates pv_forecast_data entries when the + forecast period (60 min, as used by Open-Meteo) is coarser than the plan interval + (30 min by default). + + Before the fix, each 60-min entry was annotated with only the first 30-min plan + slot's calibrated value, producing values that were approximately half the correct + amount. After the fix, both 30-min slots within the 60-min window are summed. + """ + print(" - test_pv_calibration_60min_period") + failed = False + + GEN_START = 480 # 8:00 UTC in minutes since midnight + GEN_END = 600 # 10:00 UTC (2 hours = 2 × 60-min entries; safely before noon) + FORECAST_KW = 2.0 # kW during generation window + PLAN_INTERVAL = 30 # minutes + FORECAST_PERIOD = 60 # minutes (Open-Meteo resolution) + TOL = 0.15 # 15% tolerance (matches other calibration tests; allows for minor slot-boundary effects) + + test_api = create_test_solar_api() + solar = test_api.solar + base = test_api.mock_base + base.plan_interval_minutes = PLAN_INTERVAL + + # Flat historical production matching the forecast → calibration ratio ~1.0 + # so the calibrated values should be very close to the raw forecast values. + hist = {} + days = 5 + minutes_now = base.minutes_now # 720 (noon) + for day_idx in range(days): + day = day_idx + 1 + midnight_ago = day * 1440 + minutes_now + for step in range(0, 24 * 60, 5): + minute_ago = midnight_ago - step + if minute_ago < 0: + continue + actual_min = step + if actual_min < GEN_START: + cumulative = 0.0 + elif actual_min < GEN_END: + cumulative = FORECAST_KW * (actual_min - GEN_START) / 60.0 + else: + cumulative = FORECAST_KW * (GEN_END - GEN_START) / 60.0 + hist[minute_ago] = cumulative + + def mock_minute_import_export(max_days_prev, now_utc, key, scale=1.0, required_unit=None, increment=True, smoothing=True, pad=True, _hist=hist): + """Mock historical PV data.""" + return dict(_hist) if key == "pv_today" else {} + + base.minute_data_import_export = mock_minute_import_export + solar.get_history_wrapper = lambda entity_id, days, required=False: [] + + # Build per-minute forecast arrays (as minute_data() would produce) + # Each minute in the gen window has FORECAST_KW / 60 kWh/min + total_minutes = 4 * 24 * 60 + pv_m = {} + pv_m10 = {} + for m in range(total_minutes): + val = (FORECAST_KW / 60.0) if GEN_START <= m < GEN_END else 0.0 + pv_m[m] = val + pv_m10[m] = val + + # Build 60-min forecast data entries (like Open-Meteo would produce). + # Each entry covers FORECAST_PERIOD minutes and holds FORECAST_KW * (FORECAST_PERIOD/60) kWh. + midnight = datetime(2025, 6, 15, 0, 0, 0, tzinfo=pytz.utc) + pv_data = [] + for slot in range(GEN_START, GEN_END, FORECAST_PERIOD): + ts = midnight + timedelta(minutes=slot) + kwh_per_entry = FORECAST_KW * FORECAST_PERIOD / 60.0 + pv_data.append({"period_start": ts.strftime("%Y-%m-%dT%H:%M:%S+0000"), "pv_estimate": kwh_per_entry, "pv_estimate10": kwh_per_entry, "pv_estimate90": kwh_per_entry}) + + # divide_by passed to pv_calibration = divide_by_full / period = factor. + # For kWh entries factor = 1.0. + divide_by_factor = 1.0 + + # Build historic forecast dict (per-minute power in kW, keyed by minutes-ago) + pv_forecast_hist = {} + for day_num in range(1, days + 1): + for m_of_day in range(GEN_START, GEN_END): + minutes_ago = day_num * 1440 + (minutes_now - m_of_day) + pv_forecast_hist[minutes_ago] = float(FORECAST_KW) + + with patch("solcast.history_attribute_to_minute_data", return_value=(pv_forecast_hist, days)): + adj_m, adj_m10, adj_data = solar.pv_calibration(pv_m, pv_m10, pv_data, create_pv10=True, divide_by=divide_by_factor, max_kwh=10.0, forecast_days=solar.forecast_days, period=FORECAST_PERIOD) + + # Each annotated entry should cover the full FORECAST_PERIOD minutes. + # Expected calibrated kWh per entry ≈ FORECAST_KW * FORECAST_PERIOD / 60 = 2.0 kWh. + # With the pre-fix bug, only the first 30-min plan slot was summed, giving ~1.0 kWh (half). + expected_kwh = FORECAST_KW * FORECAST_PERIOD / 60.0 + half_expected = expected_kwh / 2.0 + + entries_validated = 0 + for entry in adj_data: + cl = entry.get("pv_estimateCL") + e10 = entry.get("pv_estimate10") + e90 = entry.get("pv_estimate90") + + if cl is None or cl == 0: + continue + + entries_validated += 1 + + # The calibrated value must be close to the full 60-min kWh (not the buggy half-period value). + if abs(cl - expected_kwh) > TOL * expected_kwh: + print("ERROR: pv_estimateCL ({:.4f}) should be ~{:.4f} (full 60-min period kWh); half-period bug would give ~{:.4f}".format(cl, expected_kwh, half_expected)) + failed = True + break + + if e10 is not None and e10 > 0: + # e10 is worst-day scaling × CL; must be > 0 and plausibly related to CL + if e10 > cl * 2.0: + print("ERROR: pv_estimate10 ({:.4f}) is unexpectedly much larger than pv_estimateCL ({:.4f})".format(e10, cl)) + failed = True + break + + if e90 is not None and e90 < cl * (1.0 - TOL): + print("ERROR: pv_estimate90 ({:.4f}) should be >= pv_estimateCL ({:.4f})".format(e90, cl)) + failed = True + break + + if entries_validated == 0: + print("ERROR: pv_calibration() annotated no entries with pv_estimateCL; annotation step may have regressed") + failed = True + + test_api.cleanup() + return failed + + +def test_pv_calibration_15min_period(my_predbat): + """ + Test that pv_calibration correctly annotates pv_forecast_data entries when the + forecast period (15 min) is finer than the plan interval (30 min by default). + + Each 15-min entry covers half a 30-min plan slot, so slots_per_period=1 and only + the single plan slot that starts at the entry's timestamp is used. The annotated + pv_estimateCL should therefore be close to FORECAST_KW * 15/60 kWh (not double). + """ + print(" - test_pv_calibration_15min_period") + failed = False + + GEN_START = 480 # 8:00 UTC in minutes since midnight + GEN_END = 600 # 10:00 UTC (2 hours = 8 × 15-min entries) + FORECAST_KW = 2.0 # kW during generation window + PLAN_INTERVAL = 30 # minutes (production default) + FORECAST_PERIOD = 15 # minutes (forecast.solar / fine-resolution Solcast) + TOL = 0.15 # 15% tolerance + + test_api = create_test_solar_api() + solar = test_api.solar + base = test_api.mock_base + base.plan_interval_minutes = PLAN_INTERVAL + + # Flat historical production matching the forecast → calibration ratio ~1.0 + hist = {} + days = 5 + minutes_now = base.minutes_now # 720 (noon) + for day_idx in range(days): + day = day_idx + 1 + midnight_ago = day * 1440 + minutes_now + for step in range(0, 24 * 60, 5): + minute_ago = midnight_ago - step + if minute_ago < 0: + continue + actual_min = step + if actual_min < GEN_START: + cumulative = 0.0 + elif actual_min < GEN_END: + cumulative = FORECAST_KW * (actual_min - GEN_START) / 60.0 + else: + cumulative = FORECAST_KW * (GEN_END - GEN_START) / 60.0 + hist[minute_ago] = cumulative + + def mock_minute_import_export(max_days_prev, now_utc, key, scale=1.0, required_unit=None, increment=True, smoothing=True, pad=True, _hist=hist): + """Mock historical PV data.""" + return dict(_hist) if key == "pv_today" else {} + + base.minute_data_import_export = mock_minute_import_export + solar.get_history_wrapper = lambda entity_id, days, required=False: [] + + # Build per-minute forecast arrays + total_minutes = 4 * 24 * 60 + pv_m = {} + pv_m10 = {} + for m in range(total_minutes): + val = (FORECAST_KW / 60.0) if GEN_START <= m < GEN_END else 0.0 + pv_m[m] = val + pv_m10[m] = val + + # Build 15-min forecast data entries. + # Each entry covers FORECAST_PERIOD minutes and holds FORECAST_KW * (FORECAST_PERIOD/60) kWh. + midnight = datetime(2025, 6, 15, 0, 0, 0, tzinfo=pytz.utc) + pv_data = [] + for slot in range(GEN_START, GEN_END, FORECAST_PERIOD): + ts = midnight + timedelta(minutes=slot) + kwh_per_entry = FORECAST_KW * FORECAST_PERIOD / 60.0 + pv_data.append({"period_start": ts.strftime("%Y-%m-%dT%H:%M:%S+0000"), "pv_estimate": kwh_per_entry, "pv_estimate10": kwh_per_entry, "pv_estimate90": kwh_per_entry}) + + divide_by_factor = 1.0 + + pv_forecast_hist = {} + for day_num in range(1, days + 1): + for m_of_day in range(GEN_START, GEN_END): + minutes_ago = day_num * 1440 + (minutes_now - m_of_day) + pv_forecast_hist[minutes_ago] = float(FORECAST_KW) + + with patch("solcast.history_attribute_to_minute_data", return_value=(pv_forecast_hist, days)): + adj_m, adj_m10, adj_data = solar.pv_calibration(pv_m, pv_m10, pv_data, create_pv10=True, divide_by=divide_by_factor, max_kwh=10.0, forecast_days=solar.forecast_days, period=FORECAST_PERIOD) + + # Each 15-min entry should be annotated with the single 30-min plan slot that + # starts at the entry timestamp. slots_per_period=max(1,round(15/30))=1, so + # the value should be one plan-slot's worth of calibrated kWh ≈ FORECAST_KW*30/60=1.0. + # (The plan slot is 30 min wide; each 15-min entry maps to one such slot.) + expected_kwh_per_slot = FORECAST_KW * PLAN_INTERVAL / 60.0 # 1.0 kWh + + entries_validated = 0 + for entry in adj_data: + cl = entry.get("pv_estimateCL") + e10 = entry.get("pv_estimate10") + e90 = entry.get("pv_estimate90") + + if cl is None or cl == 0: + continue + + entries_validated += 1 + + # pv_estimateCL must not be doubled (which would happen if slots_per_period were + # incorrectly set to 2 for a 15-min forecast with a 30-min plan interval). + double_expected = expected_kwh_per_slot * 2.0 + if cl > double_expected * (1.0 + TOL): + print("ERROR: pv_estimateCL ({:.4f}) is larger than double the expected slot kWh ({:.4f}); slots may be over-accumulated".format(cl, expected_kwh_per_slot)) + failed = True + break + + if cl < expected_kwh_per_slot * (1.0 - TOL): + print("ERROR: pv_estimateCL ({:.4f}) is less than expected slot kWh ({:.4f})".format(cl, expected_kwh_per_slot)) + failed = True + break + + if e10 is not None and e10 > cl * 2.0: + print("ERROR: pv_estimate10 ({:.4f}) is unexpectedly much larger than pv_estimateCL ({:.4f})".format(e10, cl)) + failed = True + break + + if e90 is not None and e90 < cl * (1.0 - TOL): + print("ERROR: pv_estimate90 ({:.4f}) should be >= pv_estimateCL ({:.4f})".format(e90, cl)) + failed = True + break + + if entries_validated == 0: + print("ERROR: pv_calibration() annotated no entries with pv_estimateCL; annotation step may have regressed") + failed = True + + test_api.cleanup() + return failed + + # ============================================================================ # Main Test Runner # ============================================================================ @@ -2481,5 +2738,7 @@ def run_solcast_tests(my_predbat): failed |= test_pv_calibration_capped_data_clamp(my_predbat) failed |= test_pv_calibration_partial_history(my_predbat) failed |= test_pv_calibration_synthetic_values(my_predbat) + failed |= test_pv_calibration_60min_period(my_predbat) + failed |= test_pv_calibration_15min_period(my_predbat) return failed