From 4a46805a11ab2b75153531360f1dc0e2be6a501a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20A=2E=20Rodr=C3=ADguez?= Date: Wed, 18 Mar 2026 18:19:10 +0100 Subject: [PATCH] Add system test for KPI integration in simulator workflow - Add test__simulator__kpis_present_in_output asserting KPIs are present in instance[0].area after a full pipeline run, with CAPEX verified against the ESDL costInformation and system_lifetime exercised with a non-default value (25 years) - Add scripts/test_system_local.sh for running system tests with a locally built simulator-worker image via docker-compose.override.local.yml - Gitignore docker-compose.override.local.yml - Update docker-compose.yml to use simulator-worker 0.0.29beta2 (kpi-calculator integration) - Exclude KPIs from snapshot comparison in existing simulator tests to avoid snapshot drift; KPI correctness is covered by the dedicated test - Increase simulator test timeouts from 60s to 120s (100s to 160s for delete test) to account for KPI calculation time added after simulation --- .gitignore | 3 +- docker-compose.yml | 2 +- scripts/test_system_local.sh | 18 ++++ system_tests/src/test_workflows_steps.py | 106 ++++++++++++++++++++--- 4 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 scripts/test_system_local.sh diff --git a/.gitignore b/.gitignore index 264d1a1..4e4ea6b 100644 --- a/.gitignore +++ b/.gitignore @@ -219,4 +219,5 @@ fabric.properties unit_test_coverage/ .env.test -gurobi/ \ No newline at end of file +gurobi/ +docker-compose.override.local.yml diff --git a/docker-compose.yml b/docker-compose.yml index df64bb4..57fd724 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -215,7 +215,7 @@ services: - "./gurobi/gurobi.lic:/app/gurobi/gurobi.lic" omotes_simulator_worker: - image: ghcr.io/project-omotes/omotes-simulator-worker:0.0.28 + image: ghcr.io/project-omotes/omotes-simulator-worker:0.0.29beta2 restart: unless-stopped deploy: replicas: 2 diff --git a/scripts/test_system_local.sh b/scripts/test_system_local.sh new file mode 100644 index 0000000..999a86a --- /dev/null +++ b/scripts/test_system_local.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Run system tests with a locally built simulator-worker image. +# The image is built from ../../simulator-worker-OMOTES/simulator-worker +# via docker-compose.override.local.yml (gitignored). +. scripts/_select_docker_compose.sh + +export COMPOSE_PROJECT_NAME=omotes_system_tests + +ENV_FILE=".env.test" +DOCKER_COMPOSE_FILE="./docker-compose.yml -f docker-compose.override.local.yml -f system_tests/docker-compose.override.yml" + +cp .env.template ${ENV_FILE} +sed -i 's/LOG_LEVEL=[a-z]*/LOG_LEVEL=WARNING/gi' ${ENV_FILE} + +$DOCKER_COMPOSE --env-file ${ENV_FILE} -f $DOCKER_COMPOSE_FILE down -v +./scripts/setup.sh $ENV_FILE "./docker-compose.yml -f ./docker-compose.override.setup.yml" +$DOCKER_COMPOSE --env-file ${ENV_FILE} -f $DOCKER_COMPOSE_FILE build +$DOCKER_COMPOSE --env-file ${ENV_FILE} -f $DOCKER_COMPOSE_FILE up --abort-on-container-exit diff --git a/system_tests/src/test_workflows_steps.py b/system_tests/src/test_workflows_steps.py index 71a3276..d4c7955 100644 --- a/system_tests/src/test_workflows_steps.py +++ b/system_tests/src/test_workflows_steps.py @@ -37,6 +37,12 @@ port=int(os.environ.get("RABBITMQ_PORT", "5672")), ) +# KPI content is tested separately in test__simulator__kpis_present_in_output +# and changes with each kpi-calculator release, so exclude it from snapshot comparisons. +# Note: this path assumes xmltodict parses a single element (dict, not list). +# If multi-instance ESDLs are introduced, this path will need to be updated. +EXCLUDE_KPIS: set[str] = {"root['esdl:EnergySystem']['instance']['area']['KPIs']"} + SQL_CONFIG = { "host": os.environ.get("POSTGRES_HOST", "localhost"), "port": int(os.environ.get("POSTGRES_PORT", "6432")), @@ -205,13 +211,21 @@ def expect_a_result( f"The job did not finish as {expected_result}. Found {result.result_type}" ) - def compare_esdl(self, expected_esdl: str, result_esdl: str) -> None: + def compare_esdl( + self, + expected_esdl: str, + result_esdl: str, + exclude_paths: set[str] | None = None, + ) -> None: + """Compare two ESDL strings for equality after normalization. + + :param exclude_paths: Optional DeepDiff paths to ignore (e.g. EXCLUDE_KPIS). + """ expected = normalize_esdl(expected_esdl) result = normalize_esdl(result_esdl) - diff_msg = pformat(DeepDiff(expected, result)) - + diff = DeepDiff(expected, result, exclude_paths=exclude_paths) self.assertEqual( - expected, result, msg=f"Found the following differences:\n{diff_msg}" + {}, dict(diff), msg=f"Found the following differences:\n{pformat(diff)}" ) def test__grow_optimizer_default__happy_path(self) -> None: @@ -263,7 +277,7 @@ def test__simulator__happy_path(self) -> None: result_handler = OmotesJobHandler() esdl_file = retrieve_esdl_file("./test_esdl/input/simulator_tutorial.esdl") workflow_type = "simulator" - timeout_seconds = 60.0 + timeout_seconds = 120.0 params_dict = { "timestep": datetime.timedelta(hours=1), "start_time": datetime.datetime(2019, 1, 1, 0, 0, 0, tzinfo=datetime.UTC), @@ -282,7 +296,9 @@ def test__simulator__happy_path(self) -> None: expected_esdl = retrieve_esdl_file( "./test_esdl/output/test__simulator__happy_path.esdl" ) - self.compare_esdl(expected_esdl, result_handler.result.output_esdl) + self.compare_esdl( + expected_esdl, result_handler.result.output_esdl, exclude_paths=EXCLUDE_KPIS + ) # assert time series data created assert_influxdb_database_existence(result_handler.result.output_esdl, True) @@ -302,7 +318,7 @@ def test__simulator__multiple_ates_run(self) -> None: "./test_esdl/input/simulator_ates_short_run.esdl" ) workflow_type = "simulator" - timeout_seconds = 60.0 + timeout_seconds = 120.0 params_dict = { "timestep": datetime.timedelta(hours=1), "start_time": datetime.datetime(2019, 1, 1, 0, 0, 0, tzinfo=datetime.UTC), @@ -330,7 +346,9 @@ def test__simulator__multiple_ates_run(self) -> None: for result_handler in result_handlers: self.expect_a_result(result_handler, JobResult.SUCCEEDED) - self.compare_esdl(expected_esdl, result_handler.result.output_esdl) + self.compare_esdl( + expected_esdl, result_handler.result.output_esdl, exclude_paths=EXCLUDE_KPIS + ) def test__grow_optimizer_default__happy_path_1source(self) -> None: # Arrange @@ -411,7 +429,7 @@ def test__simulator__job_reference_is_set(self) -> None: result_handler = OmotesJobHandler() esdl_file = retrieve_esdl_file("./test_esdl/input/simulator_tutorial.esdl") workflow_type = "simulator" - timeout_seconds = 60.0 + timeout_seconds = 120.0 params_dict = { "timestep": datetime.timedelta(hours=1), "start_time": datetime.datetime(2019, 1, 1, 0, 0, 0), @@ -439,7 +457,7 @@ def test__simulator__check_if_progress_updates_are_received(self) -> None: result_handler = OmotesJobHandler() esdl_file = retrieve_esdl_file("./test_esdl/input/simulator_tutorial.esdl") workflow_type = "simulator" - timeout_seconds = 60.0 + timeout_seconds = 120.0 params_dict = { "timestep": datetime.timedelta(hours=1), "start_time": datetime.datetime(2019, 1, 1, 0, 0, 0), @@ -592,7 +610,7 @@ def test__simulator__delete_time_series_data_after_run(self) -> None: result_handler = OmotesJobHandler() esdl_file = retrieve_esdl_file("./test_esdl/input/simulator_tutorial.esdl") workflow_type = "simulator" - timeout_seconds = 100.0 + timeout_seconds = 160.0 params_dict = { "timestep": datetime.timedelta(hours=1), "start_time": datetime.datetime(2019, 1, 1, 0, 0, 0, tzinfo=datetime.UTC), @@ -702,3 +720,69 @@ def _watch_job(result_handler: OmotesJobHandler): self.assertTrue(str(high_priority_job.id) in ordered_job_result_ids) # check that high priority job result was not last (exact order may vary) self.assertNotEqual(str(high_priority_job.id), ordered_job_result_ids[-1]) + + def test__simulator__kpis_present_in_output(self) -> None: + """Test that KPIs are calculated and stored in the output ESDL. + + Uses simulator_ates_short_run.esdl which contains costInformation on assets, + allowing the kpi-calculator to produce non-trivial KPI results. + """ + # Arrange + result_handler = OmotesJobHandler() + esdl_file = retrieve_esdl_file( + "./test_esdl/input/simulator_ates_short_run.esdl" + ) + workflow_type = "simulator" + timeout_seconds = 120.0 + params_dict = { + "timestep": datetime.timedelta(hours=1), + "start_time": datetime.datetime(2019, 1, 1, 0, 0, 0, tzinfo=datetime.UTC), + "end_time": datetime.datetime(2019, 1, 1, 3, 0, 0, tzinfo=datetime.UTC), + "system_lifetime": 25.0, + } + + # Act + with omotes_client() as omotes_client_: + submit_a_job( + omotes_client_, esdl_file, workflow_type, params_dict, result_handler + ) + result_handler.wait_until_result(timeout_seconds) + + # Assert + self.expect_a_result(result_handler, JobResult.SUCCEEDED) + output_esh = esdl.esdl_handler.EnergySystemHandler() + output_esh.load_from_string(result_handler.result.output_esdl) + energy_system = output_esh.energy_system + + # KPIs are attached to instance[0].area, not energy_system directly + self.assertGreater( + len(energy_system.instance), 0, "Output ESDL must have at least one instance" + ) + main_area = energy_system.instance[0].area + self.assertIsNotNone(main_area, "instance[0] must have an area") + self.assertIsNotNone(main_area.KPIs, "KPIs should be present in the main area") + kpi_list = list(main_area.KPIs.kpi) + self.assertGreater(len(kpi_list), 0, "At least one KPI should be calculated") + for kpi in kpi_list: + self.assertNotEqual(kpi.name, "", "Each KPI should have a name") + + # CAPEX: the ATES asset in simulator_ates_short_run.esdl has + # investmentCosts=2333594.0 EUR (bare EUR unit, no conversion factor). + # This is the only asset with cost data so total CAPEX equals that value. + expected_capex = 2_333_594.0 + kpi_by_name = {kpi.name: kpi for kpi in kpi_list} + self.assertIn( + "High level cost breakdown [EUR]", + kpi_by_name, + "Cost breakdown KPI missing from output", + ) + cost_items = { + item.label: item.value + for item in kpi_by_name["High level cost breakdown [EUR]"].distribution.stringItem + } + self.assertAlmostEqual( + cost_items.get("CAPEX (total)", 0.0), + expected_capex, + places=1, + msg=f"CAPEX should match investmentCosts in simulator_ates_short_run.esdl; got {cost_items}", + )