From 8c65549720a437f1c9744e43c24e5b4115dc7964 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 15 Nov 2024 11:18:26 +0100 Subject: [PATCH 01/86] use new version of status2019 --- src/egon/data/datasets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets.yml b/src/egon/data/datasets.yml index 02e901686..373dd0424 100644 --- a/src/egon/data/datasets.yml +++ b/src/egon/data/datasets.yml @@ -1274,4 +1274,4 @@ home_batteries: scenario_path: sources: - url_status2019: 'https://zenodo.org/records/13865306/files/PoWerD_status2019_v2.backup' \ No newline at end of file + url_status2019: 'https://zenodo.org/records/14168340/files/PoWerD_status2019_v3.backup' \ No newline at end of file From 8ed73b8aaf32375be85d11938e3d0187516b1eec Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 24 Jan 2025 13:52:13 +0100 Subject: [PATCH 02/86] create functions to import scn and capacities --- .../datasets/scenario_path/scenario_path.py | 248 ++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 src/egon/data/datasets/scenario_path/scenario_path.py diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py new file mode 100644 index 000000000..c97481421 --- /dev/null +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -0,0 +1,248 @@ +from pathlib import Path +from urllib.request import urlretrieve +import os +import subprocess + +import numpy as np +import pandas as pd + +from egon.data import config, db +import egon.data.config + +sources = egon.data.config.datasets()["scenario_path"]["sources"] + +scn_path = ["powerd2025", "powerd2030", "powerd2035"] + +con = db.engine() + + +# load scenarios +def clean_existing_scn_path_data(scn_path): + # Clean existing data from previous executions + tables = pd.read_sql( + """ + SELECT tablename FROM pg_catalog.pg_tables + WHERE schemaname = 'grid' + """, + con, + ) + + tables = tables[ + ~tables["tablename"].isin( + [ + "egon_etrago_carrier", + "egon_etrago_temp_resolution", + "egon_etrago_ac_h2", + "egon_etrago_hv_busmap", + ] + ) + ] + + for scn_path_name in scn_path: + for table in tables["tablename"]: + db.execute_sql( + f""" + DELETE FROM grid.{table} WHERE scn_name = '{scn_path_name}'; + """ + ) + return + + +def load_scn_no_time_no_foreign(scn_name): + # load scenario data without timeseries and foreign countries data + + scn_tables = {} + for t in ["egon_etrago_bus"]: + scn_tables[t] = pd.read_sql( + f""" + SELECT * FROM grid.{t} + WHERE scn_name = '{scn_name}' + AND country = 'DE' + """, + con, + ) + + for t in [ + "egon_etrago_generator", + "egon_etrago_load", + "egon_etrago_storage", + "egon_etrago_store", + ]: + scn_tables[t] = pd.read_sql( + f""" + SELECT * FROM grid.{t} + WHERE scn_name = '{scn_name}' + AND bus IN (SELECT bus_id from grid.egon_etrago_bus + WHERE scn_name = '{scn_name}' + AND country = 'DE') + """, + con, + ) + + for t in [ + "egon_etrago_line", + "egon_etrago_link", + "egon_etrago_transformer", + ]: + scn_tables[t] = pd.read_sql( + f""" + SELECT * FROM grid.{t} + WHERE scn_name = '{scn_name}' + AND bus0 IN (SELECT bus_id from grid.egon_etrago_bus + WHERE scn_name = '{scn_name}' + AND country = 'DE') + AND bus1 IN (SELECT bus_id from grid.egon_etrago_bus + WHERE scn_name = '{scn_name}' + AND country = 'DE') + """, + con, + ) + + return scn_tables + + +def load_scn_capacies(scn1: dict, scn2: dict): + scn1 = status2019.copy() + scn2 = eGon100RE.copy() + + scn_capacities = pd.read_sql( + """ + SELECT * FROM supply.egon_scenario_capacities + """, + con, + index_col="index", + ) + + map_carrier = { + "urban_central_solar_thermal_collector": "solar_thermal_collector", + "urban_central_geo_thermal": "geo_thermal", + "urban_central_gas_boiler": "central_gas_boiler", + "urban_central_heat_pump": "central_heat_pump", + "urban_central_resistive_heater": "central_resistive_heater", + } + + scn_capacities["carrier"] = scn_capacities["carrier"].apply( + lambda x: map_carrier[x] if x in map_carrier.keys() else x + ) + + carriers_gen_from_supply = [ + "oil", + "solar", + "solar_rooftop", + "wind_onshore", + "lignite", + "coal", + "wind_offshore", + "solar_thermal_collector", + "geo_thermal", + "run_of_river", + "rural_solar_thermal", + "urban_central_gas_CHP", + "urban_central_solid_biomass_CHP", + ] + + carriers_gen = set( + carriers_gen_from_supply + + list(scn1["egon_etrago_generator"]["carrier"]) + + list(scn2["egon_etrago_generator"]["carrier"]) + ) + + all_scn = ["status2019"] + scn_path + ["eGon100RE"] + gen_capacities = pd.DataFrame(index=list(carriers_gen), columns=all_scn) + gen_capacities["status2019"] = ( + scn1["egon_etrago_generator"].groupby("carrier").p_nom.sum() + ) + gen_capacities["eGon100RE"] = ( + scn2["egon_etrago_generator"].groupby("carrier").p_nom.sum() + ) + + for scn in scn_path: + cap = scn_capacities[scn_capacities["scenario_name"] == scn] + cap = cap.set_index("carrier") + gen_capacities[scn] = cap["capacity"] + + carriers_links_from_supply = [ + "central_gas_boiler", + "central_heat_pump", + "central_resistive_heater", + "gas", + "rural_biomass_boiler", + "rural_gas_boiler", + "rural_heat_pump", + "rural_oil_boiler", + "rural_resistive_heater", + ] + + carriers_links = set( + carriers_links_from_supply + + list(scn1["egon_etrago_link"]["carrier"]) + + list(scn2["egon_etrago_link"]["carrier"]) + ) + + link_capacities = pd.DataFrame(index=list(carriers_links), columns=all_scn) + link_capacities["status2019"] = ( + scn1["egon_etrago_link"].groupby("carrier").p_nom.sum() + ) + link_capacities["eGon100RE"] = ( + scn2["egon_etrago_link"].groupby("carrier").p_nom.sum() + ) + + for scn in scn_path: + cap = scn_capacities[scn_capacities["scenario_name"] == scn] + cap = cap.set_index("carrier") + link_capacities[scn] = cap["capacity"] + + carriers_storage_from_supply = ["pumped_hydro"] + + carriers_storage = set( + carriers_storage_from_supply + + list(scn1["egon_etrago_storage"]["carrier"]) + + list(scn2["egon_etrago_storage"]["carrier"]) + ) + + storage_capacities = pd.DataFrame( + index=list(carriers_storage), columns=all_scn + ) + storage_capacities["status2019"] = ( + scn1["egon_etrago_storage"].groupby("carrier").p_nom.sum() + ) + storage_capacities["eGon100RE"] = ( + scn2["egon_etrago_storage"].groupby("carrier").p_nom.sum() + ) + + for scn in scn_path: + cap = scn_capacities[scn_capacities["scenario_name"] == scn] + cap = cap.set_index("carrier") + storage_capacities[scn] = cap["capacity"] + + # Test that all carriers from supply.scenario_capacities are included + used_carriers = ( + list(carriers_gen_from_supply) + + list(carriers_links_from_supply) + + list(carriers_storage_from_supply) + ) + + assert set(used_carriers) == set(scn_capacities.carrier.unique()) + + return gen_capacities, link_capacities, storage_capacities + + +def adjust_generators(scn1: dict, scn2: dict, new_scn: dict, name: str): + scn1 = status2019.copy() + scn2 = eGon100RE.copy() + name = "powerd2025" + + gen = new_scn["egon_etrago_generator"] + gen_scn1 = scn1["egon_etrago_generator"] + gen_scn1 = gen_scn1.set_index(["bus", "carrier"]) + + gen_scn2 = scn2["egon_etrago_generator"] + gen_scn2 = gen_scn2.set_index(["bus", "carrier"]) + + diff = gen_scn2["p_nom"] - gen_scn1["p_nom"] + + +eGon100RE = load_scn_no_time_no_foreign(scn_name="eGon100RE") +status2019 = load_scn_no_time_no_foreign(scn_name="status2019") + +cap_gen, cap_link, cap_storage = load_scn_capacies(status2019, eGon100RE) From e969fee42d6513415b7aa2772eebd5f2447b1545 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 3 Mar 2025 09:58:55 +0100 Subject: [PATCH 03/86] give structure to scenario_path --- .../data/datasets/scenario_path/__init__.py | 55 +++++++++++++++++-- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/__init__.py b/src/egon/data/datasets/scenario_path/__init__.py index ddce04656..e41048b2c 100644 --- a/src/egon/data/datasets/scenario_path/__init__.py +++ b/src/egon/data/datasets/scenario_path/__init__.py @@ -1,10 +1,55 @@ from egon.data.datasets import Dataset from egon.data.datasets.scenario_path.import_status2019 import ( - download_status2019, - import_scn_status2019, + clean_existing_scn_path_data, + import_network_structure, + import_generators, + import_loads, + import_links, + import_storage_units, + import_stores, + import_foreign, ) +def create_powerd2025(): + scn = "powerd2025" + import_network_structure(scn) + import_generators(scn) + import_loads(scn) + import_links(scn) + import_storage_units(scn) + import_stores(scn) + import_foreign(scn) + + return + + +def create_powerd2030(): + scn = "powerd2030" + import_network_structure(scn) + import_generators(scn) + import_loads(scn) + import_links(scn) + import_storage_units(scn) + import_stores(scn) + import_foreign(scn) + + return + + +def create_powerd2035(): + scn = "powerd2035" + import_network_structure(scn) + import_generators(scn) + import_loads(scn) + import_links(scn) + import_storage_units(scn) + import_stores(scn) + import_foreign(scn) + + return + + class CreateIntermediateScenarios(Dataset): def __init__(self, dependencies): super().__init__( @@ -12,7 +57,9 @@ def __init__(self, dependencies): version="0.0.1", dependencies=dependencies, tasks=( - download_status2019, - import_scn_status2019, + clean_existing_scn_path_data, + create_powerd2025, + create_powerd2030, + create_powerd2035, ), ) From 7296de9975cfe6d38a0e5ce50cca736bb24fc54f Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 3 Mar 2025 11:11:54 +0100 Subject: [PATCH 04/86] move list of scn names --- src/egon/data/datasets/scenario_path/scenario_path.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index c97481421..a083ddfd1 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -11,13 +11,11 @@ sources = egon.data.config.datasets()["scenario_path"]["sources"] -scn_path = ["powerd2025", "powerd2030", "powerd2035"] - con = db.engine() -# load scenarios -def clean_existing_scn_path_data(scn_path): +def clean_existing_scn_path_data(): + scn_path = ["powerd2025", "powerd2030", "powerd2035"] # Clean existing data from previous executions tables = pd.read_sql( """ @@ -48,6 +46,7 @@ def clean_existing_scn_path_data(scn_path): return +# load scenarios def load_scn_no_time_no_foreign(scn_name): # load scenario data without timeseries and foreign countries data @@ -221,9 +220,9 @@ def load_scn_capacies(scn1: dict, scn2: dict): + list(carriers_links_from_supply) + list(carriers_storage_from_supply) ) - + assert set(used_carriers) == set(scn_capacities.carrier.unique()) - + return gen_capacities, link_capacities, storage_capacities From 6d40f5d3b7f408c266ec5bd54015d8ec697be48e Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 3 Mar 2025 13:19:24 +0100 Subject: [PATCH 05/86] create import_network_structure --- .../datasets/scenario_path/scenario_path.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index a083ddfd1..6c9d4528b 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -46,6 +46,69 @@ def clean_existing_scn_path_data(): return +def import_network_structure(scn=str): + scn = "powerd2025" + + # Import buses + bus = pd.read_sql( + sql=""" + SELECT * from grid.egon_etrago_bus + WHERE scn_name = 'eGon100RE' + """, + con=con, + ) + + bus["scn_name"] = scn + + bus.to_sql( + name="egon_etrago_bus", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + # Import lines + line = pd.read_sql( + sql=""" + SELECT * from grid.egon_etrago_line + WHERE scn_name = 'eGon100RE' + """, + con=con, + ) + + line["scn_name"] = scn + + line.to_sql( + name="egon_etrago_line", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + # Import transformers + transformer = pd.read_sql( + sql=""" + SELECT * from grid.egon_etrago_transformer + WHERE scn_name = 'eGon100RE' + """, + con=con, + ) + + transformer["scn_name"] = scn + + transformer.to_sql( + name="egon_etrago_transformer", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + return + + # load scenarios def load_scn_no_time_no_foreign(scn_name): # load scenario data without timeseries and foreign countries data From 5b6b092eb8b924a02dba8ab5be71f9993538c214 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 4 Mar 2025 11:16:40 +0100 Subject: [PATCH 06/86] create load_scn_capacies_link --- .../datasets/scenario_path/scenario_path.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 6c9d4528b..3d63c2ef6 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -109,6 +109,103 @@ def import_network_structure(scn=str): return +def load_scn_capacies_link( + scn1="status2019", + scn2="eGon100RE", + scn_path=["powerd2025", "powerd2030", "powerd2035"], +): + + scn1_link = pd.read_sql( + f""" + SELECT * FROM grid.egon_etrago_link + WHERE scn_name = '{scn1}' + AND bus0 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = '{scn1}' + ) + AND bus1 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = '{scn1}' + ) + """, + con, + ) + + scn2_link = pd.read_sql( + f""" + SELECT * FROM grid.egon_etrago_link + WHERE scn_name = '{scn2}' + AND bus0 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = '{scn2}' + ) + AND bus1 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = '{scn2}' + ) + """, + con, + ) + + scn_capacities = pd.read_sql( + """ + SELECT * FROM supply.egon_scenario_capacities + """, + con, + index_col="index", + ) + + map_carrier = { + "urban_central_solar_thermal_collector": "solar_thermal_collector", + "urban_central_geo_thermal": "geo_thermal", + "urban_central_gas_boiler": "central_gas_boiler", + "urban_central_heat_pump": "central_heat_pump", + "urban_central_resistive_heater": "central_resistive_heater", + "gas": "OCGT", + } + + scn_capacities["carrier"] = scn_capacities["carrier"].apply( + lambda x: map_carrier[x] if x in map_carrier.keys() else x + ) + + carriers_links_from_supply = [ + "central_gas_boiler", + "central_heat_pump", + "central_resistive_heater", + "gas", + "rural_biomass_boiler", + "rural_gas_boiler", + "rural_heat_pump", + "rural_oil_boiler", + "rural_resistive_heater", + ] + + carriers_links = set( + carriers_links_from_supply + + list(scn1_link["carrier"]) + + list(scn2_link["carrier"]) + ) + + all_scn = [scn1] + scn_path + [scn2] + link_capacities = pd.DataFrame(index=list(carriers_links), columns=all_scn) + link_capacities[scn1] = scn1_link.groupby("carrier").p_nom.sum() + link_capacities[scn2] = scn2_link.groupby("carrier").p_nom.sum() + + for scn in scn_path: + cap = scn_capacities[scn_capacities["scenario_name"] == scn] + cap = cap.set_index("carrier") + link_capacities[scn] = cap["capacity"] + + return link_capacities + + + + + # load scenarios def load_scn_no_time_no_foreign(scn_name): # load scenario data without timeseries and foreign countries data From 48dad01686ba69df3da9a9230df91f9c14a348f0 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 4 Mar 2025 11:17:39 +0100 Subject: [PATCH 07/86] create load_scn_capacies_gen --- .../datasets/scenario_path/scenario_path.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 3d63c2ef6..16884bfdf 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -203,7 +203,90 @@ def load_scn_capacies_link( return link_capacities +def load_scn_capacies_gen( + scn1="status2019", + scn2="eGon100RE", + scn_path=["powerd2025", "powerd2030", "powerd2035"], +): + + scn1_gen = pd.read_sql( + f""" + SELECT * FROM grid.egon_etrago_generator + WHERE scn_name = '{scn1}' + AND bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = '{scn1}') + """, + con, + ) + + scn2_gen = pd.read_sql( + f""" + SELECT * FROM grid.egon_etrago_generator + WHERE scn_name = '{scn2}' + AND bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = '{scn2}') + """, + con, + ) + + scn_capacities = pd.read_sql( + """ + SELECT * FROM supply.egon_scenario_capacities + """, + con, + index_col="index", + ) + + map_carrier = { + "urban_central_solar_thermal_collector": "solar_thermal_collector", + "urban_central_geo_thermal": "geo_thermal", + "urban_central_gas_boiler": "central_gas_boiler", + "urban_central_heat_pump": "central_heat_pump", + "urban_central_resistive_heater": "central_resistive_heater", + "gas": "OCGT", + } + + scn_capacities["carrier"] = scn_capacities["carrier"].apply( + lambda x: map_carrier[x] if x in map_carrier.keys() else x + ) + + carriers_gen_from_supply = [ + "oil", + "solar", + "solar_rooftop", + "wind_onshore", + "lignite", + "coal", + "wind_offshore", + "solar_thermal_collector", + "geo_thermal", + "run_of_river", + "rural_solar_thermal", + "urban_central_gas_CHP", + "urban_central_solid_biomass_CHP", + ] + + carriers_gen = set( + carriers_gen_from_supply + + list(scn1_gen["carrier"]) + + list(scn2_gen["carrier"]) + ) + + all_scn = [scn1] + scn_path + [scn2] + gen_capacities = pd.DataFrame(index=list(carriers_gen), columns=all_scn) + gen_capacities[scn1] = scn1_gen.groupby("carrier").p_nom.sum() + gen_capacities[scn2] = scn2_gen.groupby("carrier").p_nom.sum() + + for scn in scn_path: + cap = scn_capacities[scn_capacities["scenario_name"] == scn] + cap = cap.set_index("carrier") + gen_capacities[scn] = cap["capacity"] + return gen_capacities # load scenarios From 68d3e5bfb952aa77eaa2498d3a0d55f96ae78445 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 4 Mar 2025 11:21:00 +0100 Subject: [PATCH 08/86] black --- .../datasets/scenario_path/scenario_path.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 16884bfdf..4d5126ba4 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -19,9 +19,9 @@ def clean_existing_scn_path_data(): # Clean existing data from previous executions tables = pd.read_sql( """ - SELECT tablename FROM pg_catalog.pg_tables - WHERE schemaname = 'grid' - """, + SELECT tablename FROM pg_catalog.pg_tables + WHERE schemaname = 'grid' + """, con, ) @@ -46,15 +46,15 @@ def clean_existing_scn_path_data(): return -def import_network_structure(scn=str): +def import_network_structure(scn="powerd2025"): scn = "powerd2025" # Import buses bus = pd.read_sql( sql=""" - SELECT * from grid.egon_etrago_bus - WHERE scn_name = 'eGon100RE' - """, + SELECT * from grid.egon_etrago_bus + WHERE scn_name = 'eGon100RE' + """, con=con, ) @@ -71,9 +71,9 @@ def import_network_structure(scn=str): # Import lines line = pd.read_sql( sql=""" - SELECT * from grid.egon_etrago_line - WHERE scn_name = 'eGon100RE' - """, + SELECT * from grid.egon_etrago_line + WHERE scn_name = 'eGon100RE' + """, con=con, ) @@ -90,9 +90,9 @@ def import_network_structure(scn=str): # Import transformers transformer = pd.read_sql( sql=""" - SELECT * from grid.egon_etrago_transformer - WHERE scn_name = 'eGon100RE' - """, + SELECT * from grid.egon_etrago_transformer + WHERE scn_name = 'eGon100RE' + """, con=con, ) @@ -131,7 +131,7 @@ def load_scn_capacies_link( ) """, con, - ) + ) scn2_link = pd.read_sql( f""" @@ -149,7 +149,7 @@ def load_scn_capacies_link( ) """, con, - ) + ) scn_capacities = pd.read_sql( """ From 205473ed96f9d289523f431b6e053f60a758686f Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 4 Mar 2025 11:24:17 +0100 Subject: [PATCH 09/86] remove not used functions --- .../datasets/scenario_path/scenario_path.py | 149 +----------------- 1 file changed, 1 insertion(+), 148 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 4d5126ba4..7bfb41002 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -340,151 +340,4 @@ def load_scn_no_time_no_foreign(scn_name): con, ) - return scn_tables - - -def load_scn_capacies(scn1: dict, scn2: dict): - scn1 = status2019.copy() - scn2 = eGon100RE.copy() - - scn_capacities = pd.read_sql( - """ - SELECT * FROM supply.egon_scenario_capacities - """, - con, - index_col="index", - ) - - map_carrier = { - "urban_central_solar_thermal_collector": "solar_thermal_collector", - "urban_central_geo_thermal": "geo_thermal", - "urban_central_gas_boiler": "central_gas_boiler", - "urban_central_heat_pump": "central_heat_pump", - "urban_central_resistive_heater": "central_resistive_heater", - } - - scn_capacities["carrier"] = scn_capacities["carrier"].apply( - lambda x: map_carrier[x] if x in map_carrier.keys() else x - ) - - carriers_gen_from_supply = [ - "oil", - "solar", - "solar_rooftop", - "wind_onshore", - "lignite", - "coal", - "wind_offshore", - "solar_thermal_collector", - "geo_thermal", - "run_of_river", - "rural_solar_thermal", - "urban_central_gas_CHP", - "urban_central_solid_biomass_CHP", - ] - - carriers_gen = set( - carriers_gen_from_supply - + list(scn1["egon_etrago_generator"]["carrier"]) - + list(scn2["egon_etrago_generator"]["carrier"]) - ) - - all_scn = ["status2019"] + scn_path + ["eGon100RE"] - gen_capacities = pd.DataFrame(index=list(carriers_gen), columns=all_scn) - gen_capacities["status2019"] = ( - scn1["egon_etrago_generator"].groupby("carrier").p_nom.sum() - ) - gen_capacities["eGon100RE"] = ( - scn2["egon_etrago_generator"].groupby("carrier").p_nom.sum() - ) - - for scn in scn_path: - cap = scn_capacities[scn_capacities["scenario_name"] == scn] - cap = cap.set_index("carrier") - gen_capacities[scn] = cap["capacity"] - - carriers_links_from_supply = [ - "central_gas_boiler", - "central_heat_pump", - "central_resistive_heater", - "gas", - "rural_biomass_boiler", - "rural_gas_boiler", - "rural_heat_pump", - "rural_oil_boiler", - "rural_resistive_heater", - ] - - carriers_links = set( - carriers_links_from_supply - + list(scn1["egon_etrago_link"]["carrier"]) - + list(scn2["egon_etrago_link"]["carrier"]) - ) - - link_capacities = pd.DataFrame(index=list(carriers_links), columns=all_scn) - link_capacities["status2019"] = ( - scn1["egon_etrago_link"].groupby("carrier").p_nom.sum() - ) - link_capacities["eGon100RE"] = ( - scn2["egon_etrago_link"].groupby("carrier").p_nom.sum() - ) - - for scn in scn_path: - cap = scn_capacities[scn_capacities["scenario_name"] == scn] - cap = cap.set_index("carrier") - link_capacities[scn] = cap["capacity"] - - carriers_storage_from_supply = ["pumped_hydro"] - - carriers_storage = set( - carriers_storage_from_supply - + list(scn1["egon_etrago_storage"]["carrier"]) - + list(scn2["egon_etrago_storage"]["carrier"]) - ) - - storage_capacities = pd.DataFrame( - index=list(carriers_storage), columns=all_scn - ) - storage_capacities["status2019"] = ( - scn1["egon_etrago_storage"].groupby("carrier").p_nom.sum() - ) - storage_capacities["eGon100RE"] = ( - scn2["egon_etrago_storage"].groupby("carrier").p_nom.sum() - ) - - for scn in scn_path: - cap = scn_capacities[scn_capacities["scenario_name"] == scn] - cap = cap.set_index("carrier") - storage_capacities[scn] = cap["capacity"] - - # Test that all carriers from supply.scenario_capacities are included - used_carriers = ( - list(carriers_gen_from_supply) - + list(carriers_links_from_supply) - + list(carriers_storage_from_supply) - ) - - assert set(used_carriers) == set(scn_capacities.carrier.unique()) - - return gen_capacities, link_capacities, storage_capacities - - -def adjust_generators(scn1: dict, scn2: dict, new_scn: dict, name: str): - scn1 = status2019.copy() - scn2 = eGon100RE.copy() - name = "powerd2025" - - gen = new_scn["egon_etrago_generator"] - gen_scn1 = scn1["egon_etrago_generator"] - gen_scn1 = gen_scn1.set_index(["bus", "carrier"]) - - gen_scn2 = scn2["egon_etrago_generator"] - gen_scn2 = gen_scn2.set_index(["bus", "carrier"]) - - diff = gen_scn2["p_nom"] - gen_scn1["p_nom"] - - -eGon100RE = load_scn_no_time_no_foreign(scn_name="eGon100RE") -status2019 = load_scn_no_time_no_foreign(scn_name="status2019") - -cap_gen, cap_link, cap_storage = load_scn_capacies(status2019, eGon100RE) + return scn_tables \ No newline at end of file From 88d8322dcc8cf8e794358edc08a9ac00fe04ded9 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 4 Mar 2025 13:09:50 +0100 Subject: [PATCH 10/86] fix import --- src/egon/data/datasets/scenario_path/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/scenario_path/__init__.py b/src/egon/data/datasets/scenario_path/__init__.py index e41048b2c..8724a6e14 100644 --- a/src/egon/data/datasets/scenario_path/__init__.py +++ b/src/egon/data/datasets/scenario_path/__init__.py @@ -1,5 +1,5 @@ from egon.data.datasets import Dataset -from egon.data.datasets.scenario_path.import_status2019 import ( +from egon.data.datasets.scenario_path.scenario_path import ( clean_existing_scn_path_data, import_network_structure, import_generators, From 50509469d67abf3a45cec6eb4568dde3aa552056 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Thu, 6 Mar 2025 09:31:36 +0100 Subject: [PATCH 11/86] implement scaling_factor for all scenarios --- src/egon/data/datasets/scenario_path/scenario_path.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 7bfb41002..1007bc169 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -13,6 +13,12 @@ con = db.engine() +scaling_factor = { + "powerd2025": (2025 - 2019) / (2040 - 2019), + "powerd2030": (2030 - 2019) / (2040 - 2019), + "powerd2035": (2035 - 2019) / (2040 - 2019), +} + def clean_existing_scn_path_data(): scn_path = ["powerd2025", "powerd2030", "powerd2035"] From 1331a48e16f87e1f10d2f7354b7c04a43e9c2da2 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Thu, 6 Mar 2025 09:33:43 +0100 Subject: [PATCH 12/86] deal with dsm and rural_heat_pump --- .../datasets/scenario_path/scenario_path.py | 155 +++++++++++++++++- 1 file changed, 154 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 1007bc169..27fbc4379 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -209,6 +209,159 @@ def load_scn_capacies_link( return link_capacities +def import_links(scn="powerd2025"): + scn = "powerd2025" + cap_link = load_scn_capacies_link() + + scn1_link = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_link + WHERE scn_name = 'status2019' + AND bus0 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'status2019' + ) + AND bus1 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'status2019' + ) + """, + con, + ) + + scn2_link = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_link + WHERE scn_name = 'eGon100RE' + AND bus0 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + AND bus1 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + """, + con, + ) + + # Dealing with dsm + dsm1 = scn1_link[scn1_link["carrier"] == "dsm"].set_index("bus0").copy() + dsm2 = scn2_link[scn2_link["carrier"] == "dsm"].set_index("bus0").copy() + dsm3 = dsm2.copy() + dsm3["scn_name"] = scn + dsm3["p_nom"] = ( + dsm1["p_nom"] + (dsm2["p_nom"] - dsm1["p_nom"]) * scaling_factor[scn] + ) + dsm3.reset_index(inplace=True) + dsm3.to_sql( + name="egon_etrago_link", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + dsm3_t = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_link_timeseries + WHERE link_id IN( + SELECT link_id FROM grid.egon_etrago_link + WHERE bus0 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + AND bus1 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + AND carrier = 'dsm') + """, + con, + ) + + dsm3_t["scn_name"] = scn + + dsm3_t.to_sql( + name="egon_etrago_link_timeseries", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + # dealing with rural_heat_pump + link_rhp1 = ( + scn1_link[scn1_link["carrier"].isin(["rural_heat_pump"])] + .copy() + .set_index("bus0") + ) + link_rhp2 = ( + scn2_link[scn2_link["carrier"].isin(["rural_heat_pump"])] + .copy() + .set_index("bus0") + ) + link_rhp3 = link_rhp2.copy() + link_rhp3["scn_name"] = scn + + link_rhp3["p_nom"] = ( + link_rhp1["p_nom"] + + (link_rhp2["p_nom"] - link_rhp1["p_nom"]) * scaling_factor[scn] + ) + factor_to_pypsaeur = ( + cap_link.at["rural_heat_pump", scn] / link_rhp3["p_nom"].sum() + ) + link_rhp3["p_nom"] = link_rhp3["p_nom"] * factor_to_pypsaeur + + link_rhp3.reset_index(inplace=True) + link_rhp3.to_sql( + name="egon_etrago_link", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + link_rhp3_t = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_link_timeseries + WHERE link_id IN( + SELECT link_id FROM grid.egon_etrago_link + WHERE bus0 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + AND bus1 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + AND carrier = 'rural_heat_pump') + """, + con, + ) + + link_rhp3_t["scn_name"] = scn + + link_rhp3_t.to_sql( + name="egon_etrago_link_timeseries", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + + return + + def load_scn_capacies_gen( scn1="status2019", scn2="eGon100RE", @@ -346,4 +499,4 @@ def load_scn_no_time_no_foreign(scn_name): con, ) - return scn_tables \ No newline at end of file + return scn_tables From badff2419cb341349e9c88122348cb360169cfdc Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Thu, 6 Mar 2025 11:26:35 +0100 Subject: [PATCH 13/86] dealing with central_gas_boiler --- .../datasets/scenario_path/scenario_path.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 27fbc4379..70229b219 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -5,6 +5,7 @@ import numpy as np import pandas as pd +import geopandas as gpd from egon.data import config, db import egon.data.config @@ -358,10 +359,78 @@ def import_links(scn="powerd2025"): index=False, ) + # Dealing with central_gas_boiler + link_cgb1 = ( + scn1_link[scn1_link["carrier"].isin(["central_gas_boiler"])] + .copy() + .set_index("bus1") + ) + link_cgb2 = ( + scn2_link[scn2_link["carrier"].isin(["central_gas_boiler"])] + .copy() + .set_index("bus1") + ) + link_cgb3 = link_cgb2.copy() + link_cgb3["scn_name"] = scn + + cgb1_geo = gpd.read_postgis( + """ + SELECT bus_id, geom FROM grid.egon_etrago_bus + WHERE scn_name = 'status2019' + AND carrier = 'central_heat' + """, + con, + geom_col="geom", + ).set_index("bus_id") + + cgb2_geo = gpd.read_postgis( + """ + SELECT bus_id, geom FROM grid.egon_etrago_bus + WHERE scn_name = 'eGon100RE' + AND carrier = 'central_heat' + """, + con, + geom_col="geom", + ).set_index("bus_id") + cgb1_to_cgb2 = {} + for g in cgb1_geo.index: + dist = cgb2_geo.distance(cgb1_geo["geom"][g]) + dist.sort_values(inplace=True) + for d in dist.index: + if d not in cgb1_to_cgb2.values(): + cgb1_to_cgb2[g] = d + break + + link_cgb1.index = link_cgb1.index.map(cgb1_to_cgb2) + missing_cgb = pd.DataFrame( + 0, + index=link_cgb2.index[~link_cgb2.index.isin(link_cgb1.index)], + columns=["p_nom"], + ) + link_cgb1 = pd.concat([link_cgb1, missing_cgb]) + + link_cgb3["p_nom"] = ( + link_cgb1["p_nom"] + + (link_cgb2["p_nom"] - link_cgb1["p_nom"]) * scaling_factor[scn] + ) + factor_to_pypsaeur = ( + cap_link.at["central_gas_boiler", scn] / link_cgb3["p_nom"].sum() + ) + link_cgb3["p_nom"] = link_cgb3["p_nom"] * factor_to_pypsaeur + + link_cgb3.reset_index(inplace=True) + link_cgb3.to_sql( + name="egon_etrago_link", + con=con, + schema="grid", + if_exists="append", + index=False, + ) return +############################################################################### def load_scn_capacies_gen( scn1="status2019", scn2="eGon100RE", From de193174578aff473fe0f2ea7151e718e6431a08 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Thu, 6 Mar 2025 16:07:27 +0100 Subject: [PATCH 14/86] dealing with OCGT --- .../datasets/scenario_path/scenario_path.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 70229b219..9ce512c75 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -427,6 +427,62 @@ def import_links(scn="powerd2025"): index=False, ) + # dealing with OCGT + link_ocgt1 = ( + scn1_link[scn1_link["carrier"].isin(["OCGT"])].copy().set_index("bus1") + ) + link_ocgt2 = ( + scn2_link[scn2_link["carrier"].isin(["OCGT"])].copy().set_index("bus1") + ) + link_ocgt3 = link_ocgt2.copy().set_index("link_id") + link_ocgt3["scn_name"] = scn + + not_in_ocgt2 = link_ocgt1[ + ~link_ocgt1.index.isin(link_ocgt2.index.unique()) + ] + ac_geo = gpd.read_postgis( + """ + SELECT bus_id, geom FROM grid.egon_etrago_bus + WHERE scn_name = 'eGon100RE' + AND carrier = 'AC' + """, + con, + geom_col="geom", + ).set_index("bus_id") + ac_ocgt1_not_in_ocgt2 = ac_geo[ac_geo.index.isin(list(not_in_ocgt2.index))] + ac_ocgt2 = ac_geo[ac_geo.index.isin(list(link_ocgt2.index))] + + ocgt1_to_ocgt2 = {} + for l in not_in_ocgt2.index.unique(): + dist = ac_ocgt2.distance(ac_ocgt1_not_in_ocgt2["geom"][l]) + dist.sort_values(inplace=True) + ocgt1_to_ocgt2[l] = dist.index[0] + + link_ocgt1.reset_index(inplace=True) + link_ocgt1["bus1"] = link_ocgt1["bus1"].apply( + lambda x: x if x not in ocgt1_to_ocgt2.keys() else ocgt1_to_ocgt2[x] + ) + link_ocgt2.reset_index(inplace=True) + for b, df in link_ocgt2.groupby("bus1"): + ids = df.link_id + ini = link_ocgt1[link_ocgt1["bus1"] == b]["p_nom"].sum() + fin = df["p_nom"].sum() + factor_bus = (ini + (fin - ini) * scaling_factor[scn]) / fin + link_ocgt3.loc[ids, "p_nom"] = ( + link_ocgt3.loc[ids, "p_nom"] * factor_bus + ) + + factor_to_pypsaeur = cap_link.at["OCGT", scn] / link_ocgt3["p_nom"].sum() + link_ocgt3["p_nom"] *= factor_to_pypsaeur + + link_ocgt3.reset_index(inplace=True) + link_ocgt3.to_sql( + name="egon_etrago_link", + con=con, + schema="grid", + if_exists="append", + index=False, + ) return From 3886052fdfd47ed16c2a6eb2c6e54b86d31838c8 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Thu, 6 Mar 2025 16:19:57 +0100 Subject: [PATCH 15/86] include years for scenarios --- src/egon/data/datasets/scenario_path/scenario_path.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 9ce512c75..ecb7ea2b1 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -20,6 +20,12 @@ "powerd2035": (2035 - 2019) / (2040 - 2019), } +year_scenario = { + "powerd2025": 2025, + "powerd2030": 2030, + "powerd2035": 2035, +} + def clean_existing_scn_path_data(): scn_path = ["powerd2025", "powerd2030", "powerd2035"] From 4a27d4cb4aa7a860ec61f660e45fcf7e94096ef8 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Thu, 6 Mar 2025 16:20:16 +0100 Subject: [PATCH 16/86] dealing with H2_grid --- .../data/datasets/scenario_path/scenario_path.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index ecb7ea2b1..9684369b1 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -489,6 +489,21 @@ def import_links(scn="powerd2025"): if_exists="append", index=False, ) + + # dealing with H2_grid + h2_grid2 = scn2_link[scn2_link["carrier"] == "H2_grid"].copy() + h2_grid3 = h2_grid2.copy() + h2_grid3["scn_name"] = scn + h2_grid3 = h2_grid3[h2_grid3["build_year"] <= year_scenario[scn]] + + h2_grid3.to_sql( + name="egon_etrago_link", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + return From a115a81e985038ba7d8d034f7a16f5c5741241b9 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Thu, 6 Mar 2025 16:32:21 +0100 Subject: [PATCH 17/86] dealing with links which are iqual as in eGon100RE --- .../datasets/scenario_path/scenario_path.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 9684369b1..a6be02f26 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -504,6 +504,32 @@ def import_links(scn="powerd2025"): index=False, ) + # dealing with identical links as in eGon100RE + identical = [ + "CH4", + "CH4_to_H2", + "H2_saltcavern", + "H2_to_CH4", + "H2_to_power", + "PtH2_O2", + "PtH2_waste_heat", + "rural_heat_store_charger", + "rural_heat_store_discharger", + "power_to_H2", + "rural_heat_store_charger", + "rural_heat_store_discharger", + "BEV_charger", + ] + identical3 = scn2_link[scn2_link["carrier"].isin(identical)].copy() + identical3["scn_name"] = scn + + identical3.to_sql( + name="egon_etrago_link", + con=con, + schema="grid", + if_exists="append", + index=False, + ) return From ea8d6c5210ee8d098fbb4e5e2aad2c9ce56def40 Mon Sep 17 00:00:00 2001 From: Lennart Zimmermann Date: Fri, 7 Mar 2025 00:59:28 +0100 Subject: [PATCH 18/86] Implement method for importing foreign_network for scenario_path --- src/egon/data/datasets/pypsaeur/__init__.py | 63 ++++++++++--------- .../data/datasets/scenario_path/__init__.py | 8 +-- .../datasets/scenario_path/scenario_path.py | 23 ++++--- 3 files changed, 54 insertions(+), 40 deletions(-) diff --git a/src/egon/data/datasets/pypsaeur/__init__.py b/src/egon/data/datasets/pypsaeur/__init__.py index 416269a54..43b85692a 100755 --- a/src/egon/data/datasets/pypsaeur/__init__.py +++ b/src/egon/data/datasets/pypsaeur/__init__.py @@ -363,7 +363,7 @@ def solve_network(): print("Pypsa-eur is not executed due to the settings of egon-data") -def read_network(planning_horizon=3): +def read_network(planning_horizon=3, year=2045): if config.settings()["egon-data"]["--run-pypsa-eur"]: with open( __path__[0] + "/datasets/pypsaeur/config_solve.yaml", "r" @@ -392,7 +392,7 @@ def read_network(planning_horizon=3): / "21122024_3h_clean_run" / "results" / "postnetworks" - / "base_s_39_lc1.25__cb40ex0-T-H-I-B-solar+p3-dist1_2045.nc" + / f"base_s_39_lc1.25__cb40ex0-T-H-I-B-solar+p3-dist1_{year}.nc" ) return pypsa.Network(target_file) @@ -575,9 +575,9 @@ def combine_decentral_and_rural_heat(network_solved, network_prepared): return network_prepared, network_solved -def neighbor_reduction(): - network_solved = read_network() - network_prepared = prepared_network(planning_horizon="2045") +def neighbor_reduction(scn_name, year=2045): + network_solved = read_network(year=year) + network_prepared = prepared_network(year=year) # network.links.drop("pipe_retrofit", axis="columns", inplace=True) @@ -619,6 +619,7 @@ def neighbor_reduction(): .mul(network_solved.snapshot_weightings.generators, axis=0) .sum() ) + methanolisation = ( network_solved.links_t.p0[ network_solved.links.loc[ @@ -642,7 +643,6 @@ def neighbor_reduction(): / 8760 ) # drop foreign lines and links from the 2nd row - network_solved.lines = network_solved.lines.drop( network_solved.lines[ ( @@ -980,7 +980,7 @@ def neighbor_reduction(): # Connect to local database engine = db.engine() - neighbors["scn_name"] = "eGon100RE" + neighbors["scn_name"] = scn_name neighbors.index = neighbors["new_index"] # Correct geometry for non AC buses @@ -1048,9 +1048,9 @@ def neighbor_reduction(): index=True, index_label="bus_id", ) - + # prepare and write neighboring crossborder lines to etrago tables - def lines_to_etrago(neighbor_lines=neighbor_lines, scn="eGon100RE"): + def lines_to_etrago(neighbor_lines=neighbor_lines, scn=scn_name): neighbor_lines["scn_name"] = scn neighbor_lines["cables"] = 3 * neighbor_lines["num_parallel"].astype( int @@ -1086,7 +1086,7 @@ def lines_to_etrago(neighbor_lines=neighbor_lines, scn="eGon100RE"): .set_crs(4326) ) - neighbor_lines["lifetime"] = get_sector_parameters("electricity", scn)[ + neighbor_lines["lifetime"] = get_sector_parameters("electricity", 'eGon100RE')[ "lifetime" ]["ac_ehv_overhead_line"] @@ -1099,9 +1099,9 @@ def lines_to_etrago(neighbor_lines=neighbor_lines, scn="eGon100RE"): index_label="line_id", ) - lines_to_etrago(neighbor_lines=neighbor_lines, scn="eGon100RE") + lines_to_etrago(neighbor_lines=neighbor_lines, scn=scn_name) - def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): + def links_to_etrago(neighbor_links, scn=scn_name, extendable=True): """Prepare and write neighboring crossborder links to eTraGo table This function prepare the neighboring crossborder links @@ -1270,7 +1270,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): neighbor_links = neighbor_links[ ~neighbor_links.carrier.isin(excluded_carriers) ] - + # Combine CHP_CC and CHP chp_cc = neighbor_links[ neighbor_links.carrier == "urban central gas CHP CC" @@ -1284,29 +1284,35 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): neighbor_links.Link == row.Link.replace("CHP CC", "CHP"), "p_nom" ] += row.p_nom neighbor_links.drop(index, inplace=True) - + # Combine heat pumps # Like in Germany, there are air heat pumps in central heat grids # and ground heat pumps in rural areas + + #consider aso urban decentral air heat pumps in rural_heat_pumps + mask = neighbor_links["Link"].str.contains("urban decentral air", na=False) + neighbor_links.loc[mask, "Link"] = neighbor_links.loc[mask, "Link"].str.replace("urban decentral", "rural") rural_air = neighbor_links[neighbor_links.carrier == "rural air heat pump"] + for index, row in rural_air.iterrows(): neighbor_links.loc[ neighbor_links.Link == row.Link.replace("air", "ground"), "p_nom_opt", ] += row.p_nom_opt + neighbor_links.loc[ neighbor_links.Link == row.Link.replace("air", "ground"), "p_nom" ] += row.p_nom neighbor_links.drop(index, inplace=True) links_to_etrago( neighbor_links[neighbor_links.carrier.isin(extendable_links_carriers)], - "eGon100RE", + scn_name, ) links_to_etrago( neighbor_links[ ~neighbor_links.carrier.isin(extendable_links_carriers) ], - "eGon100RE", + scn_name, extendable=False, ) # Include links time-series @@ -1352,7 +1358,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ev_p_max_pu.rename(columns={i: new_index[0]}, inplace=True) # prepare neighboring generators for etrago tables - neighbor_gens["scn_name"] = "eGon100RE" + neighbor_gens["scn_name"] = scn_name neighbor_gens["p_nom"] = neighbor_gens["p_nom_opt"] neighbor_gens["p_nom_extendable"] = False @@ -1396,7 +1402,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) # prepare neighboring loads for etrago tables - neighbor_loads["scn_name"] = "eGon100RE" + neighbor_loads["scn_name"] = scn_name # Unify carrier names neighbor_loads.carrier = neighbor_loads.carrier.str.replace(" ", "_") @@ -1418,6 +1424,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): columns=["Load"], errors="ignore", ) + neighbor_loads.to_sql( "egon_etrago_load", @@ -1429,7 +1436,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) # prepare neighboring stores for etrago tables - neighbor_stores["scn_name"] = "eGon100RE" + neighbor_stores["scn_name"] = scn_name # Unify carrier names neighbor_stores.carrier = neighbor_stores.carrier.str.replace(" ", "_") @@ -1481,7 +1488,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) # prepare neighboring storage_units for etrago tables - neighbor_storage["scn_name"] = "eGon100RE" + neighbor_storage["scn_name"] = scn_name # Unify carrier names neighbor_storage.carrier = neighbor_storage.carrier.str.replace(" ", "_") @@ -1513,7 +1520,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): columns=["scn_name", "temp_id", "p_set"], index=neighbor_loads_t.columns, ) - neighbor_loads_t_etrago["scn_name"] = "eGon100RE" + neighbor_loads_t_etrago["scn_name"] = scn_name neighbor_loads_t_etrago["temp_id"] = 1 for i in neighbor_loads_t.columns: neighbor_loads_t_etrago["p_set"][i] = neighbor_loads_t[ @@ -1534,7 +1541,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): columns=["scn_name", "temp_id", "p_max_pu", "efficiency"], index=neighbor_eff_t.columns.to_list() + ev_p_max_pu.columns.to_list(), ) - neighbor_link_t_etrago["scn_name"] = "eGon100RE" + neighbor_link_t_etrago["scn_name"] = scn_name neighbor_link_t_etrago["temp_id"] = 1 for i in neighbor_eff_t.columns: neighbor_link_t_etrago["efficiency"][i] = neighbor_eff_t[ @@ -1557,7 +1564,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): columns=["scn_name", "temp_id", "p_max_pu"], index=neighbor_gens_t.columns, ) - neighbor_gens_t_etrago["scn_name"] = "eGon100RE" + neighbor_gens_t_etrago["scn_name"] = scn_name neighbor_gens_t_etrago["temp_id"] = 1 for i in neighbor_gens_t.columns: neighbor_gens_t_etrago["p_max_pu"][i] = neighbor_gens_t[ @@ -1578,7 +1585,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): columns=["scn_name", "temp_id", "e_min_pu"], index=neighbor_stores_t.columns, ) - neighbor_stores_t_etrago["scn_name"] = "eGon100RE" + neighbor_stores_t_etrago["scn_name"] = scn_name neighbor_stores_t_etrago["temp_id"] = 1 for i in neighbor_stores_t.columns: neighbor_stores_t_etrago["e_min_pu"][i] = neighbor_stores_t[ @@ -1599,7 +1606,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): columns=["scn_name", "temp_id", "inflow"], index=neighbor_storage_t.columns, ) - neighbor_storage_t_etrago["scn_name"] = "eGon100RE" + neighbor_storage_t_etrago["scn_name"] = scn_name neighbor_storage_t_etrago["temp_id"] = 1 for i in neighbor_storage_t.columns: neighbor_storage_t_etrago["inflow"][i] = neighbor_storage_t[ @@ -1620,7 +1627,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): neighbor_lines_t_etrago = pd.DataFrame( columns=["scn_name", "s_max_pu"], index=neighbor_lines_t.columns ) - neighbor_lines_t_etrago["scn_name"] = "eGon100RE" + neighbor_lines_t_etrago["scn_name"] = scn_name for i in neighbor_lines_t.columns: neighbor_lines_t_etrago["s_max_pu"][i] = neighbor_lines_t[ @@ -1637,7 +1644,7 @@ def links_to_etrago(neighbor_links, scn="eGon100RE", extendable=True): ) -def prepared_network(planning_horizon=3): +def prepared_network(planning_horizon=3, year=2045): if egon.data.config.settings()["egon-data"]["--run-pypsa-eur"]: with open( __path__[0] + "/datasets/pypsaeur/config_prepare.yaml", "r" @@ -1667,7 +1674,7 @@ def prepared_network(planning_horizon=3): / "results" / "prenetworks" / "prenetwork_post-manipulate_pre-solve" - / "base_s_39_lc1.25__cb40ex0-T-H-I-B-solar+p3-dist1_2045.nc" + / f"base_s_39_lc1.25__cb40ex0-T-H-I-B-solar+p3-dist1_{year}.nc" ) return pypsa.Network(target_file.absolute().as_posix()) diff --git a/src/egon/data/datasets/scenario_path/__init__.py b/src/egon/data/datasets/scenario_path/__init__.py index e41048b2c..046668114 100644 --- a/src/egon/data/datasets/scenario_path/__init__.py +++ b/src/egon/data/datasets/scenario_path/__init__.py @@ -1,5 +1,5 @@ from egon.data.datasets import Dataset -from egon.data.datasets.scenario_path.import_status2019 import ( +from egon.data.datasets.scenario_path.scenario_path import ( clean_existing_scn_path_data, import_network_structure, import_generators, @@ -19,7 +19,7 @@ def create_powerd2025(): import_links(scn) import_storage_units(scn) import_stores(scn) - import_foreign(scn) + import_foreign(scn, year=2025) return @@ -32,7 +32,7 @@ def create_powerd2030(): import_links(scn) import_storage_units(scn) import_stores(scn) - import_foreign(scn) + import_foreign(scn, year=2030) return @@ -45,7 +45,7 @@ def create_powerd2035(): import_links(scn) import_storage_units(scn) import_stores(scn) - import_foreign(scn) + import_foreign(scn, year=2035) return diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 6c9d4528b..641c9022a 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -9,6 +9,8 @@ from egon.data import config, db import egon.data.config +from egon.data.datasets.pypsaeur import neighbor_reduction + sources = egon.data.config.datasets()["scenario_path"]["sources"] con = db.engine() @@ -47,14 +49,13 @@ def clean_existing_scn_path_data(): def import_network_structure(scn=str): - scn = "powerd2025" # Import buses bus = pd.read_sql( sql=""" - SELECT * from grid.egon_etrago_bus - WHERE scn_name = 'eGon100RE' - """, + SELECT * from grid.egon_etrago_bus + WHERE scn_name = 'eGon100RE' AND carrier = 'AC' + """, con=con, ) @@ -71,8 +72,10 @@ def import_network_structure(scn=str): # Import lines line = pd.read_sql( sql=""" - SELECT * from grid.egon_etrago_line - WHERE scn_name = 'eGon100RE' + SELECT * from grid.egon_etrago_line + WHERE scn_name = 'eGon100RE' AND bus0 IN + (SElECT bus_id FROM grid.egon_etrago_bus + WHERE country ='DE') """, con=con, ) @@ -105,7 +108,6 @@ def import_network_structure(scn=str): if_exists="append", index=False, ) - return @@ -307,4 +309,9 @@ def adjust_generators(scn1: dict, scn2: dict, new_scn: dict, name: str): eGon100RE = load_scn_no_time_no_foreign(scn_name="eGon100RE") status2019 = load_scn_no_time_no_foreign(scn_name="status2019") -cap_gen, cap_link, cap_storage = load_scn_capacies(status2019, eGon100RE) +#cap_gen, cap_link, cap_storage = load_scn_capacies(status2019, eGon100RE) + + + +def import_foreign(scn_name, year): + neighbor_reduction(scn_name, year) From 4b7c72554dbe94eaf52adcbb2223db47b46d59ed Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Sat, 8 Mar 2025 22:51:50 +0100 Subject: [PATCH 19/86] dealing with links only present in eGon100RE --- .../datasets/scenario_path/scenario_path.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index a6be02f26..9cabc9059 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -530,6 +530,30 @@ def import_links(scn="powerd2025"): if_exists="append", index=False, ) + + # dealing with carriers which are only present in eGon100RE and scn path. + # This links are only scaled + scale_carriers = [ + "rural_resistive_heater", + "rural_gas_boiler", + "central_resistive_heater", + ] + scale3 = scn2_link[scn2_link["carrier"].isin(scale_carriers)].copy() + scale3["scn_name"] = scn + + for c, df in scale3.groupby("carrier"): + id = df.index + objective = cap_link.at[c, scn] + scale3.loc[id, "p_nom"] *= objective / scale3.loc[id, "p_nom"].sum() + + scale3.to_sql( + name="egon_etrago_link", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + return From afd5982a29f7cc42bc7b74586f4051b0bc3e9cd4 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Sat, 8 Mar 2025 22:52:26 +0100 Subject: [PATCH 20/86] dealing with geo_thermal and fossil fuel generators --- .../datasets/scenario_path/scenario_path.py | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 9cabc9059..4e3c24b49 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -644,6 +644,85 @@ def load_scn_capacies_gen( return gen_capacities +def import_generators(scn="powerd2025"): + scn = "powerd2025" + cap_gen = load_scn_capacies_gen() + + scn1_gen = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_generator + WHERE scn_name = 'status2019' + AND bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'status2019' + ) + """, + con, + ) + + scn2_gen = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_generator + WHERE scn_name = 'eGon100RE' + AND bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + """, + con, + ) + + # Dealing with geo_thermal + geo3 = scn2_gen[scn2_gen["carrier"] == "geo_thermal"].copy() + geo3["scn_name"] = scn + + objective = cap_gen.at["geo_thermal", scn] + geo3["p_nom"] *= objective / geo3["p_nom"].sum() + + geo3.to_sql( + name="egon_etrago_generator", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + # Dealing with oil, coal and lignite + + fossil_carriers = [ + "oil", + "coal", + "lignite", + ] + + fossil_carriers = cap_gen.loc[fossil_carriers, scn][ + ~cap_gen.loc[fossil_carriers, scn].isna() + ].index.values + + if len(fossil_carriers) == 0: + return + + fossil3 = scn1_gen[scn1_gen["carrier"].isin(fossil_carriers)].copy() + fossil3["scn_name"] = scn + + for c, df in fossil3.groupby("carrier"): + id = df.index + objective = cap_gen.at[c, scn] + fossil3.loc[id, "p_nom"] *= objective / fossil3.loc[id, "p_nom"].sum() + + fossil3.to_sql( + name="egon_etrago_gen", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + return + + # load scenarios def load_scn_no_time_no_foreign(scn_name): # load scenario data without timeseries and foreign countries data From 1517b8f59f3988bc73d414c9779de9f3b90943c7 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Sun, 9 Mar 2025 12:06:36 +0100 Subject: [PATCH 21/86] dealing with RE generators with time --- .../datasets/scenario_path/scenario_path.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 4e3c24b49..1344df5b6 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -720,6 +720,98 @@ def import_generators(scn="powerd2025"): index=False, ) + # Dialing with run_of_river, solar_rooftop, wind_offshore, wind_onshore + # and solar + var_carriers = [ + "run_of_river", + "solar_rooftop", + "wind_offshore", + "wind_onshore", + "solar", + ] + gen_var1 = ( + scn1_gen[scn1_gen["carrier"].isin(var_carriers)] + .copy() + .set_index("generator_id") + ) + gen_var2 = ( + scn2_gen[scn2_gen["carrier"].isin(var_carriers)] + .copy() + .set_index("generator_id") + ) + gen_var3 = gen_var2.copy() + gen_var3["scn_name"] = scn + + total_gen_var = pd.concat([gen_var1, gen_var2]).copy() + + new_gen_from_2019 = [] + for c, df in total_gen_var.groupby(["carrier", "bus"]): + df1 = df[df["scn_name"] == "status2019"] + df2 = df[df["scn_name"] == "eGon100RE"] + init = df1.p_nom.sum() + final = df2.p_nom.sum() + + if (init != 0) & (final != 0): + ids = df2.index + gen_var3.loc[ids, "p_nom"] = gen_var3.loc[ids, "p_nom"] - ( + (final - init) * scaling_factor[scn] + ) + + elif final == 0: + new_gen = df1.copy() + [new_gen_from_2019.append(x) for x in list(new_gen.index)] + new_gen["scn_name"] = scn + new_gen["p_nom"] *= scaling_factor[scn] + gen_var3 = pd.concat([gen_var3, new_gen]) + + for c, df in gen_var3.groupby("carrier"): + factor_to_pypsaeur = cap_gen.at[c, scn] / df["p_nom"].sum() + gen_var3.loc[df.index, "p_nom"] *= factor_to_pypsaeur + + gen_var3.reset_index(inplace=True) + gen_var3.to_sql( + name="egon_etrago_generator", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + gen_var3_t_2019 = pd.read_sql( + f""" + SELECT * FROM grid.egon_etrago_generator_timeseries + WHERE generator_id IN {tuple(new_gen_from_2019)} + AND scn_name = 'status2019' + """, + con, + ) + + gen_var3_t_100RE = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_generator_timeseries + WHERE generator_id IN( + SELECT generator_id FROM grid.egon_etrago_generator + WHERE bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + AND scn_name = 'eGon100RE') + """, + con, + ) + + gen_var3_t = pd.concat([gen_var3_t_2019, gen_var3_t_100RE]) + gen_var3_t["scn_name"] = scn + + gen_var3_t.to_sql( + name="egon_etrago_generator_timeseries", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + return From 9f916c5d91254a71348d8999921c7fe72b794f63 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Sun, 9 Mar 2025 12:55:20 +0100 Subject: [PATCH 22/86] dealing with generators ch4 and o2 --- .../datasets/scenario_path/scenario_path.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 1344df5b6..3f31d589e 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -812,6 +812,37 @@ def import_generators(scn="powerd2025"): index=False, ) + # Dealing with O2 + oxy3 = scn2_gen[scn2_gen["carrier"] == "O2"].copy() + oxy3["scn_name"] = scn + + oxy3.to_sql( + name="egon_etrago_generator", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + # Dealing with CH4 + gas1 = scn1_gen[scn1_gen["carrier"] == "CH4"].copy() + gas2 = scn2_gen[scn2_gen["carrier"] == "CH4"].copy() + gas3 = scn2_gen[scn2_gen["carrier"] == "CH4"].copy() + gas3["scn_name"] = scn + + obj = ( + gas1["p_nom"].sum() + + (gas2["p_nom"].sum() - gas1["p_nom"].sum()) * scaling_factor[scn] + ) + gas3["p_nom"] *= obj / gas3["p_nom"].sum() + + gas3.to_sql( + name="egon_etrago_generator", + con=con, + schema="grid", + if_exists="append", + index=False, + ) return From 8e3cfb2e2a1fd050fee80b62a751740e340da587 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Sun, 9 Mar 2025 12:56:20 +0100 Subject: [PATCH 23/86] delete function load_scn_no_time_no_foreign --- .../datasets/scenario_path/scenario_path.py | 54 ------------------- 1 file changed, 54 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 3f31d589e..0639fd3f0 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -844,57 +844,3 @@ def import_generators(scn="powerd2025"): index=False, ) return - - -# load scenarios -def load_scn_no_time_no_foreign(scn_name): - # load scenario data without timeseries and foreign countries data - - scn_tables = {} - for t in ["egon_etrago_bus"]: - scn_tables[t] = pd.read_sql( - f""" - SELECT * FROM grid.{t} - WHERE scn_name = '{scn_name}' - AND country = 'DE' - """, - con, - ) - - for t in [ - "egon_etrago_generator", - "egon_etrago_load", - "egon_etrago_storage", - "egon_etrago_store", - ]: - scn_tables[t] = pd.read_sql( - f""" - SELECT * FROM grid.{t} - WHERE scn_name = '{scn_name}' - AND bus IN (SELECT bus_id from grid.egon_etrago_bus - WHERE scn_name = '{scn_name}' - AND country = 'DE') - """, - con, - ) - - for t in [ - "egon_etrago_line", - "egon_etrago_link", - "egon_etrago_transformer", - ]: - scn_tables[t] = pd.read_sql( - f""" - SELECT * FROM grid.{t} - WHERE scn_name = '{scn_name}' - AND bus0 IN (SELECT bus_id from grid.egon_etrago_bus - WHERE scn_name = '{scn_name}' - AND country = 'DE') - AND bus1 IN (SELECT bus_id from grid.egon_etrago_bus - WHERE scn_name = '{scn_name}' - AND country = 'DE') - """, - con, - ) - - return scn_tables From c45c65390886b085980e1c8742c93f2040c36bad Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Sun, 9 Mar 2025 14:42:09 +0100 Subject: [PATCH 24/86] dealing with loads only present in eGon100RE --- .../datasets/scenario_path/scenario_path.py | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 0639fd3f0..cc0cf0357 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -844,3 +844,106 @@ def import_generators(scn="powerd2025"): index=False, ) return + + +def import_loads(scn): + scn = "powerd2025" + + scn1_load = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_load + WHERE scn_name = 'status2019' + AND bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'status2019' + ) + """, + con, + index_col="load_id", + ) + + scn2_load = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_load + WHERE scn_name = 'eGon100RE' + AND bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + """, + con, + index_col="load_id", + ) + + scn3_load = scn2_load.copy() + + scn1_load_t = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_load_timeseries + WHERE scn_name = 'status2019' + AND load_id IN ( + SELECT load_id from grid.egon_etrago_load + WHERE bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'status2019' + ) + AND scn_name = 'status2019') + """, + con, + index_col="load_id", + ) + + scn2_load_t = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_load_timeseries + WHERE scn_name = 'eGon100RE' + AND load_id IN ( + SELECT load_id from grid.egon_etrago_load + WHERE bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + AND scn_name = 'eGon100RE') + """, + con, + index_col="load_id", + ) + + only100_carrier = [ + "O2", + "H2_hgv_load", + "H2_for_industry", + "CH4_for_industry", + ] + only100 = scn2_load[scn2_load["carrier"].isin(only100_carrier)] + + for c, df in only100.groupby("carrier"): + a = scn2_load_t.loc[df.index, "p_set"].apply( + lambda x: np.array(x).sum() + ) + print(f"{c}:{a.sum()}") + + for c, df in only100.groupby("carrier"): + scn2_load_t.loc[df.index, "p_set"] = scn2_load_t.loc[ + df.index, "p_set" + ].apply(lambda x: np.array(x) * scaling_factor[scn]) + + for c, df in only100.groupby("carrier"): + a = scn2_load_t.loc[df.index, "p_set"].apply( + lambda x: np.array(x).sum() + ) + print(f"{c}:{a.sum()}") + + scn3_load["scn_year"] = scn + scn3_load.reset_index(inplace=True) + scn3_load.to_sql( + name="egon_etrago_generator", + con=con, + schema="grid", + if_exists="append", + index=False, + ) From a383d0b494bb13f3bcbd2a2006ea950e7e166f2b Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Sun, 9 Mar 2025 19:37:22 +0100 Subject: [PATCH 25/86] Dealing with land_transport_EV loads --- .../datasets/scenario_path/scenario_path.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index cc0cf0357..119664f47 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -913,6 +913,7 @@ def import_loads(scn): index_col="load_id", ) + # dealing with loads only present in eGon100RE only100_carrier = [ "O2", "H2_hgv_load", @@ -938,6 +939,28 @@ def import_loads(scn): ) print(f"{c}:{a.sum()}") + # Dealing with land_transport_EV loads + evl1 = scn1_load[scn1_load["carrier"] == "land_transport_EV"].copy() + evl2 = scn2_load[scn2_load["carrier"] == "land_transport_EV"].copy() + evl1_total = ( + scn1_load_t.loc[evl1.index, "p_set"] + .apply(lambda x: np.array(x).sum()) + .sum() + ) + evl2_total = ( + scn2_load_t.loc[evl2.index, "p_set"] + .apply(lambda x: np.array(x).sum()) + .sum() + ) + + objective = evl1_total + (evl2_total - evl1_total) * scaling_factor[scn] + + scn2_load_t.loc[evl2.index, "p_set"] = scn2_load_t.loc[ + evl2.index, "p_set" + ].apply(lambda x: np.array(x) * objective / evl2_total) + + # Dealing with AC loads + scn3_load["scn_year"] = scn scn3_load.reset_index(inplace=True) scn3_load.to_sql( From 426db9b17d2c89dcdefa07ac4a3d0e45d27dfd68 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Sun, 9 Mar 2025 20:21:13 +0100 Subject: [PATCH 26/86] dealing with AC loads --- .../datasets/scenario_path/scenario_path.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 119664f47..73e53d271 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -960,6 +960,30 @@ def import_loads(scn): ].apply(lambda x: np.array(x) * objective / evl2_total) # Dealing with AC loads + ac_load = pd.concat( + [ + scn1_load[scn1_load["carrier"] == "AC"], + scn2_load[scn2_load["carrier"] == "AC"], + ] + ).copy() + + for b, df in ac_load.groupby("bus"): + df1 = df[df["scn_name"] == "status2019"] + df2 = df[df["scn_name"] == "eGon100RE"] + df1_total = ( + scn1_load_t.loc[df1.index, "p_set"] + .apply(lambda x: np.array(x).sum()) + .sum() + ) + df2_total = ( + scn2_load_t.loc[df2.index, "p_set"] + .apply(lambda x: np.array(x).sum()) + .sum() + ) + objective = df1_total + (df2_total - df1_total) * scaling_factor[scn] + scn2_load_t.loc[df2.index, "p_set"] = scn2_load_t.loc[ + df2.index, "p_set" + ].apply(lambda x: np.array(x) * objective / df2_total) scn3_load["scn_year"] = scn scn3_load.reset_index(inplace=True) From e6668ec7e836b1387ae0db91e32362b22f826224 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Sun, 9 Mar 2025 20:22:06 +0100 Subject: [PATCH 27/86] dealing with rural_heat loads --- .../datasets/scenario_path/scenario_path.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 73e53d271..4e2e5a74f 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -985,6 +985,26 @@ def import_loads(scn): df2.index, "p_set" ].apply(lambda x: np.array(x) * objective / df2_total) + # Dealing with rural_heat loads + rh1 = scn1_load[scn1_load["carrier"] == "rural_heat"].copy() + rh2 = scn2_load[scn2_load["carrier"] == "rural_heat"].copy() + rh1_total = ( + scn1_load_t.loc[rh1.index, "p_set"] + .apply(lambda x: np.array(x).sum()) + .sum() + ) + rh2_total = ( + scn2_load_t.loc[rh2.index, "p_set"] + .apply(lambda x: np.array(x).sum()) + .sum() + ) + + objective = rh1_total + (rh2_total - rh1_total) * scaling_factor[scn] + + scn2_load_t.loc[rh2.index, "p_set"] = scn2_load_t.loc[ + rh2.index, "p_set" + ].apply(lambda x: np.array(x) * objective / rh2_total) + scn3_load["scn_year"] = scn scn3_load.reset_index(inplace=True) scn3_load.to_sql( From 01a133d78b3f2a1fbbeba74a3c2fa3220366e287 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Sun, 9 Mar 2025 20:48:55 +0100 Subject: [PATCH 28/86] Dealing with central_heat loads --- .../datasets/scenario_path/scenario_path.py | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 4e2e5a74f..4bcbad8ef 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1005,10 +1005,40 @@ def import_loads(scn): rh2.index, "p_set" ].apply(lambda x: np.array(x) * objective / rh2_total) - scn3_load["scn_year"] = scn + # Dealing with central_heat loads + ch1 = scn1_load[scn1_load["carrier"] == "central_heat"].copy() + ch2 = scn2_load[scn2_load["carrier"] == "central_heat"].copy() + ch1_total = ( + scn1_load_t.loc[ch1.index, "p_set"] + .apply(lambda x: np.array(x).sum()) + .sum() + ) + ch2_total = ( + scn2_load_t.loc[ch2.index, "p_set"] + .apply(lambda x: np.array(x).sum()) + .sum() + ) + + objective = ch1_total + (ch2_total - ch1_total) * scaling_factor[scn] + + scn2_load_t.loc[ch2.index, "p_set"] = scn2_load_t.loc[ + ch2.index, "p_set" + ].apply(lambda x: np.array(x) * objective / ch2_total) + + scn3_load["scn_name"] = scn scn3_load.reset_index(inplace=True) scn3_load.to_sql( - name="egon_etrago_generator", + name="egon_etrago_load", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + scn2_load_t["p_set"] = scn2_load_t["p_set"].apply(list) + scn2_load_t["scn_name"] = scn + scn2_load_t.reset_index(inplace=True) + scn2_load_t.to_sql( + name="egon_etrago_load_timeseries", con=con, schema="grid", if_exists="append", From 480dac108500dfb478a4a8630308520ee2117081 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Sun, 9 Mar 2025 20:50:46 +0100 Subject: [PATCH 29/86] create function sto_units and stores --- src/egon/data/datasets/scenario_path/scenario_path.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 4bcbad8ef..adc802320 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1044,3 +1044,10 @@ def import_loads(scn): if_exists="append", index=False, ) + + +def import_storage_units(scn): + return + +def import_stores(scn): + return \ No newline at end of file From 3b4ffeda438754b9b1750326830ae249e7af4db5 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 10 Mar 2025 07:17:46 +0100 Subject: [PATCH 30/86] include other buses --- .../datasets/scenario_path/scenario_path.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index c135feffd..9002a8eae 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -61,10 +61,10 @@ def clean_existing_scn_path_data(): return -def import_network_structure(scn:str): +def import_network_structure(scn: str): # Import buses - bus = pd.read_sql( + bus_ac = pd.read_sql( sql=""" SELECT * from grid.egon_etrago_bus WHERE scn_name = 'eGon100RE' AND carrier = 'AC' @@ -72,6 +72,16 @@ def import_network_structure(scn:str): con=con, ) + other_buses = pd.read_sql( + sql=""" + SELECT * from grid.egon_etrago_bus + WHERE scn_name = 'eGon100RE' AND carrier <> 'AC' AND country = 'DE' + """, + con=con, + ) + + bus = pd.concat([bus_ac, other_buses]) + bus["scn_name"] = scn bus.to_sql( @@ -218,7 +228,7 @@ def load_scn_capacies_link( return link_capacities -def import_links(scn:str): +def import_links(scn: str): cap_link = load_scn_capacies_link() @@ -645,7 +655,7 @@ def load_scn_capacies_gen( return gen_capacities -def import_generators(scn:str): +def import_generators(scn: str): cap_gen = load_scn_capacies_gen() @@ -847,7 +857,7 @@ def import_generators(scn:str): return -def import_loads(scn:str): +def import_loads(scn: str): scn1_load = pd.read_sql( """ From 2829226680ca9f5ce872e34a6cc0a9f0e4194af2 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 10 Mar 2025 07:52:40 +0100 Subject: [PATCH 31/86] fix missin bus1 OCGT --- src/egon/data/datasets/scenario_path/scenario_path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 9002a8eae..e6b8ca0ce 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -452,7 +452,7 @@ def import_links(scn: str): link_ocgt2 = ( scn2_link[scn2_link["carrier"].isin(["OCGT"])].copy().set_index("bus1") ) - link_ocgt3 = link_ocgt2.copy().set_index("link_id") + link_ocgt3 = link_ocgt2.copy().reset_index().set_index("link_id") link_ocgt3["scn_name"] = scn not_in_ocgt2 = link_ocgt1[ From 5116a51a654fa1fc2fe448163cff5d0032d79056 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 10 Mar 2025 08:28:15 +0100 Subject: [PATCH 32/86] dealing with storage_units --- .../datasets/scenario_path/scenario_path.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index e6b8ca0ce..df79e6112 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1057,6 +1057,69 @@ def import_loads(scn: str): def import_storage_units(scn): + + scn1_su = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_storage + WHERE scn_name = 'status2019' + AND bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'status2019' + ) + """, + con, + ) + + scn2_su = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_storage + WHERE scn_name = 'eGon100RE' + AND bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + """, + con, + ) + + # Dealing with battery + battery3 = scn2_su[scn2_su["carrier"] == "battery"].copy() + battery3["scn_name"] = scn + + battery3.to_sql( + name="egon_etrago_storage", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + # Dealing with pumped_hydro + capacity_ph = pd.read_sql( + f""" + SELECT capacity FROM supply.egon_scenario_capacities + WHERE carrier = 'pumped_hydro' AND scenario_name = '{scn}' + """, + con, + ).iat[0, 0] + + ph1 = scn1_su[scn1_su["carrier"] == "pumped_hydro"] + ph2 = scn2_su[scn2_su["carrier"] == "pumped_hydro"] + ph3 = ph2.copy() + + ph3["p_nom"] *= capacity_ph / ph3["p_nom"].sum() + + ph3["scn_name"] = scn + ph3.to_sql( + name="egon_etrago_storage", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + return From 002377ed37b53607b710893e16185ee6c2aa4688 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 10 Mar 2025 09:59:40 +0100 Subject: [PATCH 33/86] deal with stores --- .../datasets/scenario_path/scenario_path.py | 111 +++++++++++++++++- 1 file changed, 109 insertions(+), 2 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index df79e6112..aca93de3a 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1056,7 +1056,7 @@ def import_loads(scn: str): ) -def import_storage_units(scn): +def import_storage_units(scn: str): scn1_su = pd.read_sql( """ @@ -1123,7 +1123,114 @@ def import_storage_units(scn): return -def import_stores(scn): +def import_stores(scn: str): + scn1_store = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_store + WHERE scn_name = 'status2019' + AND bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'status2019' + ) + """, + con, + ) + + scn2_store = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_store + WHERE scn_name = 'eGon100RE' + AND bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + """, + con, + ) + + scn3_store = scn2_store.copy() + + scn2_store_t = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_store_timeseries + WHERE scn_name = 'eGon100RE' + AND store_id IN ( + SELECT store_id from grid.egon_etrago_store + WHERE bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + AND scn_name = 'eGon100RE') + """, + con, + index_col="store_id", + ) + + all_stores = pd.concat([scn1_store, scn2_store]).copy() + + # dealing with stores only present in eGon100RE + only100_carrier = [ + "central_heat_store", + "rural_heat_store", + "battery_storage", + "H2_overground", + "CH4", + "H2_underground", + ] + only100_store = scn2_store[scn2_store["carrier"].isin(only100_carrier)] + for c, df in only100_store.groupby("carrier"): + scn3_store.loc[df.index, "e_nom"] *= scaling_factor[scn] + + # Dealing with DSM + dsm = all_stores[all_stores["carrier"] == "dsm"].copy() + dsm_buses = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_bus + WHERE scn_name IN ('eGon100RE', 'status2019') + AND carrier = 'dsm' + """, + con, + index_col="bus_id", + ) + map_scn1_to_scn2 = {} + for b, df in dsm_buses.groupby(["x", "y"]): + bus1 = df.index[df["scn_name"] == "status2019"][0] + bus2 = df.index[df["scn_name"] == "eGon100RE"][0] + map_scn1_to_scn2[bus1] = bus2 + + dsm["bus"] = dsm["bus"].apply( + lambda x: map_scn1_to_scn2[x] if x in map_scn1_to_scn2.keys() else x + ) + + for b, df in dsm.groupby("bus"): + init = df["e_nom"][df["scn_name"] == "status2019"].sum() + final = df["e_nom"][df["scn_name"] == "eGon100RE"].sum() + objective = init + (final - init) * scaling_factor[scn] + scn3_store.loc[df.index, "e_nom"] *= ( + objective / scn3_store.loc[df.index, "e_nom"].sum() + ) + + scn3_store["scn_name"] = scn + scn3_store.to_sql( + name="egon_etrago_store", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + scn2_store_t["scn_name"] = scn + scn2_store_t.reset_index(inplace=True) + scn2_store_t.to_sql( + name="egon_etrago_store_timeseries", + con=con, + schema="grid", + if_exists="append", + index=False, + ) return From 98c49f0e5ef7da0799896e556afbc81558ea1a00 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 10 Mar 2025 11:09:10 +0100 Subject: [PATCH 34/86] fix sql for gen_timeseries --- .../datasets/scenario_path/scenario_path.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index aca93de3a..c69c795a3 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -798,16 +798,18 @@ def import_generators(scn: str): ) gen_var3_t_100RE = pd.read_sql( - """ + f""" SELECT * FROM grid.egon_etrago_generator_timeseries WHERE generator_id IN( - SELECT generator_id FROM grid.egon_etrago_generator - WHERE bus IN ( - SELECT bus_id FROM grid.egon_etrago_bus - WHERE country = 'DE' - AND scn_name = 'eGon100RE' - ) - AND scn_name = 'eGon100RE') + SELECT generator_id FROM grid.egon_etrago_generator + WHERE scn_name = 'eGon100RE' + AND carrier IN {tuple(var_carriers)} + AND bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + )) + AND scn_name = 'eGon100RE' """, con, ) From b63112d254a7563b30e94669b2d0bcd9fba4248e Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 11 Mar 2025 10:16:48 +0100 Subject: [PATCH 35/86] include domestic lines in foreign countries --- .../datasets/scenario_path/scenario_path.py | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index c69c795a3..410f8ab09 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -93,16 +93,38 @@ def import_network_structure(scn: str): ) # Import lines - line = pd.read_sql( + line_de = pd.read_sql( sql=""" SELECT * from grid.egon_etrago_line - WHERE scn_name = 'eGon100RE' AND bus0 IN - (SElECT bus_id FROM grid.egon_etrago_bus - WHERE country ='DE') - """, + WHERE scn_name = 'eGon100RE' + AND ((bus0 IN (SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE')) + OR (bus1 IN (SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE'))) + """, + con=con, + index_col="line_id", + ) + line_foreign = pd.read_sql( + sql=""" + SELECT * FROM grid.egon_etrago_line + WHERE line_id IN (SELECT l.line_id FROM grid.egon_etrago_line l + JOIN grid.egon_etrago_bus b1 ON l.bus0 = b1.bus_id + JOIN grid.egon_etrago_bus b2 ON l.bus1 = b2.bus_id + WHERE b1.country = b2.country + AND l.scn_name = 'eGon100RE' + AND b1.country <> 'DE' + AND b2.country <> 'DE') + AND scn_name = 'eGon100RE' + """, con=con, + index_col="line_id", ) + line = pd.concat([line_de, line_foreign]) + line.reset_index(inplace=True) line["scn_name"] = scn line.to_sql( From eb7459464f92df4e22474425c732039625c7d989 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 11 Mar 2025 12:05:15 +0100 Subject: [PATCH 36/86] deal with links joining DE and foreign countries --- .../datasets/scenario_path/scenario_path.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 410f8ab09..46ec71d0a 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1260,3 +1260,60 @@ def import_stores(scn: str): def import_foreign(scn_name, year): neighbor_reduction(scn_name, year) + + # import links joining DE and foreign countries + link_foreign = pd.read_sql( + sql=""" + SELECT * from grid.egon_etrago_link + WHERE scn_name = 'eGon100RE' + AND ((bus0 IN (SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE')) + AND (bus1 IN (SELECT bus_id FROM grid.egon_etrago_bus + WHERE country <> 'DE' + AND scn_name = 'eGon100RE'))) + OR ((bus0 IN (SELECT bus_id FROM grid.egon_etrago_bus + WHERE country <> 'DE' + AND scn_name = 'eGon100RE')) + AND (bus1 IN (SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE'))) + """, + con=con, + index_col="link_id", + ) + + bus_ch4_h2 = pd.read_sql( + sql=f""" + SELECT * from grid.egon_etrago_bus + WHERE scn_name IN ('eGon100RE', '{scn_name}') + AND carrier IN ('CH4', 'H2') + AND country <> 'DE' + """, + con=con, + index_col="bus_id", + ) + + map_scn1_to_scn2 = {} + for b, df in bus_ch4_h2.groupby(["x", "y", "carrier"]): + bus1 = df.index[df["scn_name"] == "eGon100RE"][0] + bus2 = df.index[df["scn_name"] == scn_name][0] + map_scn1_to_scn2[bus1] = bus2 + + link_foreign["bus0"] = link_foreign["bus0"].apply( + lambda x: map_scn1_to_scn2[x] if x in map_scn1_to_scn2.keys() else x + ) + + link_foreign["bus1"] = link_foreign["bus1"].apply( + lambda x: map_scn1_to_scn2[x] if x in map_scn1_to_scn2.keys() else x + ) + + link_foreign["scn_name"] = scn_name + link_foreign.reset_index(inplace=True) + link_foreign.to_sql( + name="egon_etrago_link", + con=con, + schema="grid", + if_exists="append", + index=False, + ) From e94a1433b1cc9a765ce8cd1f929dfa944a1a9a98 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Wed, 12 Mar 2025 08:29:22 +0100 Subject: [PATCH 37/86] fix nan values in build_year for H2_grid links --- src/egon/data/datasets/scenario_path/scenario_path.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 46ec71d0a..c1be72c20 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -528,6 +528,7 @@ def import_links(scn: str): h2_grid2 = scn2_link[scn2_link["carrier"] == "H2_grid"].copy() h2_grid3 = h2_grid2.copy() h2_grid3["scn_name"] = scn + h2_grid3.loc[h2_grid3["build_year"].isna(), "build_year"] = 0 h2_grid3 = h2_grid3[h2_grid3["build_year"] <= year_scenario[scn]] h2_grid3.to_sql( From 1d33150fca784f974dd271a0147d62f64ab24f6e Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Thu, 13 Mar 2025 10:08:53 +0100 Subject: [PATCH 38/86] define import_efficiency_and_costs_gen --- .../datasets/scenario_path/scenario_path.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index c1be72c20..7f0779ea9 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -5,6 +5,7 @@ import numpy as np import pandas as pd +import pypsa import geopandas as gpd from egon.data import config, db @@ -156,6 +157,54 @@ def import_network_structure(scn: str): return +def import_efficiency_and_costs_gen(scn): + target_file = ( + Path(".") + / "data_bundle_powerd_data" + / "pypsa_eur" + / "21122024_3h_clean_run" + / "results" + / "postnetworks" + / f"base_s_39_lc1.25__cb40ex0-T-H-I-B-solar+p3-dist1_{year_scenario[scn]}.nc" + ) + n = pypsa.Network(target_file) + + buses_de = n.buses[n.buses.country == "DE"] + + gen_de = n.generators[n.generators.bus.isin(buses_de.index)].copy() + gen_de.carrier.replace( + { + "onwind": "wind_onshore", + "ror": "run_of_river", + "offwind-ac": "wind_offshore", + "offwind-dc": "wind_offshore", + "offwind-float": "wind_offshore", + "urban_central_solar_thermal": "urban_central_solar_thermal_collector", + "residential_rural_solar_thermal": "residential_rural_solar_thermal_collector", + "services_rural_solar_thermal": "services_rural_solar_thermal_collector", + "solar-hsat": "solar", + }, + inplace=True, + ) + + gen_cap = pd.DataFrame( + columns=["p_nom_opt", "marginal_cost", "capital_cost", "efficiency"] + ) + for g, df in gen_de.groupby("carrier"): + gen_cap.loc[g, "p_nom_opt"] = df["p_nom_opt"].sum() + gen_cap.loc[g, "marginal_cost"] = ( + (df["p_nom_opt"] * df["marginal_cost"]) / df["p_nom_opt"].sum() + ).sum() + gen_cap.loc[g, "capital_cost"] = ( + (df["p_nom_opt"] * df["capital_cost"]) / df["p_nom_opt"].sum() + ).sum() + gen_cap.loc[g, "efficiency"] = ( + (df["p_nom_opt"] * df["efficiency"]) / df["p_nom_opt"].sum() + ).sum() + + return gen_cap + + def load_scn_capacies_link( scn1="status2019", scn2="eGon100RE", From 1df763a1bc8f02126c414784d41da52850b941dc Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Thu, 13 Mar 2025 10:13:02 +0100 Subject: [PATCH 39/86] deffine import_efficiency_and_costs_link --- .../datasets/scenario_path/scenario_path.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 7f0779ea9..3883f463d 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -205,6 +205,78 @@ def import_efficiency_and_costs_gen(scn): return gen_cap +def import_efficiency_and_costs_link(scn): + target_file = ( + Path(".") + / "data_bundle_powerd_data" + / "pypsa_eur" + / "21122024_3h_clean_run" + / "results" + / "postnetworks" + / f"base_s_39_lc1.25__cb40ex0-T-H-I-B-solar+p3-dist1_{year_scenario[scn]}.nc" + ) + n = pypsa.Network(target_file) + + buses_de = n.buses[n.buses.country == "DE"] + + link_de = n.links[ + (n.links.bus0.isin(buses_de.index)) + | (n.links.bus1.isin(buses_de.index)) + ].copy() + + link_de.carrier = link_de.carrier.str.replace(" ", "_") + + link_de.carrier.replace( + { + "H2_Electrolysis": "power_to_H2", + "H2_Fuel_Cell": "H2_to_power", + "H2_pipeline_retrofitted": "H2_retrofit", + "SMR": "CH4_to_H2", + "Sabatier": "H2_to_CH4", + "gas_for_industry": "CH4_for_industry", + "gas_pipeline": "CH4", + "urban_central_gas_boiler": "central_gas_boiler", + "urban_central_resistive_heater": "central_resistive_heater", + "urban_central_water_tanks_charger": "central_heat_store_charger", + "urban_central_water_tanks_discharger": "central_heat_store_discharger", + "rural_water_tanks_charger": "rural_heat_store_charger", + "rural_water_tanks_discharger": "rural_heat_store_discharger", + "urban_central_gas_CHP": "central_gas_CHP", + "urban_central_air_heat_pump": "central_heat_pump", + "rural_ground_heat_pump": "rural_heat_pump", + "CCGT": "OCGT", + }, + inplace=True, + ) + + link_cap = pd.DataFrame( + columns=[ + "p_nom_opt", + "marginal_cost", + "capital_cost", + "efficiency", + "efficiency2", + ] + ) + + for l, df in link_de.groupby("carrier"): + link_cap.loc[l, "p_nom_opt"] = df["p_nom_opt"].sum() + link_cap.loc[l, "marginal_cost"] = ( + (df["p_nom_opt"] * df["marginal_cost"]) / df["p_nom_opt"].sum() + ).sum() + link_cap.loc[l, "capital_cost"] = ( + (df["p_nom_opt"] * df["capital_cost"]) / df["p_nom_opt"].sum() + ).sum() + link_cap.loc[l, "efficiency"] = ( + (df["p_nom_opt"] * df["efficiency"]) / df["p_nom_opt"].sum() + ).sum() + link_cap.loc[l, "efficiency2"] = ( + (df["p_nom_opt"] * df["efficiency2"]) / df["p_nom_opt"].sum() + ).sum() + + return link_cap + + def load_scn_capacies_link( scn1="status2019", scn2="eGon100RE", From bb98919d1112504d3e942f8728bce1bc256ca804 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Thu, 13 Mar 2025 10:50:36 +0100 Subject: [PATCH 40/86] unify functions to read efficiency and costs --- .../datasets/scenario_path/scenario_path.py | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 3883f463d..41de98f68 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -157,7 +157,7 @@ def import_network_structure(scn: str): return -def import_efficiency_and_costs_gen(scn): +def import_efficiency_and_costs(scn): target_file = ( Path(".") / "data_bundle_powerd_data" @@ -172,6 +172,7 @@ def import_efficiency_and_costs_gen(scn): buses_de = n.buses[n.buses.country == "DE"] gen_de = n.generators[n.generators.bus.isin(buses_de.index)].copy() + gen_de.carrier = gen_de.carrier.str.replace(" ", "_") gen_de.carrier.replace( { "onwind": "wind_onshore", @@ -183,6 +184,7 @@ def import_efficiency_and_costs_gen(scn): "residential_rural_solar_thermal": "residential_rural_solar_thermal_collector", "services_rural_solar_thermal": "services_rural_solar_thermal_collector", "solar-hsat": "solar", + "urban_central_geo_thermal": "geo_thermal", }, inplace=True, ) @@ -202,23 +204,6 @@ def import_efficiency_and_costs_gen(scn): (df["p_nom_opt"] * df["efficiency"]) / df["p_nom_opt"].sum() ).sum() - return gen_cap - - -def import_efficiency_and_costs_link(scn): - target_file = ( - Path(".") - / "data_bundle_powerd_data" - / "pypsa_eur" - / "21122024_3h_clean_run" - / "results" - / "postnetworks" - / f"base_s_39_lc1.25__cb40ex0-T-H-I-B-solar+p3-dist1_{year_scenario[scn]}.nc" - ) - n = pypsa.Network(target_file) - - buses_de = n.buses[n.buses.country == "DE"] - link_de = n.links[ (n.links.bus0.isin(buses_de.index)) | (n.links.bus1.isin(buses_de.index)) @@ -274,7 +259,7 @@ def import_efficiency_and_costs_link(scn): (df["p_nom_opt"] * df["efficiency2"]) / df["p_nom_opt"].sum() ).sum() - return link_cap + return pd.concat([gen_cap, link_cap]) def load_scn_capacies_link( From 888668ba17de6305dd0d906397aaa4f71a2f943a Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 14 Mar 2025 08:58:09 +0100 Subject: [PATCH 41/86] include other fuel costs for scn_path interpolation --- .../scenario_parameters/parameters.py | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/src/egon/data/datasets/scenario_parameters/parameters.py b/src/egon/data/datasets/scenario_parameters/parameters.py index d365ad7c5..22ffc839f 100755 --- a/src/egon/data/datasets/scenario_parameters/parameters.py +++ b/src/egon/data/datasets/scenario_parameters/parameters.py @@ -99,14 +99,15 @@ def global_settings(scenario): parameters = { "weather_year": 2011, "population_year": 2050, - "fuel_costs": { # Netzentwicklungsplan Strom 2035, Version 2021, 1. Entwurf, p. 39, table 6 - "oil": 73.8, # [EUR/MWh] - "gas": 25.6, # [EUR/MWh] - "coal": 20.2, # [EUR/MWh] - "lignite": 4.0, # [EUR/MWh] + "fuel_costs": { # Netzentwicklungsplan Strom 2035, Version 2021, 2. Entwurf, p. 47, table 7 https://www.netzentwicklungsplan.de/sites/default/files/2023-03/NEP_2037_2045_V2023_1_Entwurf_Kap2.pdf + "oil": 33.7, # [EUR/MWh] + "gas": 19.4, # [EUR/MWh] + "coal": 6.7, # [EUR/MWh] + "lignite": 6.5, # [EUR/MWh] "nuclear": 1.7, # [EUR/MWh] + "biomass": 40, # Dummyvalue, ToDo: Find a suitable source }, - "co2_costs": 76.5, # [EUR/t_CO2] + "co2_costs": 100, # [EUR/t_CO2] "co2_emissions": { # Netzentwicklungsplan Strom 2035, Version 2021, 1. Entwurf, p. 40, table 8 "waste": 0.165, # [t_CO2/MW_th] "lignite": 0.393, # [t_CO2/MW_th] @@ -492,6 +493,37 @@ def electricity(scenario): # Insert marginal_costs in EUR/MWh # marginal cost can include fuel, C02 and operation and maintenance costs parameters["marginal_cost"] = { + "oil": global_settings(scenario)["fuel_costs"]["oil"] + / read_costs(costs, "oil", "efficiency") + + read_costs(costs, "oil", "VOM") + + global_settings(scenario)["co2_costs"] + * global_settings(scenario)["co2_emissions"]["oil"] + / read_costs(costs, "oil", "efficiency"), + "other_non_renewable": global_settings(scenario)["fuel_costs"][ + "gas" + ] / read_costs(costs, "OCGT", "efficiency") + + global_settings(scenario)["co2_costs"] + * global_settings(scenario)["co2_emissions"][ + "other_non_renewable" + ] / read_costs(costs, "OCGT", "efficiency"), + "lignite": global_settings(scenario)["fuel_costs"]["lignite"] + / read_costs(costs, "lignite", "efficiency") + + read_costs(costs, "lignite", "VOM") + + global_settings(scenario)["co2_costs"] + * global_settings(scenario)["co2_emissions"]["lignite"] + / read_costs(costs, "lignite", "efficiency"), + "coal": global_settings(scenario)["fuel_costs"]["coal"] + / read_costs(costs, "coal", "efficiency") + + read_costs(costs, "coal", "VOM") + + global_settings(scenario)["co2_costs"] + * global_settings(scenario)["co2_emissions"]["coal"] + / read_costs(costs, "coal", "efficiency"), + "nuclear": global_settings(scenario)["fuel_costs"]["nuclear"] + / read_costs(costs, "nuclear", "efficiency") + + read_costs(costs, "nuclear", "VOM"), + "biomass": global_settings(scenario)["fuel_costs"]["biomass"] + / read_costs(costs, "biomass", "efficiency") + + read_costs(costs, "biomass CHP", "VOM"), "wind_offshore": read_costs(costs, "offwind", "VOM"), "wind_onshore": read_costs(costs, "onwind", "VOM"), "solar": read_costs(costs, "solar", "VOM"), From bfd0c69ae85b5739d037addea34941efa72407f5 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 14 Mar 2025 09:21:02 +0100 Subject: [PATCH 42/86] interpolate marginal costs generators --- .../datasets/scenario_path/scenario_path.py | 83 ++++++++++++++----- 1 file changed, 63 insertions(+), 20 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 41de98f68..5f0e428ac 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -12,15 +12,16 @@ import egon.data.config from egon.data.datasets.pypsaeur import neighbor_reduction +from egon.data.datasets.scenario_parameters import get_sector_parameters sources = egon.data.config.datasets()["scenario_path"]["sources"] con = db.engine() scaling_factor = { - "powerd2025": (2025 - 2019) / (2040 - 2019), - "powerd2030": (2030 - 2019) / (2040 - 2019), - "powerd2035": (2035 - 2019) / (2040 - 2019), + "powerd2025": (2025 - 2019) / (2045 - 2019), + "powerd2030": (2030 - 2019) / (2045 - 2019), + "powerd2035": (2035 - 2019) / (2045 - 2019), } year_scenario = { @@ -784,9 +785,31 @@ def load_scn_capacies_gen( return gen_capacities -def import_generators(scn: str): +def interpolate_marginal_costs(scn): + marg_cost1 = get_sector_parameters( + sector="electricity", scenario="status2019" + )["marginal_cost"] + marg_cost2 = get_sector_parameters( + sector="electricity", scenario="eGon100RE" + )["marginal_cost"] + marg_cost3 = {} + for f in marg_cost2.keys(): + marg_cost3[f] = ( + marg_cost1[f] + + (marg_cost2[f] - marg_cost1[f]) * scaling_factor[scn] + ) + + marg_cost3["run_of_river"] = 0 + marg_cost3["solar_rooftop"] = 0 + + return marg_cost3 + +def import_generators(scn: str): + scn = "powerd2025" cap_gen = load_scn_capacies_gen() + eff_and_costs = import_efficiency_and_costs(scn) + marg_cost3 = interpolate_marginal_costs(scn) scn1_gen = pd.read_sql( """ @@ -817,6 +840,9 @@ def import_generators(scn: str): # Dealing with geo_thermal geo3 = scn2_gen[scn2_gen["carrier"] == "geo_thermal"].copy() geo3["scn_name"] = scn + geo3["marginal_cost"] = eff_and_costs.loc["geo_thermal", "marginal_cost"] + geo3["capital_cost"] = eff_and_costs.loc["geo_thermal", "capital_cost"] + geo3["efficiency"] = eff_and_costs.loc["geo_thermal", "efficiency"] objective = cap_gen.at["geo_thermal", scn] geo3["p_nom"] *= objective / geo3["p_nom"].sum() @@ -830,7 +856,6 @@ def import_generators(scn: str): ) # Dealing with oil, coal and lignite - fossil_carriers = [ "oil", "coal", @@ -841,24 +866,32 @@ def import_generators(scn: str): ~cap_gen.loc[fossil_carriers, scn].isna() ].index.values - if len(fossil_carriers) == 0: - return + if len(fossil_carriers) > 0: + fossil3 = scn1_gen[scn1_gen["carrier"].isin(fossil_carriers)].copy() + fossil3["scn_name"] = scn - fossil3 = scn1_gen[scn1_gen["carrier"].isin(fossil_carriers)].copy() - fossil3["scn_name"] = scn + for fc in fossil_carriers: + fossil3.loc[fossil3["carrier"] == fc, "marginal_cost"] = ( + marg_cost3[fc] + ) + fossil3.loc[fossil3["carrier"] == fc, "capital_cost"] = ( + eff_and_costs.loc[fc, "capital_cost"] + ) - for c, df in fossil3.groupby("carrier"): - id = df.index - objective = cap_gen.at[c, scn] - fossil3.loc[id, "p_nom"] *= objective / fossil3.loc[id, "p_nom"].sum() + for c, df in fossil3.groupby("carrier"): + id = df.index + objective = cap_gen.at[c, scn] + fossil3.loc[id, "p_nom"] *= ( + objective / fossil3.loc[id, "p_nom"].sum() + ) - fossil3.to_sql( - name="egon_etrago_gen", - con=con, - schema="grid", - if_exists="append", - index=False, - ) + fossil3.to_sql( + name="egon_etrago_gen", + con=con, + schema="grid", + if_exists="append", + index=False, + ) # Dialing with run_of_river, solar_rooftop, wind_offshore, wind_onshore # and solar @@ -907,6 +940,10 @@ def import_generators(scn: str): for c, df in gen_var3.groupby("carrier"): factor_to_pypsaeur = cap_gen.at[c, scn] / df["p_nom"].sum() gen_var3.loc[df.index, "p_nom"] *= factor_to_pypsaeur + gen_var3.loc[df.index, "marginal_cost"] = marg_cost3[c] + gen_var3.loc[df.index, "capital_cost"] = eff_and_costs.loc[ + c, "capital_cost" + ] gen_var3.reset_index(inplace=True) gen_var3.to_sql( @@ -978,6 +1015,12 @@ def import_generators(scn: str): ) gas3["p_nom"] *= obj / gas3["p_nom"].sum() + gas3["marginal_cost"] = ( + gas1["marginal_cost"].mean() + + (gas2["marginal_cost"].mean() - gas1["marginal_cost"].mean()) + * scaling_factor[scn] + ) + gas3.to_sql( name="egon_etrago_generator", con=con, From 14f0bc6bd22edf2b526eef1738cef29d0a9590e1 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 14 Mar 2025 09:41:31 +0100 Subject: [PATCH 43/86] =?UTF-8?q?interpolate=20marginal=20costs=20links?= =?UTF-8?q?=C3=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasets/scenario_path/scenario_path.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 5f0e428ac..4b475c10b 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -520,6 +520,15 @@ def import_links(scn: str): link_cgb3 = link_cgb2.copy() link_cgb3["scn_name"] = scn + cgb_marg_cost = ( + link_cgb1["marginal_cost"].mean() + + ( + link_cgb2["marginal_cost"].mean() + - link_cgb1["marginal_cost"].mean() + ) + * scaling_factor[scn] + ) + cgb1_geo = gpd.read_postgis( """ SELECT bus_id, geom FROM grid.egon_etrago_bus @@ -566,6 +575,7 @@ def import_links(scn: str): link_cgb3["p_nom"] = link_cgb3["p_nom"] * factor_to_pypsaeur link_cgb3.reset_index(inplace=True) + link_cgb3["marginal_cost"] = cgb_marg_cost link_cgb3.to_sql( name="egon_etrago_link", con=con, @@ -583,6 +593,14 @@ def import_links(scn: str): ) link_ocgt3 = link_ocgt2.copy().reset_index().set_index("link_id") link_ocgt3["scn_name"] = scn + ocgt_marg_cost = ( + link_ocgt1["marginal_cost"].mean() + + ( + link_ocgt2["marginal_cost"].mean() + - link_ocgt1["marginal_cost"].mean() + ) + * scaling_factor[scn] + ) not_in_ocgt2 = link_ocgt1[ ~link_ocgt1.index.isin(link_ocgt2.index.unique()) @@ -623,6 +641,7 @@ def import_links(scn: str): link_ocgt3["p_nom"] *= factor_to_pypsaeur link_ocgt3.reset_index(inplace=True) + link_ocgt3["marginal_cost"] = ocgt_marg_cost link_ocgt3.to_sql( name="egon_etrago_link", con=con, @@ -806,7 +825,7 @@ def interpolate_marginal_costs(scn): def import_generators(scn: str): - scn = "powerd2025" + cap_gen = load_scn_capacies_gen() eff_and_costs = import_efficiency_and_costs(scn) marg_cost3 = interpolate_marginal_costs(scn) From 46920e5b37bddc37f7c2de0093cb4fed15fc03f2 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 17 Mar 2025 09:33:38 +0100 Subject: [PATCH 44/86] convert isolated H2_grid into H2 buses --- .../datasets/scenario_path/scenario_path.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 4b475c10b..9c892b32d 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -7,6 +7,7 @@ import pandas as pd import pypsa import geopandas as gpd +from shapely.geometry import Point, LineString from egon.data import config, db import egon.data.config @@ -715,6 +716,95 @@ def import_links(scn: str): index=False, ) + # Convert isolated H2_grid buses to H2. Include links CH4_to_H2. + h2_grid_b = gpd.read_postgis( + f""" + SELECT * FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = 'H2_grid' + and country = 'DE' + """, + con, + geom_col="geom", + ).set_index("bus_id") + + h2_grid_to_h2 = h2_grid_b[ + ~h2_grid_b.index.isin( + pd.concat([h2_grid3["bus0"], h2_grid3["bus1"]], ignore_index=True) + ) + ].copy() + + h2_grid_to_h2["carrier"] = "H2" + h2_grid_to_h2["scn_name"] = scn + + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND bus_id IN {tuple(h2_grid_to_h2.index)} + """ + ) + + ch4_b = gpd.read_postgis( + f""" + SELECT * FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = 'CH4' + and country = 'DE' + """, + con, + geom_col="geom", + ).set_index("bus_id") + + closest_ch4 = {} + for b in h2_grid_to_h2.index: + dist = ch4_b.distance(h2_grid_to_h2["geom"][b]) + closest_ch4[b] = dist.idxmin() + + refference = identical3[identical3["carrier"] == "CH4_to_H2"].head(1) + + new_ch4_to_h2 = gpd.GeoDataFrame(columns=identical3.columns) + for bus in h2_grid_to_h2.index: + new_ch4_to_h2.loc[bus, :] = refference.values + new_ch4_to_h2.loc[bus, "bus0"] = closest_ch4[bus] + new_ch4_to_h2.loc[bus, "bus1"] = bus + new_ch4_to_h2.loc[bus, "geom"] = LineString( + [ch4_b.at[closest_ch4[bus], "geom"], h2_grid_to_h2.at[bus, "geom"]] + ) + new_ch4_to_h2.set_geometry("geom", inplace=True) + new_ch4_to_h2.set_crs(crs=4326, inplace=True) + + next_link_id = ( + pd.read_sql( + f""" + SELECT MAX(link_id) FROM grid.egon_etrago_link + """, + con, + ).iat[0, 0] + + 1 + ) + + new_ch4_to_h2["link_id"] = range( + next_link_id, next_link_id + len(new_ch4_to_h2) + ) + new_ch4_to_h2["build_year"] = new_ch4_to_h2["build_year"].apply(int) + + new_ch4_to_h2.to_postgis( + name="egon_etrago_link", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + h2_grid_to_h2.reset_index(inplace=True) + h2_grid_to_h2.to_postgis( + name="egon_etrago_bus", + con=con, + schema="grid", + if_exists="append", + index=False, + ) return From 8e34a32be2cfcafcf980b0a44e0453cfb1bb387a Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 17 Mar 2025 11:56:19 +0100 Subject: [PATCH 45/86] dealing with oil and biomass rural_boilers --- .../datasets/scenario_path/scenario_path.py | 82 ++++++++++++++++++- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 9c892b32d..82957049d 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -332,10 +332,8 @@ def load_scn_capacies_link( "central_heat_pump", "central_resistive_heater", "gas", - "rural_biomass_boiler", "rural_gas_boiler", "rural_heat_pump", - "rural_oil_boiler", "rural_resistive_heater", ] @@ -873,6 +871,8 @@ def load_scn_capacies_gen( "rural_solar_thermal", "urban_central_gas_CHP", "urban_central_solid_biomass_CHP", + "rural_biomass_boiler", + "rural_oil_boiler", ] carriers_gen = set( @@ -915,7 +915,7 @@ def interpolate_marginal_costs(scn): def import_generators(scn: str): - + scn = "powerd2025" cap_gen = load_scn_capacies_gen() eff_and_costs = import_efficiency_and_costs(scn) marg_cost3 = interpolate_marginal_costs(scn) @@ -1137,6 +1137,82 @@ def import_generators(scn: str): if_exists="append", index=False, ) + + # dealing with rural_biomass_boiler and rural_oil_boiler + load_rh = pd.read_sql( + f""" + SELECT * FROM grid.egon_etrago_load + WHERE scn_name = '{scn}' + AND carrier = 'rural_heat' + AND bus in (SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = '{scn}') + """, + con, + index_col="load_id", + ) + + refference_oil = scn1_gen[scn1_gen["carrier"] == "oil"].head(1) + refference_oil["scn_name"] = scn + refference_oil["carrier"] = "rural_biomass_boiler" + refference_oil["marginal_cost"] = marg_cost3["oil"] + refference_biomass = scn1_gen[scn1_gen["carrier"] == "oil"].head(1) + refference_biomass["scn_name"] = scn + refference_biomass["carrier"] = "rural_oil_boiler" + refference_biomass["marginal_cost"] = marg_cost3["biomass"] + + load_t_rh = pd.read_sql( + f""" + SELECT * FROM grid.egon_etrago_load_timeseries + WHERE load_id IN( + SELECT load_id FROM grid.egon_etrago_load + WHERE bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = '{scn}' + ) + AND scn_name = '{scn}' + ) + AND load_id IN (SELECT load_id FROM grid.egon_etrago_load + WHERE carrier = 'rural_heat' + AND scn_name = '{scn}') + AND scn_name = '{scn}' + """, + con, + index_col="load_id", + ) + + new_gen_boiler = pd.DataFrame(columns=scn2_gen.columns) + for load, df in load_t_rh.iterrows(): + bus = int(load_rh.at[load, "bus"]) + inst_capacity = float(np.array(df.at["p_set"]).max()) + new_gen_boiler.loc[str(bus) + "bio", :] = refference_biomass.values + new_gen_boiler.loc[str(bus) + "bio", "bus"] = bus + new_gen_boiler.loc[str(bus) + "bio", "p_nom"] = inst_capacity + new_gen_boiler.loc[str(bus) + "oil", :] = refference_oil.values + new_gen_boiler.loc[str(bus) + "oil", "bus"] = bus + new_gen_boiler.loc[str(bus) + "oil", "p_nom"] = inst_capacity + + next_gen_id = ( + pd.read_sql( + """ + SELECT MAX(generator_id) FROM grid.egon_etrago_generator + """, + con, + ).iat[0, 0] + + 1 + ) + + new_gen_boiler["generator_id"] = range( + next_gen_id, next_gen_id + len(new_gen_boiler) + ) + new_gen_boiler.to_sql( + name="egon_etrago_generator", + con=con, + schema="grid", + if_exists="append", + index=False, + ) return From 163c234373ce785a70bc19741a66bdfafc624e4b Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 17 Mar 2025 13:15:08 +0100 Subject: [PATCH 46/86] dealing with chp biomass --- .../datasets/scenario_path/scenario_path.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 82957049d..354dc3a46 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -851,6 +851,7 @@ def load_scn_capacies_gen( "urban_central_heat_pump": "central_heat_pump", "urban_central_resistive_heater": "central_resistive_heater", "gas": "OCGT", + "urban_central_solid_biomass_CHP":"central_biomass_CHP" } scn_capacities["carrier"] = scn_capacities["carrier"].apply( @@ -1213,6 +1214,79 @@ def import_generators(scn: str): if_exists="append", index=False, ) + + # deal with urban_central_biomass + ucbe3 = scn1_gen[scn1_gen["carrier"] == "central_biomass_CHP"].copy() + ucbe3["scn_name"] = scn + + obj = cap_gen.at["central_biomass_CHP", scn] * 0.3 + ucbe3["p_nom"] *= obj / ucbe3["p_nom"].sum() + + ucbe3.to_sql( + name="egon_etrago_generator", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + # deal with urban_central_biomass_heat + ucbh3 = scn1_gen[scn1_gen["carrier"] == "central_biomass_CHP_heat"].copy() + ucbh3["scn_name"] = scn + + obj = cap_gen.at["central_biomass_CHP", scn] * 0.7 + ucbh3["p_nom"] *= obj / ucbe3["p_nom"].sum() + + chb1_geo = gpd.read_postgis( + """ + SELECT bus_id, geom FROM grid.egon_etrago_bus + WHERE scn_name = 'status2019' + AND carrier = 'central_heat' + """, + con, + geom_col="geom", + ).set_index("bus_id") + + chb2_geo = gpd.read_postgis( + f""" + SELECT bus_id, geom FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = 'central_heat' + """, + con, + geom_col="geom", + ).set_index("bus_id") + + chb1_to_chb2 = {} + for l in chb1_geo.index: + dist = chb2_geo.distance(chb1_geo["geom"][l]) + dist.sort_values(inplace=True) + chb1_to_chb2[l] = dist.index[0] + + ucbh3["bus"] = ucbh3["bus"].map(chb1_to_chb2) + + next_gen_id = ( + pd.read_sql( + """ + SELECT MAX(generator_id) FROM grid.egon_etrago_generator + """, + con, + ).iat[0, 0] + + 1 + ) + + ucbh3["generator_id"] = range( + next_gen_id, next_gen_id + len(ucbh3) + ) + + ucbh3.to_sql( + name="egon_etrago_generator", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + return From fc84795a1b168512249a8d5a0f42cc29d49c71d9 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Mon, 17 Mar 2025 16:44:00 +0100 Subject: [PATCH 47/86] delete fixed scenario name --- src/egon/data/datasets/scenario_path/scenario_path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 354dc3a46..d4884525a 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -916,7 +916,7 @@ def interpolate_marginal_costs(scn): def import_generators(scn: str): - scn = "powerd2025" + cap_gen = load_scn_capacies_gen() eff_and_costs = import_efficiency_and_costs(scn) marg_cost3 = interpolate_marginal_costs(scn) From 626eb25ea7d86dd6f9767273459a2776ec657599 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 18 Mar 2025 07:13:25 +0100 Subject: [PATCH 48/86] fixes related to fossil generators --- src/egon/data/datasets/scenario_path/__init__.py | 6 +++--- src/egon/data/datasets/scenario_path/scenario_path.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/__init__.py b/src/egon/data/datasets/scenario_path/__init__.py index 046668114..877002a06 100644 --- a/src/egon/data/datasets/scenario_path/__init__.py +++ b/src/egon/data/datasets/scenario_path/__init__.py @@ -14,8 +14,8 @@ def create_powerd2025(): scn = "powerd2025" import_network_structure(scn) - import_generators(scn) import_loads(scn) + import_generators(scn) import_links(scn) import_storage_units(scn) import_stores(scn) @@ -27,8 +27,8 @@ def create_powerd2025(): def create_powerd2030(): scn = "powerd2030" import_network_structure(scn) - import_generators(scn) import_loads(scn) + import_generators(scn) import_links(scn) import_storage_units(scn) import_stores(scn) @@ -40,8 +40,8 @@ def create_powerd2030(): def create_powerd2035(): scn = "powerd2035" import_network_structure(scn) - import_generators(scn) import_loads(scn) + import_generators(scn) import_links(scn) import_storage_units(scn) import_stores(scn) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index d4884525a..7962bd4b7 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -429,6 +429,7 @@ def import_links(scn: str): AND scn_name = 'eGon100RE' ) AND carrier = 'dsm') + AND scn_name = 'eGon100RE' """, con, ) @@ -491,6 +492,7 @@ def import_links(scn: str): AND scn_name = 'eGon100RE' ) AND carrier = 'rural_heat_pump') + AND scn_name = 'eGon100RE' """, con, ) @@ -996,7 +998,7 @@ def import_generators(scn: str): ) fossil3.to_sql( - name="egon_etrago_gen", + name="egon_etrago_generator", con=con, schema="grid", if_exists="append", From d3fefae0d684f49a8e30b25e2efd7463beb3a5ea Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 18 Mar 2025 09:08:04 +0100 Subject: [PATCH 49/86] fix mixed biomass and oil boilers --- src/egon/data/datasets/scenario_path/scenario_path.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 7962bd4b7..69eb2cc21 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1157,11 +1157,11 @@ def import_generators(scn: str): refference_oil = scn1_gen[scn1_gen["carrier"] == "oil"].head(1) refference_oil["scn_name"] = scn - refference_oil["carrier"] = "rural_biomass_boiler" + refference_oil["carrier"] = "rural_oil_boiler" refference_oil["marginal_cost"] = marg_cost3["oil"] refference_biomass = scn1_gen[scn1_gen["carrier"] == "oil"].head(1) refference_biomass["scn_name"] = scn - refference_biomass["carrier"] = "rural_oil_boiler" + refference_biomass["carrier"] = "rural_biomass_boiler" refference_biomass["marginal_cost"] = marg_cost3["biomass"] load_t_rh = pd.read_sql( From 7f8b6593a89f5a2b13ced498fc2a1e2f9d9ff846 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 18 Mar 2025 09:29:46 +0100 Subject: [PATCH 50/86] Add biogas generators in Germany --- .../datasets/scenario_path/scenario_path.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 69eb2cc21..5931fdd7a 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1141,6 +1141,34 @@ def import_generators(scn: str): index=False, ) + # Dealing with biogas in Germany + biogas1 = scn1_gen[scn1_gen["carrier"] == "CH4"].copy() + biogas2 = scn2_gen[scn2_gen["carrier"] == "CH4"].copy() + biogas3 = scn2_gen[scn2_gen["carrier"] == "CH4"].copy() + biogas3["scn_name"] = scn + biogas3["carrier"] = "biogas" + + obj = ( + biogas1["p_nom"].sum() + + (biogas2["p_nom"].sum() - biogas1["p_nom"].sum()) * scaling_factor[scn] + ) + biogas3["p_nom"] *= obj / biogas3["p_nom"].sum() + + # Interpolate costs based on gas fuel costs from status2019 and eGon100RE + # egon.data.scenario_parameters + biogas3["marginal_cost"] = ( + 20.16 + (19.4 - 20.16) + * scaling_factor[scn] + ) + + biogas3.to_sql( + name="egon_etrago_generator", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + # dealing with rural_biomass_boiler and rural_oil_boiler load_rh = pd.read_sql( f""" From ef88cc617228740717b04d206022b3dcb390d94c Mon Sep 17 00:00:00 2001 From: Lennart Zimmermann Date: Tue, 18 Mar 2025 13:42:13 +0100 Subject: [PATCH 51/86] Implementing missing rural heat loads for powerd2025 --- src/egon/data/datasets/pypsaeur/__init__.py | 52 +++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/egon/data/datasets/pypsaeur/__init__.py b/src/egon/data/datasets/pypsaeur/__init__.py index 43b85692a..8c1680362 100755 --- a/src/egon/data/datasets/pypsaeur/__init__.py +++ b/src/egon/data/datasets/pypsaeur/__init__.py @@ -575,6 +575,53 @@ def combine_decentral_and_rural_heat(network_solved, network_prepared): return network_prepared, network_solved + +def combine_residenial_services_heat_load(network_prepared): + + residential_rural_loads = network_prepared.loads[ + network_prepared.loads.carrier.str.contains("residential rural heat") + ] + + for i, row in residential_rural_loads.iterrows(): + if i in network_prepared.loads_t.p_set.columns: + network_prepared.loads_t.p_set[ + i.replace("residential", "services") + ] += network_prepared.loads_t.p_set[i] + network_prepared.mremove("Load", residential_rural_loads.index) + + + target_carriers = ["services rural heat"] + + rural_heat_loads = network_prepared.loads[ + network_prepared.loads.carrier.isin(target_carriers) + ] + + #rename carrier + network_prepared.loads.loc[rural_heat_loads.index, "carrier"] = "rural heat" + + #rename bus + network_prepared.loads.loc[rural_heat_loads.index, "bus"] = ( + network_prepared.loads.loc[rural_heat_loads.index, "bus"] + .str.replace("services rural heat", "rural heat") + .str.replace("residential rural heat", "rural heat") + ) + + #rename index + network_prepared.loads.rename( + index=lambda x: x.replace("services rural heat", "rural heat") + .replace("residential rural heat", "rural heat"), + inplace=True + ) + + #rename timeseries columns + network_prepared.loads_t.p_set.columns = ( + network_prepared.loads_t.p_set.columns.str.replace("services rural heat", "rural heat") + .str.replace("residential rural heat", "rural heat") + ) + + return network_prepared + + def neighbor_reduction(scn_name, year=2045): network_solved = read_network(year=year) network_prepared = prepared_network(year=year) @@ -909,6 +956,11 @@ def neighbor_reduction(scn_name, year=2045): # loads # imported from prenetwork in 1h-resolution + + #adjusting index-, bus- and carrier-name for loads for powerd2025 + if scn_name=='powerd2025': + network_prepared = combine_residenial_services_heat_load(network_prepared) + neighbor_loads = network_prepared.loads[ network_prepared.loads.bus.isin(neighbors.index) ] From 70ff714e0a2d2647948c5dad3d443d0f96cf225c Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 18 Mar 2025 15:28:54 +0100 Subject: [PATCH 52/86] Include central heat store charger in discharger --- .../datasets/scenario_path/scenario_path.py | 82 +++++++++++++------ 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 5931fdd7a..121033820 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -12,7 +12,7 @@ from egon.data import config, db import egon.data.config -from egon.data.datasets.pypsaeur import neighbor_reduction +from egon.data.datasets.pypsaeur import neighbor_reduction, prepared_network from egon.data.datasets.scenario_parameters import get_sector_parameters sources = egon.data.config.datasets()["scenario_path"]["sources"] @@ -678,8 +678,8 @@ def import_links(scn: str): "rural_heat_store_charger", "rural_heat_store_discharger", "power_to_H2", - "rural_heat_store_charger", - "rural_heat_store_discharger", + "central_heat_store_charger", + "central_heat_store_discharger", "BEV_charger", ] identical3 = scn2_link[scn2_link["carrier"].isin(identical)].copy() @@ -1386,6 +1386,19 @@ def import_loads(scn: str): index_col="load_id", ) + pre_network = prepared_network(year=scn.replace("powerd", "")) + + annual_sum_gas = pre_network.loads.loc["DE0 0 gas for industry", "p_set"] * 8760 + annual_sum_h2 = pre_network.loads.loc["DE0 0 H2 for industry", "p_set"] * 8760 + annual_sum_powerd_gas = 0 + for gas_load in scn2_load[scn2_load.carrier=="CH4_for_industry"].index: + annual_sum_powerd_gas += sum(scn2_load_t.p_set[gas_load]) + annual_sum_powerd_h2 = 0 + for h2_load in scn2_load[scn2_load.carrier=="H2_for_industry"].index: + annual_sum_powerd_h2 += sum(scn2_load_t.p_set[h2_load]) + + factor_h2 = annual_sum_h2 / annual_sum_powerd_h2 + factor_gas = annual_sum_gas / annual_sum_powerd_gas # dealing with loads only present in eGon100RE only100_carrier = [ "O2", @@ -1402,9 +1415,18 @@ def import_loads(scn: str): print(f"{c}:{a.sum()}") for c, df in only100.groupby("carrier"): - scn2_load_t.loc[df.index, "p_set"] = scn2_load_t.loc[ - df.index, "p_set" - ].apply(lambda x: np.array(x) * scaling_factor[scn]) + if c == "H2_for_industry": + scn2_load_t.loc[df.index, "p_set"] = scn2_load_t.loc[ + df.index, "p_set" + ].apply(lambda x: np.array(x)* factor_h2) + elif c == "CH4_for_industry": + scn2_load_t.loc[df.index, "p_set"] = scn2_load_t.loc[ + df.index, "p_set" + ].apply(lambda x: np.array(x) * factor_gas) + else: + scn2_load_t.loc[df.index, "p_set"] = scn2_load_t.loc[ + df.index, "p_set" + ].apply(lambda x: np.array(x) * scaling_factor[scn]) for c, df in only100.groupby("carrier"): a = scn2_load_t.loc[df.index, "p_set"].apply( @@ -1413,20 +1435,15 @@ def import_loads(scn: str): print(f"{c}:{a.sum()}") # Dealing with land_transport_EV loads - evl1 = scn1_load[scn1_load["carrier"] == "land_transport_EV"].copy() evl2 = scn2_load[scn2_load["carrier"] == "land_transport_EV"].copy() - evl1_total = ( - scn1_load_t.loc[evl1.index, "p_set"] - .apply(lambda x: np.array(x).sum()) - .sum() - ) + evl2_total = ( scn2_load_t.loc[evl2.index, "p_set"] .apply(lambda x: np.array(x).sum()) .sum() ) - objective = evl1_total + (evl2_total - evl1_total) * scaling_factor[scn] + objective = pre_network.loads_t.p_set["DE0 0 land transport EV"].sum() scn2_load_t.loc[evl2.index, "p_set"] = scn2_load_t.loc[ evl2.index, "p_set" @@ -1440,6 +1457,10 @@ def import_loads(scn: str): ] ).copy() + total_ac_pypsaeur = ( + pre_network.loads_t.p_set["DE0 0"].sum() + + pre_network.loads_t.p_set["DE0 0 industry electricity"].sum()) + for b, df in ac_load.groupby("bus"): df1 = df[df["scn_name"] == "status2019"] df2 = df[df["scn_name"] == "eGon100RE"] @@ -1458,41 +1479,48 @@ def import_loads(scn: str): df2.index, "p_set" ].apply(lambda x: np.array(x) * objective / df2_total) + total_ac_scn2 = scn2_load_t.loc[ + scn2_load[scn2_load["carrier"] == "AC"].index, "p_set" + ].apply(lambda x: np.array(x).sum()).sum() + + scn2_load_t.loc[ scn2_load[scn2_load["carrier"] == "AC"].index, "p_set"] = scn2_load_t.loc[ + scn2_load[scn2_load["carrier"] == "AC"].index, "p_set" + ].apply(lambda x: np.array(x) * total_ac_pypsaeur / total_ac_scn2) + # Dealing with rural_heat loads - rh1 = scn1_load[scn1_load["carrier"] == "rural_heat"].copy() rh2 = scn2_load[scn2_load["carrier"] == "rural_heat"].copy() - rh1_total = ( - scn1_load_t.loc[rh1.index, "p_set"] - .apply(lambda x: np.array(x).sum()) - .sum() - ) + rh2_total = ( scn2_load_t.loc[rh2.index, "p_set"] .apply(lambda x: np.array(x).sum()) .sum() ) - objective = rh1_total + (rh2_total - rh1_total) * scaling_factor[scn] + if scn == "powerd2025": + objective = ( + pre_network.loads_t.p_set["DE0 0 residential rural heat"].sum() + + pre_network.loads_t.p_set["DE0 0 services rural heat"].sum() + + pre_network.loads_t.p_set["DE0 0 residential urban decentral heat"].sum() + + pre_network.loads_t.p_set["DE0 0 services urban decentral heat"].sum() + ) + else: + objective = (pre_network.loads_t.p_set["DE0 0 rural heat"].sum() + +pre_network.loads_t.p_set["DE0 0 urban decentral heat"].sum()) scn2_load_t.loc[rh2.index, "p_set"] = scn2_load_t.loc[ rh2.index, "p_set" ].apply(lambda x: np.array(x) * objective / rh2_total) # Dealing with central_heat loads - ch1 = scn1_load[scn1_load["carrier"] == "central_heat"].copy() ch2 = scn2_load[scn2_load["carrier"] == "central_heat"].copy() - ch1_total = ( - scn1_load_t.loc[ch1.index, "p_set"] - .apply(lambda x: np.array(x).sum()) - .sum() - ) + ch2_total = ( scn2_load_t.loc[ch2.index, "p_set"] .apply(lambda x: np.array(x).sum()) .sum() ) - objective = ch1_total + (ch2_total - ch1_total) * scaling_factor[scn] + objective = pre_network.loads_t.p_set["DE0 0 urban central heat"].sum() scn2_load_t.loc[ch2.index, "p_set"] = scn2_load_t.loc[ ch2.index, "p_set" From 2fcd7960d5fa78c9b9802c3b16f3930504402334 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Tue, 18 Mar 2025 15:29:10 +0100 Subject: [PATCH 53/86] Do not scale O2 demands --- src/egon/data/datasets/scenario_path/scenario_path.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 121033820..5341a3adf 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1423,6 +1423,10 @@ def import_loads(scn: str): scn2_load_t.loc[df.index, "p_set"] = scn2_load_t.loc[ df.index, "p_set" ].apply(lambda x: np.array(x) * factor_gas) + elif c =="O2": + scn2_load_t.loc[df.index, "p_set"] = scn2_load_t.loc[ + df.index, "p_set" + ] else: scn2_load_t.loc[df.index, "p_set"] = scn2_load_t.loc[ df.index, "p_set" From 748e98cac52845d5aa53f421c2eb779981ff9e4d Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 18 Mar 2025 15:42:30 +0100 Subject: [PATCH 54/86] include urban_central_biomass CHP --- .../datasets/scenario_path/scenario_path.py | 166 +++++++++++++----- 1 file changed, 122 insertions(+), 44 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 5931fdd7a..c1f756e94 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -853,7 +853,7 @@ def load_scn_capacies_gen( "urban_central_heat_pump": "central_heat_pump", "urban_central_resistive_heater": "central_resistive_heater", "gas": "OCGT", - "urban_central_solid_biomass_CHP":"central_biomass_CHP" + "urban_central_solid_biomass_CHP": "central_biomass_CHP", } scn_capacities["carrier"] = scn_capacities["carrier"].apply( @@ -1150,16 +1150,14 @@ def import_generators(scn: str): obj = ( biogas1["p_nom"].sum() - + (biogas2["p_nom"].sum() - biogas1["p_nom"].sum()) * scaling_factor[scn] + + (biogas2["p_nom"].sum() - biogas1["p_nom"].sum()) + * scaling_factor[scn] ) biogas3["p_nom"] *= obj / biogas3["p_nom"].sum() # Interpolate costs based on gas fuel costs from status2019 and eGon100RE # egon.data.scenario_parameters - biogas3["marginal_cost"] = ( - 20.16 + (19.4 - 20.16) - * scaling_factor[scn] - ) + biogas3["marginal_cost"] = 20.16 + (19.4 - 20.16) * scaling_factor[scn] biogas3.to_sql( name="egon_etrago_generator", @@ -1244,57 +1242,137 @@ def import_generators(scn: str): if_exists="append", index=False, ) - - # deal with urban_central_biomass - ucbe3 = scn1_gen[scn1_gen["carrier"] == "central_biomass_CHP"].copy() - ucbe3["scn_name"] = scn - obj = cap_gen.at["central_biomass_CHP", scn] * 0.3 - ucbe3["p_nom"] *= obj / ucbe3["p_nom"].sum() + # deal with urban_central_biomass_heat + new_bio_heat = pd.DataFrame(columns=scn1_gen.columns) + + ref_bio_heat = scn1_gen[ + scn1_gen["carrier"] == "central_biomass_CHP_heat" + ].head(1) + ref_bio_heat["scn_name"] = scn + ref_bio_heat["carrier"] = "central_biomass_CHP_heat" + + central_heat_demand = pd.read_sql( + f""" + SELECT load_id, p_set FROM grid.egon_etrago_load_timeseries + WHERE load_id IN (SELECT load_id FROM grid.egon_etrago_load + WHERE carrier = 'central_heat' + AND scn_name = '{scn}' + AND bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = '{scn}' + ) + ) + AND scn_name = '{scn}' + """, + con, + index_col="load_id", + ) - ucbe3.to_sql( + central_heat_demand["p_max"] = ( + central_heat_demand["p_set"].apply(np.array).apply(max) + ) + central_heat_demand = central_heat_demand[ + central_heat_demand["p_max"] > 0.1 + ] + obj = cap_gen.at["central_biomass_CHP", scn] * 0.825 + central_heat_demand["p_nom"] = ( + central_heat_demand["p_max"] * obj / central_heat_demand["p_max"].sum() + ) + central_heat_demand.drop(columns="p_set", inplace=True) + + load_chd = pd.read_sql( + f""" + SELECT load_id, bus FROM grid.egon_etrago_load + WHERE scn_name = '{scn}' + AND carrier = 'central_heat' + """, + con, + ).set_index("load_id") + + central_heat_demand["bus"] = central_heat_demand.index.map(load_chd["bus"]) + + for g, df in central_heat_demand.iterrows(): + bus = int(central_heat_demand.at[g, "bus"]) + inst_capacity = float(central_heat_demand.at[g, "p_nom"]) + new_bio_heat.loc[str(bus) + "bio", :] = ref_bio_heat.values + new_bio_heat.loc[str(bus) + "bio", "bus"] = bus + new_bio_heat.loc[str(bus) + "bio", "p_nom"] = inst_capacity + + next_gen_id = ( + pd.read_sql( + """ + SELECT MAX(generator_id) FROM grid.egon_etrago_generator + """, + con, + ).iat[0, 0] + + 1 + ) + + new_bio_heat["generator_id"] = range( + next_gen_id, next_gen_id + len(new_bio_heat) + ) + + new_bio_heat.to_sql( name="egon_etrago_generator", con=con, schema="grid", if_exists="append", index=False, ) - - # deal with urban_central_biomass_heat - ucbh3 = scn1_gen[scn1_gen["carrier"] == "central_biomass_CHP_heat"].copy() - ucbh3["scn_name"] = scn - obj = cap_gen.at["central_biomass_CHP", scn] * 0.7 - ucbh3["p_nom"] *= obj / ucbe3["p_nom"].sum() - - chb1_geo = gpd.read_postgis( - """ + # deal with urban_central_biomass elec + new_bio_elec = pd.DataFrame(columns=scn1_gen.columns) + + ref_bio_elec = scn1_gen[scn1_gen["carrier"] == "central_biomass_CHP"].head( + 1 + ) + ref_bio_elec["scn_name"] = scn + ref_bio_elec["carrier"] = "central_biomass_CHP" + ref_bio_elec["marginal_cost"] = marg_cost3["biomass"] + + chb_geo = ( + gpd.read_postgis( + f""" SELECT bus_id, geom FROM grid.egon_etrago_bus - WHERE scn_name = 'status2019' + WHERE scn_name = '{scn}' AND carrier = 'central_heat' + AND country = 'DE' """, - con, - geom_col="geom", - ).set_index("bus_id") + con, + geom_col="geom", + ) + .set_index("bus_id") + .to_crs(3035) + ) - chb2_geo = gpd.read_postgis( - f""" + acb_geo = ( + gpd.read_postgis( + f""" SELECT bus_id, geom FROM grid.egon_etrago_bus WHERE scn_name = '{scn}' - AND carrier = 'central_heat' + AND carrier = 'AC' + AND country = 'DE' + AND v_nom = '110' """, - con, - geom_col="geom", - ).set_index("bus_id") + con, + geom_col="geom", + ) + .set_index("bus_id") + .to_crs(3035) + ) - chb1_to_chb2 = {} - for l in chb1_geo.index: - dist = chb2_geo.distance(chb1_geo["geom"][l]) + new_bio_heat.set_index("bus", inplace=True) + for l in new_bio_heat.index: + dist = acb_geo.distance(chb_geo["geom"][l]) dist.sort_values(inplace=True) - chb1_to_chb2[l] = dist.index[0] - - ucbh3["bus"] = ucbh3["bus"].map(chb1_to_chb2) - + bus = int(dist.index[0]) + inst_capacity = float(new_bio_heat.at[l, "p_nom"]) / 0.825 * 0.2694 + new_bio_elec.loc[str(l) + "bio_elec", :] = ref_bio_elec.values + new_bio_elec.loc[str(l) + "bio_elec", "bus"] = bus + new_bio_elec.loc[str(l) + "bio_elec", "p_nom"] = inst_capacity + next_gen_id = ( pd.read_sql( """ @@ -1305,18 +1383,18 @@ def import_generators(scn: str): + 1 ) - ucbh3["generator_id"] = range( - next_gen_id, next_gen_id + len(ucbh3) + new_bio_elec["generator_id"] = range( + next_gen_id, next_gen_id + len(new_bio_elec) ) - - ucbh3.to_sql( + + new_bio_elec.to_sql( name="egon_etrago_generator", con=con, schema="grid", if_exists="append", index=False, ) - + return From b16a7fb7fa6e0d22b0a43fad84ff79eb4de90a00 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 18 Mar 2025 15:49:29 +0100 Subject: [PATCH 55/86] skip when no isolated h2_grid bus --- .../datasets/scenario_path/scenario_path.py | 130 +++++++++--------- 1 file changed, 67 insertions(+), 63 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index c1f756e94..c5541cd6a 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -734,77 +734,81 @@ def import_links(scn: str): ) ].copy() - h2_grid_to_h2["carrier"] = "H2" - h2_grid_to_h2["scn_name"] = scn + if len(h2_grid_to_h2) > 0: + h2_grid_to_h2["carrier"] = "H2" + h2_grid_to_h2["scn_name"] = scn - db.execute_sql( - f""" - DELETE FROM grid.egon_etrago_bus - WHERE scn_name = '{scn}' - AND bus_id IN {tuple(h2_grid_to_h2.index)} - """ - ) - - ch4_b = gpd.read_postgis( - f""" - SELECT * FROM grid.egon_etrago_bus - WHERE scn_name = '{scn}' - AND carrier = 'CH4' - and country = 'DE' - """, - con, - geom_col="geom", - ).set_index("bus_id") - - closest_ch4 = {} - for b in h2_grid_to_h2.index: - dist = ch4_b.distance(h2_grid_to_h2["geom"][b]) - closest_ch4[b] = dist.idxmin() - - refference = identical3[identical3["carrier"] == "CH4_to_H2"].head(1) - - new_ch4_to_h2 = gpd.GeoDataFrame(columns=identical3.columns) - for bus in h2_grid_to_h2.index: - new_ch4_to_h2.loc[bus, :] = refference.values - new_ch4_to_h2.loc[bus, "bus0"] = closest_ch4[bus] - new_ch4_to_h2.loc[bus, "bus1"] = bus - new_ch4_to_h2.loc[bus, "geom"] = LineString( - [ch4_b.at[closest_ch4[bus], "geom"], h2_grid_to_h2.at[bus, "geom"]] + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND bus_id IN {tuple(h2_grid_to_h2.index)} + """ ) - new_ch4_to_h2.set_geometry("geom", inplace=True) - new_ch4_to_h2.set_crs(crs=4326, inplace=True) - next_link_id = ( - pd.read_sql( + ch4_b = gpd.read_postgis( f""" - SELECT MAX(link_id) FROM grid.egon_etrago_link - """, + SELECT * FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = 'CH4' + and country = 'DE' + """, con, - ).iat[0, 0] - + 1 - ) + geom_col="geom", + ).set_index("bus_id") + + closest_ch4 = {} + for b in h2_grid_to_h2.index: + dist = ch4_b.distance(h2_grid_to_h2["geom"][b]) + closest_ch4[b] = dist.idxmin() + + refference = identical3[identical3["carrier"] == "CH4_to_H2"].head(1) + + new_ch4_to_h2 = gpd.GeoDataFrame(columns=identical3.columns) + for bus in h2_grid_to_h2.index: + new_ch4_to_h2.loc[bus, :] = refference.values + new_ch4_to_h2.loc[bus, "bus0"] = closest_ch4[bus] + new_ch4_to_h2.loc[bus, "bus1"] = bus + new_ch4_to_h2.loc[bus, "geom"] = LineString( + [ + ch4_b.at[closest_ch4[bus], "geom"], + h2_grid_to_h2.at[bus, "geom"], + ] + ) + new_ch4_to_h2.set_geometry("geom", inplace=True) + new_ch4_to_h2.set_crs(crs=4326, inplace=True) - new_ch4_to_h2["link_id"] = range( - next_link_id, next_link_id + len(new_ch4_to_h2) - ) - new_ch4_to_h2["build_year"] = new_ch4_to_h2["build_year"].apply(int) + next_link_id = ( + pd.read_sql( + f""" + SELECT MAX(link_id) FROM grid.egon_etrago_link + """, + con, + ).iat[0, 0] + + 1 + ) - new_ch4_to_h2.to_postgis( - name="egon_etrago_link", - con=con, - schema="grid", - if_exists="append", - index=False, - ) + new_ch4_to_h2["link_id"] = range( + next_link_id, next_link_id + len(new_ch4_to_h2) + ) + new_ch4_to_h2["build_year"] = new_ch4_to_h2["build_year"].apply(int) - h2_grid_to_h2.reset_index(inplace=True) - h2_grid_to_h2.to_postgis( - name="egon_etrago_bus", - con=con, - schema="grid", - if_exists="append", - index=False, - ) + new_ch4_to_h2.to_postgis( + name="egon_etrago_link", + con=con, + schema="grid", + if_exists="append", + index=False, + ) + + h2_grid_to_h2.reset_index(inplace=True) + h2_grid_to_h2.to_postgis( + name="egon_etrago_bus", + con=con, + schema="grid", + if_exists="append", + index=False, + ) return From 50ef6b40d94dad522a178d9591d3b1ae69283ee3 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 18 Mar 2025 16:17:58 +0100 Subject: [PATCH 56/86] assign new generator_id to biogas generators --- .../data/datasets/scenario_path/scenario_path.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index c5541cd6a..490bab2f5 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1163,6 +1163,20 @@ def import_generators(scn: str): # egon.data.scenario_parameters biogas3["marginal_cost"] = 20.16 + (19.4 - 20.16) * scaling_factor[scn] + next_gen_id = ( + pd.read_sql( + """ + SELECT MAX(generator_id) FROM grid.egon_etrago_generator + """, + con, + ).iat[0, 0] + + 1 + ) + + biogas3["generator_id"] = range( + next_gen_id, next_gen_id + len(biogas3) + ) + biogas3.to_sql( name="egon_etrago_generator", con=con, From f3db42b408d2c446b46a81c79385b257956d2739 Mon Sep 17 00:00:00 2001 From: Lennart Zimmermann Date: Tue, 18 Mar 2025 16:42:22 +0100 Subject: [PATCH 57/86] Fix missing heat_pump timeseries for powerd2025 --- src/egon/data/datasets/pypsaeur/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/pypsaeur/__init__.py b/src/egon/data/datasets/pypsaeur/__init__.py index 8c1680362..ea3c91128 100755 --- a/src/egon/data/datasets/pypsaeur/__init__.py +++ b/src/egon/data/datasets/pypsaeur/__init__.py @@ -577,7 +577,7 @@ def combine_decentral_and_rural_heat(network_solved, network_prepared): def combine_residenial_services_heat_load(network_prepared): - + ###combining loads residential_rural_loads = network_prepared.loads[ network_prepared.loads.carrier.str.contains("residential rural heat") ] @@ -619,6 +619,11 @@ def combine_residenial_services_heat_load(network_prepared): .str.replace("residential rural heat", "rural heat") ) + ####combining heat_pumps + network_prepared.links_t.efficiency.columns = (network_prepared.links_t.efficiency.columns.str.replace("services rural", "rural heat") + .str.replace("residential rural", "rural") + ) + return network_prepared From c5c5f79365d65afa446ffc18244d58764739b20e Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 18 Mar 2025 17:19:13 +0100 Subject: [PATCH 58/86] include efficiency fossil fuels generators --- .../datasets/scenario_path/scenario_path.py | 61 ++++++++++++------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index c039cc544..0a02ed7ef 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -982,6 +982,12 @@ def import_generators(scn: str): ~cap_gen.loc[fossil_carriers, scn].isna() ].index.values + efficiency = { + "oil": 0.35, + "coal": 0.33, + "lignite": 0.33, + } + if len(fossil_carriers) > 0: fossil3 = scn1_gen[scn1_gen["carrier"].isin(fossil_carriers)].copy() fossil3["scn_name"] = scn @@ -996,7 +1002,7 @@ def import_generators(scn: str): for c, df in fossil3.groupby("carrier"): id = df.index - objective = cap_gen.at[c, scn] + objective = cap_gen.at[c, scn] * efficiency[c] fossil3.loc[id, "p_nom"] *= ( objective / fossil3.loc[id, "p_nom"].sum() ) @@ -1173,9 +1179,7 @@ def import_generators(scn: str): + 1 ) - biogas3["generator_id"] = range( - next_gen_id, next_gen_id + len(biogas3) - ) + biogas3["generator_id"] = range(next_gen_id, next_gen_id + len(biogas3)) biogas3.to_sql( name="egon_etrago_generator", @@ -1484,13 +1488,17 @@ def import_loads(scn: str): pre_network = prepared_network(year=scn.replace("powerd", "")) - annual_sum_gas = pre_network.loads.loc["DE0 0 gas for industry", "p_set"] * 8760 - annual_sum_h2 = pre_network.loads.loc["DE0 0 H2 for industry", "p_set"] * 8760 + annual_sum_gas = ( + pre_network.loads.loc["DE0 0 gas for industry", "p_set"] * 8760 + ) + annual_sum_h2 = ( + pre_network.loads.loc["DE0 0 H2 for industry", "p_set"] * 8760 + ) annual_sum_powerd_gas = 0 - for gas_load in scn2_load[scn2_load.carrier=="CH4_for_industry"].index: + for gas_load in scn2_load[scn2_load.carrier == "CH4_for_industry"].index: annual_sum_powerd_gas += sum(scn2_load_t.p_set[gas_load]) annual_sum_powerd_h2 = 0 - for h2_load in scn2_load[scn2_load.carrier=="H2_for_industry"].index: + for h2_load in scn2_load[scn2_load.carrier == "H2_for_industry"].index: annual_sum_powerd_h2 += sum(scn2_load_t.p_set[h2_load]) factor_h2 = annual_sum_h2 / annual_sum_powerd_h2 @@ -1514,12 +1522,12 @@ def import_loads(scn: str): if c == "H2_for_industry": scn2_load_t.loc[df.index, "p_set"] = scn2_load_t.loc[ df.index, "p_set" - ].apply(lambda x: np.array(x)* factor_h2) + ].apply(lambda x: np.array(x) * factor_h2) elif c == "CH4_for_industry": scn2_load_t.loc[df.index, "p_set"] = scn2_load_t.loc[ df.index, "p_set" ].apply(lambda x: np.array(x) * factor_gas) - elif c =="O2": + elif c == "O2": scn2_load_t.loc[df.index, "p_set"] = scn2_load_t.loc[ df.index, "p_set" ] @@ -1559,7 +1567,8 @@ def import_loads(scn: str): total_ac_pypsaeur = ( pre_network.loads_t.p_set["DE0 0"].sum() - + pre_network.loads_t.p_set["DE0 0 industry electricity"].sum()) + + pre_network.loads_t.p_set["DE0 0 industry electricity"].sum() + ) for b, df in ac_load.groupby("bus"): df1 = df[df["scn_name"] == "status2019"] @@ -1579,13 +1588,17 @@ def import_loads(scn: str): df2.index, "p_set" ].apply(lambda x: np.array(x) * objective / df2_total) - total_ac_scn2 = scn2_load_t.loc[ - scn2_load[scn2_load["carrier"] == "AC"].index, "p_set" - ].apply(lambda x: np.array(x).sum()).sum() + total_ac_scn2 = ( + scn2_load_t.loc[scn2_load[scn2_load["carrier"] == "AC"].index, "p_set"] + .apply(lambda x: np.array(x).sum()) + .sum() + ) - scn2_load_t.loc[ scn2_load[scn2_load["carrier"] == "AC"].index, "p_set"] = scn2_load_t.loc[ - scn2_load[scn2_load["carrier"] == "AC"].index, "p_set" - ].apply(lambda x: np.array(x) * total_ac_pypsaeur / total_ac_scn2) + scn2_load_t.loc[scn2_load[scn2_load["carrier"] == "AC"].index, "p_set"] = ( + scn2_load_t.loc[ + scn2_load[scn2_load["carrier"] == "AC"].index, "p_set" + ].apply(lambda x: np.array(x) * total_ac_pypsaeur / total_ac_scn2) + ) # Dealing with rural_heat loads rh2 = scn2_load[scn2_load["carrier"] == "rural_heat"].copy() @@ -1600,12 +1613,18 @@ def import_loads(scn: str): objective = ( pre_network.loads_t.p_set["DE0 0 residential rural heat"].sum() + pre_network.loads_t.p_set["DE0 0 services rural heat"].sum() - + pre_network.loads_t.p_set["DE0 0 residential urban decentral heat"].sum() - + pre_network.loads_t.p_set["DE0 0 services urban decentral heat"].sum() + + pre_network.loads_t.p_set[ + "DE0 0 residential urban decentral heat" + ].sum() + + pre_network.loads_t.p_set[ + "DE0 0 services urban decentral heat" + ].sum() ) else: - objective = (pre_network.loads_t.p_set["DE0 0 rural heat"].sum() - +pre_network.loads_t.p_set["DE0 0 urban decentral heat"].sum()) + objective = ( + pre_network.loads_t.p_set["DE0 0 rural heat"].sum() + + pre_network.loads_t.p_set["DE0 0 urban decentral heat"].sum() + ) scn2_load_t.loc[rh2.index, "p_set"] = scn2_load_t.loc[ rh2.index, "p_set" From cbd5b8e01812f1665fe845677aecedc8c442e546 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 18 Mar 2025 17:27:06 +0100 Subject: [PATCH 59/86] standarize argument scn in all functions --- .../data/datasets/scenario_path/scenario_path.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 0a02ed7ef..6e64a49f8 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1844,8 +1844,8 @@ def import_stores(scn: str): return -def import_foreign(scn_name, year): - neighbor_reduction(scn_name, year) +def import_foreign(scn: str, year): + neighbor_reduction(scn, year) # import links joining DE and foreign countries link_foreign = pd.read_sql( @@ -1872,7 +1872,7 @@ def import_foreign(scn_name, year): bus_ch4_h2 = pd.read_sql( sql=f""" SELECT * from grid.egon_etrago_bus - WHERE scn_name IN ('eGon100RE', '{scn_name}') + WHERE scn_name IN ('eGon100RE', '{scn}') AND carrier IN ('CH4', 'H2') AND country <> 'DE' """, @@ -1883,7 +1883,7 @@ def import_foreign(scn_name, year): map_scn1_to_scn2 = {} for b, df in bus_ch4_h2.groupby(["x", "y", "carrier"]): bus1 = df.index[df["scn_name"] == "eGon100RE"][0] - bus2 = df.index[df["scn_name"] == scn_name][0] + bus2 = df.index[df["scn_name"] == scn][0] map_scn1_to_scn2[bus1] = bus2 link_foreign["bus0"] = link_foreign["bus0"].apply( @@ -1894,7 +1894,7 @@ def import_foreign(scn_name, year): lambda x: map_scn1_to_scn2[x] if x in map_scn1_to_scn2.keys() else x ) - link_foreign["scn_name"] = scn_name + link_foreign["scn_name"] = scn link_foreign.reset_index(inplace=True) link_foreign.to_sql( name="egon_etrago_link", @@ -1902,4 +1902,4 @@ def import_foreign(scn_name, year): schema="grid", if_exists="append", index=False, - ) + ) \ No newline at end of file From 47930038efd27db6b985cf3a50e729bffcdc3978 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 18 Mar 2025 17:38:14 +0100 Subject: [PATCH 60/86] include rural oil and biomass boilers in foreign countries --- .../datasets/scenario_path/scenario_path.py | 91 ++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 6e64a49f8..468779873 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1902,4 +1902,93 @@ def import_foreign(scn: str, year): schema="grid", if_exists="append", index=False, - ) \ No newline at end of file + ) + + # dealing with rural_biomass_boiler and rural_oil_boiler + load_rh = pd.read_sql( + f""" + SELECT * FROM grid.egon_etrago_load + WHERE scn_name = '{scn}' + AND carrier = 'rural_heat' + AND bus in (SELECT bus_id FROM grid.egon_etrago_bus + WHERE country <> 'DE' + AND scn_name = '{scn}') + """, + con, + index_col="load_id", + ) + + scn1_gen = pd.read_sql( + f""" + SELECT * FROM grid.egon_etrago_generator + WHERE scn_name = '{scn}' + AND bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = '{scn}') + AND carrier = 'oil' + """, + con, + ) + marg_cost3 = interpolate_marginal_costs(scn) + refference_oil = scn1_gen[scn1_gen["carrier"] == "oil"].head(1) + refference_oil["scn_name"] = scn + refference_oil["carrier"] = "rural_oil_boiler" + refference_oil["marginal_cost"] = marg_cost3["oil"] + refference_biomass = scn1_gen[scn1_gen["carrier"] == "oil"].head(1) + refference_biomass["scn_name"] = scn + refference_biomass["carrier"] = "rural_biomass_boiler" + refference_biomass["marginal_cost"] = marg_cost3["biomass"] + + load_t_rh = pd.read_sql( + f""" + SELECT * FROM grid.egon_etrago_load_timeseries + WHERE load_id IN( + SELECT load_id FROM grid.egon_etrago_load + WHERE bus IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country <> 'DE' + AND scn_name = '{scn}' + ) + AND scn_name = '{scn}' + ) + AND load_id IN (SELECT load_id FROM grid.egon_etrago_load + WHERE carrier = 'rural_heat' + AND scn_name = '{scn}') + AND scn_name = '{scn}' + """, + con, + index_col="load_id", + ) + + new_gen_boiler = pd.DataFrame(columns=scn1_gen.columns) + for load, df in load_t_rh.iterrows(): + bus = int(load_rh.at[load, "bus"]) + inst_capacity = float(np.array(df.at["p_set"]).max()) + new_gen_boiler.loc[str(bus) + "bio", :] = refference_biomass.values + new_gen_boiler.loc[str(bus) + "bio", "bus"] = bus + new_gen_boiler.loc[str(bus) + "bio", "p_nom"] = inst_capacity + new_gen_boiler.loc[str(bus) + "oil", :] = refference_oil.values + new_gen_boiler.loc[str(bus) + "oil", "bus"] = bus + new_gen_boiler.loc[str(bus) + "oil", "p_nom"] = inst_capacity + + next_gen_id = ( + pd.read_sql( + """ + SELECT MAX(generator_id) FROM grid.egon_etrago_generator + """, + con, + ).iat[0, 0] + + 1 + ) + + new_gen_boiler["generator_id"] = range( + next_gen_id, next_gen_id + len(new_gen_boiler) + ) + new_gen_boiler.to_sql( + name="egon_etrago_generator", + con=con, + schema="grid", + if_exists="append", + index=False, + ) From 6a1473c57ea9e522f9e7561a974d2b081785ef28 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Wed, 19 Mar 2025 08:47:48 +0100 Subject: [PATCH 61/86] delete associated elements to converted H2_grid buses --- .../data/datasets/scenario_path/__init__.py | 6 +- .../datasets/scenario_path/scenario_path.py | 58 +++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/__init__.py b/src/egon/data/datasets/scenario_path/__init__.py index 877002a06..342279875 100644 --- a/src/egon/data/datasets/scenario_path/__init__.py +++ b/src/egon/data/datasets/scenario_path/__init__.py @@ -16,9 +16,9 @@ def create_powerd2025(): import_network_structure(scn) import_loads(scn) import_generators(scn) - import_links(scn) import_storage_units(scn) import_stores(scn) + import_links(scn) import_foreign(scn, year=2025) return @@ -29,9 +29,9 @@ def create_powerd2030(): import_network_structure(scn) import_loads(scn) import_generators(scn) - import_links(scn) import_storage_units(scn) import_stores(scn) + import_links(scn) import_foreign(scn, year=2030) return @@ -42,9 +42,9 @@ def create_powerd2035(): import_network_structure(scn) import_loads(scn) import_generators(scn) - import_links(scn) import_storage_units(scn) import_stores(scn) + import_links(scn) import_foreign(scn, year=2035) return diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 468779873..8cbcc76e1 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -738,6 +738,8 @@ def import_links(scn: str): h2_grid_to_h2["carrier"] = "H2" h2_grid_to_h2["scn_name"] = scn + # delete H2_grid isolated, associated H2_saltcaverns links, + # H2_saltcavern buses and H2_underground stores db.execute_sql( f""" DELETE FROM grid.egon_etrago_bus @@ -746,6 +748,62 @@ def import_links(scn: str): """ ) + del_h2_salcavern_link = pd.read_sql( + f""" + SELECT link_id, bus0, bus1, carrier FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND carrier = 'H2_saltcavern' + AND ((bus0 IN {tuple(h2_grid_to_h2.index)}) OR + (bus1 IN {tuple(h2_grid_to_h2.index)})) + """, + con, + ) + + del_h2_salcavern_bus = pd.read_sql( + f""" + SELECT bus_id, carrier FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = 'H2_saltcavern' + AND ((bus_id IN {tuple(del_h2_salcavern_link.bus0)}) OR + (bus_id IN {tuple(del_h2_salcavern_link.bus1)})) + """, + con, + ) + + del_h2_underground_store = pd.read_sql( + f""" + SELECT store_id, carrier, bus FROM grid.egon_etrago_store + WHERE scn_name = '{scn}' + AND carrier = 'H2_underground' + AND bus IN {tuple(del_h2_salcavern_bus.bus_id)} + """, + con, + ) + + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND link_id IN {tuple(del_h2_salcavern_link.link_id)} + """ + ) + + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND bus_id IN {tuple(del_h2_salcavern_bus.bus_id)} + """ + ) + + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_store + WHERE scn_name = '{scn}' + AND store_id IN {tuple(del_h2_underground_store.store_id)} + """ + ) + ch4_b = gpd.read_postgis( f""" SELECT * FROM grid.egon_etrago_bus From ece26c3f07ccde8580971c8c4945a73a135f2d18 Mon Sep 17 00:00:00 2001 From: Lennart Zimmermann Date: Wed, 19 Mar 2025 09:25:54 +0100 Subject: [PATCH 62/86] Exclude isolated FR bus for scenario_path --- src/egon/data/datasets/scenario_path/scenario_path.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 5931fdd7a..571cdcb41 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -71,9 +71,10 @@ def import_network_structure(scn: str): sql=""" SELECT * from grid.egon_etrago_bus WHERE scn_name = 'eGon100RE' AND carrier = 'AC' + AND (x!=9.4506 AND y!=42.5288) """, con=con, - ) + ) #exclude isolated FR bus with defined koordinates other_buses = pd.read_sql( sql=""" From 9afee61ec3f7c542c301a8e3d0741a397a32a0e7 Mon Sep 17 00:00:00 2001 From: Lennart Zimmermann Date: Fri, 21 Mar 2025 10:20:48 +0100 Subject: [PATCH 63/86] Transform links (coal, lignite...) into generators for foreign countries --- src/egon/data/datasets/pypsaeur/__init__.py | 82 +++++++++++++++++++-- 1 file changed, 74 insertions(+), 8 deletions(-) diff --git a/src/egon/data/datasets/pypsaeur/__init__.py b/src/egon/data/datasets/pypsaeur/__init__.py index ea3c91128..73c347414 100755 --- a/src/egon/data/datasets/pypsaeur/__init__.py +++ b/src/egon/data/datasets/pypsaeur/__init__.py @@ -627,6 +627,66 @@ def combine_residenial_services_heat_load(network_prepared): return network_prepared + +def import_missing_gens(neighbors, network_solved, scn_name): + carriers_to_keep=['oil', 'lignite', 'coal', 'urban central solid biomass CHP'] + marg = margina_cost_missing_gens() + + for carrier in carriers_to_keep: + links = network_solved.links[network_solved.links.carrier == carrier] + links_neighbor = links[ + links['bus1'].isin(neighbors.index) + ] + marg_cost= marg[scn_name][carrier] + print(marg_cost) + for idx, link in links_neighbor.iterrows(): + if carrier == 'urban central solid biomass CHP': + network_solved.add("Generator", f"gen_{idx}_electrical", + bus=link.bus1, + p_nom= link.p_nom_opt*link.efficiency, + carrier="central_biomass_CHP", + marginal_cost=marg_cost + ) + network_solved.add("Generator", f"gen_{idx}_heat", + bus=link.bus2, + p_nom=link.p_nom_opt*link.efficiency2, + carrier="central_biomass_CHP_heat", + ) + else: + network_solved.add("Generator", f"gen_{idx}", + bus=link.bus1, + p_nom=link.p_nom_opt*link.efficiency, + carrier=carrier, + marginal_cost=marg_cost + ) + return network_solved + +def margina_cost_missing_gens(): + marginal_costs = { + "powerd2025": { + "oil": 164.90901098901102, + "lignite": 67.38601398601398, + "coal": 76.07459207459208, + "urban central solid biomass CHP": 39.69634478996181 + }, + "powerd2030": { + "oil": 169.8246153846154, + "lignite": 86.11148018648018, + "coal": 88.3854895104895, + "urban central solid biomass CHP": 51.573854337152206 + }, + "powerd2035": { + "oil": 174.7402197802198, + "lignite": 104.834, + "coal": 100.7, + "urban central solid biomass CHP": 63.451363884342605 + } + } + + return marginal_costs + + + def neighbor_reduction(scn_name, year=2045): network_solved = read_network(year=year) network_prepared = prepared_network(year=year) @@ -656,7 +716,21 @@ def neighbor_reduction(scn_name, year=2045): network_solved.buses = network_solved.buses.drop( network_solved.buses.loc[foreign_buses.index].index ) + + # Set country tag for all buses + network_solved.buses.country = network_solved.buses.index.str[:2] + neighbors = network_solved.buses[network_solved.buses.country != "DE"] + + neighbors["new_index"] = ( + db.next_etrago_id("bus") + neighbors.reset_index().index + ) + #keep links that are connected to an central EU-bus(solid biomass, oil, llignite, coal) + #transform them to generators + if scn_name !='eGon100RE': + network_solved = import_missing_gens(neighbors, network_solved, scn_name) + + # Add H2 demand of Fischer-Tropsch process and methanolisation # to industrial H2 demands industrial_hydrogen = network_prepared.loads.loc[ @@ -845,14 +919,6 @@ def neighbor_reduction(scn_name, year=2045): # writing components of neighboring countries to etrago tables - # Set country tag for all buses - network_solved.buses.country = network_solved.buses.index.str[:2] - neighbors = network_solved.buses[network_solved.buses.country != "DE"] - - neighbors["new_index"] = ( - db.next_etrago_id("bus") + neighbors.reset_index().index - ) - # Use index of AC buses created by electrical_neigbors foreign_ac_buses = db.select_dataframe( """ From c0b7f2da81075902d065b304f2d49b32d31d7fd5 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 21 Mar 2025 11:11:20 +0100 Subject: [PATCH 64/86] include marginal costs solar_rooftop scn_path --- src/egon/data/datasets/scenario_path/scenario_path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 271dc5a56..9cde39135 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -975,7 +975,7 @@ def interpolate_marginal_costs(scn): ) marg_cost3["run_of_river"] = 0 - marg_cost3["solar_rooftop"] = 0 + marg_cost3["solar_rooftop"] = 0.01 return marg_cost3 From 696eee50f8b1f7a8ae0a24cebae7b9b01864e980 Mon Sep 17 00:00:00 2001 From: Lennart Zimmermann Date: Fri, 21 Mar 2025 11:17:10 +0100 Subject: [PATCH 65/86] adust parameters for electrolyzers according to ISE values --- .../datasets/scenario_path/scenario_path.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 271dc5a56..ec5773182 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -14,6 +14,9 @@ from egon.data.datasets.pypsaeur import neighbor_reduction, prepared_network from egon.data.datasets.scenario_parameters import get_sector_parameters +from egon.data.datasets.scenario_parameters.parameters import ( + annualize_capital_costs, +) sources = egon.data.config.datasets()["scenario_path"]["sources"] @@ -868,6 +871,40 @@ def import_links(scn: str): if_exists="append", index=False, ) + + ###adjust electrolyzer parameters according to Fraunhofer ISE + efficiency = { + 'powerd2025': 0.6535, + 'powerd2030': 0.6666, + 'powerd2035': 0.6805, + } + capital_cost = { + 'powerd2025': 706_000, + 'powerd2030': 504_000, + 'powerd2035': 452_000, + } + lifetime = { + 'powerd2025': 20, + 'powerd2030': 25, + 'powerd2035': 25, + } + + old_cost =annualize_capital_costs(357_000, 30, 0.05) + new_cost = annualize_capital_costs(capital_cost[scn], lifetime[scn], 0.05) + + sql_update= f""" + UPDATE grid.egon_etrago_link + SET capital_cost = (capital_cost - {old_cost} + {new_cost}) + WHERE carrier = 'power_to_H2' AND scn_name = '{scn}'; + + UPDATE grid.egon_etrago_link + SET efficiency = {efficiency[scn]} + WHERE carrier = 'power_to_H2' AND scn_name = '{scn}'; + + """ + db.execute_sql(sql_update) + + return From feb519eafb59ac7e380e6644ec93c23e570630b2 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 21 Mar 2025 13:45:43 +0100 Subject: [PATCH 66/86] correct marginal cost biomass and oil boiler generators --- .../scenario_parameters/parameters.py | 4 ++ .../datasets/scenario_path/scenario_path.py | 66 +++++++++++-------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/src/egon/data/datasets/scenario_parameters/parameters.py b/src/egon/data/datasets/scenario_parameters/parameters.py index 22ffc839f..36fb30bce 100755 --- a/src/egon/data/datasets/scenario_parameters/parameters.py +++ b/src/egon/data/datasets/scenario_parameters/parameters.py @@ -1223,6 +1223,8 @@ def heat(scenario): "central_gas_boiler": read_costs( costs, "central gas boiler", "VOM" ), + "rural_biomass_boiler": 51, # considering an efficiency of 0.8 + "rural_oil_boiler": 67.2, # considering an efficiency of 0.9 "central_resistive_heater": read_costs( costs, "central resistive heater", "VOM" ), @@ -1311,6 +1313,8 @@ def heat(scenario): "central_gas_boiler": read_costs( costs, "central gas boiler", "VOM" ), + "rural_biomass_boiler": 10, # considering an efficiency of 0.8 + "rural_oil_boiler": 59.7, # considering an efficiency of 0.9 "central_resistive_heater": read_costs( costs, "central resistive heater", "VOM" ), diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index a1258ef02..589510cb9 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -77,7 +77,7 @@ def import_network_structure(scn: str): AND (x!=9.4506 AND y!=42.5288) """, con=con, - ) #exclude isolated FR bus with defined koordinates + ) # exclude isolated FR bus with defined koordinates other_buses = pd.read_sql( sql=""" @@ -871,28 +871,28 @@ def import_links(scn: str): if_exists="append", index=False, ) - - ###adjust electrolyzer parameters according to Fraunhofer ISE + + ###adjust electrolyzer parameters according to Fraunhofer ISE efficiency = { - 'powerd2025': 0.6535, - 'powerd2030': 0.6666, - 'powerd2035': 0.6805, - } + "powerd2025": 0.6535, + "powerd2030": 0.6666, + "powerd2035": 0.6805, + } capital_cost = { - 'powerd2025': 706_000, - 'powerd2030': 504_000, - 'powerd2035': 452_000, - } + "powerd2025": 706_000, + "powerd2030": 504_000, + "powerd2035": 452_000, + } lifetime = { - 'powerd2025': 20, - 'powerd2030': 25, - 'powerd2035': 25, - } + "powerd2025": 20, + "powerd2030": 25, + "powerd2035": 25, + } - old_cost =annualize_capital_costs(357_000, 30, 0.05) + old_cost = annualize_capital_costs(357_000, 30, 0.05) new_cost = annualize_capital_costs(capital_cost[scn], lifetime[scn], 0.05) - sql_update= f""" + sql_update = f""" UPDATE grid.egon_etrago_link SET capital_cost = (capital_cost - {old_cost} + {new_cost}) WHERE carrier = 'power_to_H2' AND scn_name = '{scn}'; @@ -903,8 +903,7 @@ def import_links(scn: str): """ db.execute_sql(sql_update) - - + return @@ -998,18 +997,31 @@ def load_scn_capacies_gen( def interpolate_marginal_costs(scn): - marg_cost1 = get_sector_parameters( + marg_cost1_elec = get_sector_parameters( sector="electricity", scenario="status2019" )["marginal_cost"] - marg_cost2 = get_sector_parameters( + marg_cost2_elec = get_sector_parameters( sector="electricity", scenario="eGon100RE" )["marginal_cost"] + + marg_cost1_heat = get_sector_parameters( + sector="heat", scenario="status2019" + )["marginal_cost"] + marg_cost2_heat = get_sector_parameters( + sector="heat", scenario="eGon100RE" + )["marginal_cost"] + + marg_cost1 = {**marg_cost1_elec, **marg_cost1_heat} + marg_cost2 = {**marg_cost2_elec, **marg_cost2_heat} marg_cost3 = {} for f in marg_cost2.keys(): - marg_cost3[f] = ( - marg_cost1[f] - + (marg_cost2[f] - marg_cost1[f]) * scaling_factor[scn] - ) + if (f in marg_cost1.keys()) & (f in marg_cost2.keys()): + marg_cost3[f] = ( + marg_cost1[f] + + (marg_cost2[f] - marg_cost1[f]) * scaling_factor[scn] + ) + else: + continue marg_cost3["run_of_river"] = 0 marg_cost3["solar_rooftop"] = 0.01 @@ -1302,11 +1314,11 @@ def import_generators(scn: str): refference_oil = scn1_gen[scn1_gen["carrier"] == "oil"].head(1) refference_oil["scn_name"] = scn refference_oil["carrier"] = "rural_oil_boiler" - refference_oil["marginal_cost"] = marg_cost3["oil"] + refference_oil["marginal_cost"] = marg_cost3["rural_oil_boiler"] refference_biomass = scn1_gen[scn1_gen["carrier"] == "oil"].head(1) refference_biomass["scn_name"] = scn refference_biomass["carrier"] = "rural_biomass_boiler" - refference_biomass["marginal_cost"] = marg_cost3["biomass"] + refference_biomass["marginal_cost"] = marg_cost3["rural_biomass_boiler"] load_t_rh = pd.read_sql( f""" From 2d52a50d349e6049815f5aeaabd09d8293f87511 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 21 Mar 2025 13:57:07 +0100 Subject: [PATCH 67/86] dont delete anything from egon_mv_grid_district --- src/egon/data/datasets/scenario_path/scenario_path.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 589510cb9..9b2815738 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -53,6 +53,7 @@ def clean_existing_scn_path_data(): "egon_etrago_temp_resolution", "egon_etrago_ac_h2", "egon_etrago_hv_busmap", + "egon_mv_grid_district", ] ) ] From 310918d0a417cf5500edf0f25864ef5427755f86 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 21 Mar 2025 15:18:22 +0100 Subject: [PATCH 68/86] fix missing installed fossil gen capacities --- src/egon/data/datasets/pypsaeur/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/pypsaeur/__init__.py b/src/egon/data/datasets/pypsaeur/__init__.py index 73c347414..f07a735ae 100755 --- a/src/egon/data/datasets/pypsaeur/__init__.py +++ b/src/egon/data/datasets/pypsaeur/__init__.py @@ -643,19 +643,19 @@ def import_missing_gens(neighbors, network_solved, scn_name): if carrier == 'urban central solid biomass CHP': network_solved.add("Generator", f"gen_{idx}_electrical", bus=link.bus1, - p_nom= link.p_nom_opt*link.efficiency, + p_nom_opt= link.p_nom_opt*link.efficiency, carrier="central_biomass_CHP", marginal_cost=marg_cost ) network_solved.add("Generator", f"gen_{idx}_heat", bus=link.bus2, - p_nom=link.p_nom_opt*link.efficiency2, + p_nom_opt=link.p_nom_opt*link.efficiency2, carrier="central_biomass_CHP_heat", ) else: network_solved.add("Generator", f"gen_{idx}", bus=link.bus1, - p_nom=link.p_nom_opt*link.efficiency, + p_nom_opt=link.p_nom_opt*link.efficiency, carrier=carrier, marginal_cost=marg_cost ) From b6c73bfa0dfb49b7260acac403627950142fee3c Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 21 Mar 2025 15:45:33 +0100 Subject: [PATCH 69/86] fix biomass cost de --- src/egon/data/datasets/scenario_path/scenario_path.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 9b2815738..c1a0700a4 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1026,6 +1026,7 @@ def interpolate_marginal_costs(scn): marg_cost3["run_of_river"] = 0 marg_cost3["solar_rooftop"] = 0.01 + marg_cost3["rural_biomass_boiler"] = marg_cost2_heat["rural_biomass_boiler"] return marg_cost3 From a768cfa5f39fc576466e9f2779a7c2ef6f964e15 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 25 Mar 2025 10:35:54 +0100 Subject: [PATCH 70/86] include all link time-series --- .../datasets/scenario_path/scenario_path.py | 99 +++++++------------ 1 file changed, 36 insertions(+), 63 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index c1a0700a4..92a29aaef 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -418,37 +418,6 @@ def import_links(scn: str): index=False, ) - dsm3_t = pd.read_sql( - """ - SELECT * FROM grid.egon_etrago_link_timeseries - WHERE link_id IN( - SELECT link_id FROM grid.egon_etrago_link - WHERE bus0 IN ( - SELECT bus_id FROM grid.egon_etrago_bus - WHERE country = 'DE' - AND scn_name = 'eGon100RE' - ) - AND bus1 IN ( - SELECT bus_id FROM grid.egon_etrago_bus - WHERE country = 'DE' - AND scn_name = 'eGon100RE' - ) - AND carrier = 'dsm') - AND scn_name = 'eGon100RE' - """, - con, - ) - - dsm3_t["scn_name"] = scn - - dsm3_t.to_sql( - name="egon_etrago_link_timeseries", - con=con, - schema="grid", - if_exists="append", - index=False, - ) - # dealing with rural_heat_pump link_rhp1 = ( scn1_link[scn1_link["carrier"].isin(["rural_heat_pump"])] @@ -481,37 +450,6 @@ def import_links(scn: str): index=False, ) - link_rhp3_t = pd.read_sql( - """ - SELECT * FROM grid.egon_etrago_link_timeseries - WHERE link_id IN( - SELECT link_id FROM grid.egon_etrago_link - WHERE bus0 IN ( - SELECT bus_id FROM grid.egon_etrago_bus - WHERE country = 'DE' - AND scn_name = 'eGon100RE' - ) - AND bus1 IN ( - SELECT bus_id FROM grid.egon_etrago_bus - WHERE country = 'DE' - AND scn_name = 'eGon100RE' - ) - AND carrier = 'rural_heat_pump') - AND scn_name = 'eGon100RE' - """, - con, - ) - - link_rhp3_t["scn_name"] = scn - - link_rhp3_t.to_sql( - name="egon_etrago_link_timeseries", - con=con, - schema="grid", - if_exists="append", - index=False, - ) - # Dealing with central_gas_boiler link_cgb1 = ( scn1_link[scn1_link["carrier"].isin(["central_gas_boiler"])] @@ -905,6 +843,39 @@ def import_links(scn: str): """ db.execute_sql(sql_update) + # dealing with links time-series + for carrier in scn2_link.carrier.unique(): + ts = pd.read_sql( + """ + SELECT * FROM grid.egon_etrago_link_timeseries + WHERE link_id IN( + SELECT link_id FROM grid.egon_etrago_link + WHERE bus0 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + AND bus1 IN ( + SELECT bus_id FROM grid.egon_etrago_bus + WHERE country = 'DE' + AND scn_name = 'eGon100RE' + ) + AND carrier = '{carrier}') + AND scn_name = 'eGon100RE' + """, + con, + ) + + if len(ts): + ts["scn_name"] = scn + + ts.to_sql( + name="egon_etrago_link_timeseries", + con=con, + schema="grid", + if_exists="append", + index=False, + ) return @@ -1026,7 +997,9 @@ def interpolate_marginal_costs(scn): marg_cost3["run_of_river"] = 0 marg_cost3["solar_rooftop"] = 0.01 - marg_cost3["rural_biomass_boiler"] = marg_cost2_heat["rural_biomass_boiler"] + marg_cost3["rural_biomass_boiler"] = marg_cost2_heat[ + "rural_biomass_boiler" + ] return marg_cost3 From ad7ad49c4be54cc2595e557e15de460629539bbf Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 25 Mar 2025 11:11:55 +0100 Subject: [PATCH 71/86] avoid H2_grid islands --- .../datasets/scenario_path/scenario_path.py | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 92a29aaef..60bd7a02d 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -671,11 +671,45 @@ def import_links(scn: str): geom_col="geom", ).set_index("bus_id") - h2_grid_to_h2 = h2_grid_b[ - ~h2_grid_b.index.isin( - pd.concat([h2_grid3["bus0"], h2_grid3["bus1"]], ignore_index=True) - ) - ].copy() + if scn == "powerd2025": + h2_grid_to_h2 = h2_grid_b + elif scn == "powerd2030": + h2_grid_to_h2 = h2_grid_b[ + ( + ~h2_grid_b.index.isin( + pd.concat( + [h2_grid3["bus0"], h2_grid3["bus1"]], ignore_index=True + ) + ) + ) + | ( + h2_grid_b.index.isin( + [ + 45185, + 45279, + 45075, + 45074, + 45073, + 45066, + 45067, + 45248, + 45071, + 45070, + 45250, + 45168, + 45270, + 45159, + 45160, + 45236, + 45301, + 45278, + 45182, + ] + ) + ) + ].copy() + elif scn == "powerd2035": + h2_grid_to_h2 = [] if len(h2_grid_to_h2) > 0: h2_grid_to_h2["carrier"] = "H2" From 4c2a0af96fe2e4e5028378d44da4f86175445c65 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Tue, 25 Mar 2025 15:32:42 +0100 Subject: [PATCH 72/86] include other buses in islands powerd2030 --- .../data/datasets/scenario_path/scenario_path.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 60bd7a02d..1f42e071d 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -704,6 +704,18 @@ def import_links(scn: str): 45301, 45278, 45182, + 45090, + 45091, + 45092, + 45101, + 45108, + 45253, + 45254, + 45256, + 45324, + 45325, + 45326, + 45357, ] ) ) From c1934d3d49c08f80a7212dd6fb7d4efbbf0c1917 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Wed, 26 Mar 2025 11:03:44 +0100 Subject: [PATCH 73/86] delete H2_grid links from islands --- .../datasets/scenario_path/scenario_path.py | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 1f42e071d..6ee954fc2 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -721,14 +721,32 @@ def import_links(scn: str): ) ].copy() elif scn == "powerd2035": - h2_grid_to_h2 = [] + h2_grid_to_h2 = h2_grid_b[ + ( + ~h2_grid_b.index.isin( + pd.concat( + [h2_grid3["bus0"], h2_grid3["bus1"]], ignore_index=True + ) + ) + ) + | ( + h2_grid_b.index.isin( + [ + 45070, + 45250, + 45278, + 45182, + ] + ) + ) + ].copy() if len(h2_grid_to_h2) > 0: h2_grid_to_h2["carrier"] = "H2" h2_grid_to_h2["scn_name"] = scn # delete H2_grid isolated, associated H2_saltcaverns links, - # H2_saltcavern buses and H2_underground stores + # H2_saltcavern buses, H2_underground stores and H2_grid links (islands) db.execute_sql( f""" DELETE FROM grid.egon_etrago_bus @@ -737,6 +755,17 @@ def import_links(scn: str): """ ) + del_h2_grid_link = pd.read_sql( + f""" + SELECT link_id, bus0, bus1, carrier FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND carrier = 'H2_grid' + AND ((bus0 IN {tuple(h2_grid_to_h2.index)}) OR + (bus1 IN {tuple(h2_grid_to_h2.index)})) + """, + con, + ) + del_h2_salcavern_link = pd.read_sql( f""" SELECT link_id, bus0, bus1, carrier FROM grid.egon_etrago_link @@ -769,6 +798,14 @@ def import_links(scn: str): con, ) + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND link_id IN {tuple(del_h2_grid_link.link_id)} + """ + ) + db.execute_sql( f""" DELETE FROM grid.egon_etrago_link From 9c05a5550c8b96d7e7d5ce4fd27248fb2fff5a8a Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Wed, 26 Mar 2025 12:10:24 +0100 Subject: [PATCH 74/86] execute delete sql only if there is something to delete --- .../datasets/scenario_path/scenario_path.py | 60 ++++++++++--------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 6ee954fc2..eb065f2b9 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -798,37 +798,41 @@ def import_links(scn: str): con, ) - db.execute_sql( - f""" - DELETE FROM grid.egon_etrago_link - WHERE scn_name = '{scn}' - AND link_id IN {tuple(del_h2_grid_link.link_id)} - """ - ) + if len(del_h2_grid_link) > 0: + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND link_id IN {tuple(del_h2_grid_link.link_id)} + """ + ) - db.execute_sql( - f""" - DELETE FROM grid.egon_etrago_link - WHERE scn_name = '{scn}' - AND link_id IN {tuple(del_h2_salcavern_link.link_id)} - """ - ) + if len(del_h2_salcavern_link) > 0: + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND link_id IN {tuple(del_h2_salcavern_link.link_id)} + """ + ) - db.execute_sql( - f""" - DELETE FROM grid.egon_etrago_bus - WHERE scn_name = '{scn}' - AND bus_id IN {tuple(del_h2_salcavern_bus.bus_id)} - """ - ) + if len(del_h2_salcavern_bus) > 0: + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND bus_id IN {tuple(del_h2_salcavern_bus.bus_id)} + """ + ) - db.execute_sql( - f""" - DELETE FROM grid.egon_etrago_store - WHERE scn_name = '{scn}' - AND store_id IN {tuple(del_h2_underground_store.store_id)} - """ - ) + if len(del_h2_underground_store) > 0: + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_store + WHERE scn_name = '{scn}' + AND store_id IN {tuple(del_h2_underground_store.store_id)} + """ + ) ch4_b = gpd.read_postgis( f""" From 7983624bdf314bd08d970e99bd576617b21e7e55 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Wed, 26 Mar 2025 13:39:25 +0100 Subject: [PATCH 75/86] fix empty list in sql queries --- .../datasets/scenario_path/scenario_path.py | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index eb065f2b9..c7b6eaf76 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -777,26 +777,32 @@ def import_links(scn: str): con, ) - del_h2_salcavern_bus = pd.read_sql( - f""" - SELECT bus_id, carrier FROM grid.egon_etrago_bus - WHERE scn_name = '{scn}' - AND carrier = 'H2_saltcavern' - AND ((bus_id IN {tuple(del_h2_salcavern_link.bus0)}) OR - (bus_id IN {tuple(del_h2_salcavern_link.bus1)})) - """, - con, - ) + if len(del_h2_salcavern_link) > 0: + del_h2_salcavern_bus = pd.read_sql( + f""" + SELECT bus_id, carrier FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = 'H2_saltcavern' + AND ((bus_id IN {tuple(del_h2_salcavern_link.bus0)}) OR + (bus_id IN {tuple(del_h2_salcavern_link.bus1)})) + """, + con, + ) + else: + del_h2_salcavern_bus = [] - del_h2_underground_store = pd.read_sql( - f""" - SELECT store_id, carrier, bus FROM grid.egon_etrago_store - WHERE scn_name = '{scn}' - AND carrier = 'H2_underground' - AND bus IN {tuple(del_h2_salcavern_bus.bus_id)} - """, - con, - ) + if len(del_h2_salcavern_bus) > 0: + del_h2_underground_store = pd.read_sql( + f""" + SELECT store_id, carrier, bus FROM grid.egon_etrago_store + WHERE scn_name = '{scn}' + AND carrier = 'H2_underground' + AND bus IN {tuple(del_h2_salcavern_bus.bus_id)} + """, + con, + ) + else: + del_h2_underground_store = [] if len(del_h2_grid_link) > 0: db.execute_sql( From 030e7978be984cc55672823aa75a3069cb271c5d Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Wed, 26 Mar 2025 15:14:52 +0100 Subject: [PATCH 76/86] delete cossbording H2_grid links connected to H2 buses --- .../datasets/scenario_path/scenario_path.py | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index c7b6eaf76..de84de447 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -671,6 +671,16 @@ def import_links(scn: str): geom_col="geom", ).set_index("bus_id") + h2_grid_links = gpd.read_postgis( + f""" + SELECT * FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND carrier = 'H2_grid' + """, + con, + geom_col="geom", + ).set_index("link_id") + if scn == "powerd2025": h2_grid_to_h2 = h2_grid_b elif scn == "powerd2030": @@ -678,7 +688,8 @@ def import_links(scn: str): ( ~h2_grid_b.index.isin( pd.concat( - [h2_grid3["bus0"], h2_grid3["bus1"]], ignore_index=True + [h2_grid_links["bus0"], h2_grid_links["bus1"]], + ignore_index=True, ) ) ) @@ -904,6 +915,34 @@ def import_links(scn: str): index=False, ) + # delete H2_grid and H2_pipeline links connected to H2 buses + + delete_pipe = pd.read_sql( + f""" + SELECT link_id, bus0, bus1, carrier FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND carrier IN ('H2_grid', 'H2_pipeline') + AND ((bus0 IN (SELECT bus_id FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = 'H2' + AND country = 'DE')) OR + (bus1 IN (SELECT bus_id FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = 'H2' + AND country = 'DE'))) + """, + con, + ) + + if len(delete_pipe) > 0: + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND link_id IN {tuple(delete_pipe.link_id)} + """ + ) + ###adjust electrolyzer parameters according to Fraunhofer ISE efficiency = { "powerd2025": 0.6535, From 401cf898c9fa06640c7fb7d1bbc37941ccc914ff Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Wed, 26 Mar 2025 15:53:34 +0100 Subject: [PATCH 77/86] move function to delete cossborder H2_grid links --- .../datasets/scenario_path/scenario_path.py | 55 +++++++++---------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index de84de447..94cde8d7b 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -915,34 +915,6 @@ def import_links(scn: str): index=False, ) - # delete H2_grid and H2_pipeline links connected to H2 buses - - delete_pipe = pd.read_sql( - f""" - SELECT link_id, bus0, bus1, carrier FROM grid.egon_etrago_link - WHERE scn_name = '{scn}' - AND carrier IN ('H2_grid', 'H2_pipeline') - AND ((bus0 IN (SELECT bus_id FROM grid.egon_etrago_bus - WHERE scn_name = '{scn}' - AND carrier = 'H2' - AND country = 'DE')) OR - (bus1 IN (SELECT bus_id FROM grid.egon_etrago_bus - WHERE scn_name = '{scn}' - AND carrier = 'H2' - AND country = 'DE'))) - """, - con, - ) - - if len(delete_pipe) > 0: - db.execute_sql( - f""" - DELETE FROM grid.egon_etrago_link - WHERE scn_name = '{scn}' - AND link_id IN {tuple(delete_pipe.link_id)} - """ - ) - ###adjust electrolyzer parameters according to Fraunhofer ISE efficiency = { "powerd2025": 0.6535, @@ -2207,3 +2179,30 @@ def import_foreign(scn: str, year): if_exists="append", index=False, ) + + # delete H2_grid and H2_pipeline links connected to H2 buses + delete_pipe = pd.read_sql( + f""" + SELECT link_id, bus0, bus1, carrier FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND carrier IN ('H2_grid', 'H2_pipeline') + AND ((bus0 IN (SELECT bus_id FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = 'H2' + AND country = 'DE')) OR + (bus1 IN (SELECT bus_id FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = 'H2' + AND country = 'DE'))) + """, + con, + ) + + if len(delete_pipe) > 0: + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND link_id IN {tuple(delete_pipe.link_id)} + """ + ) From bf47f64ce4f67fc28c29eb4ec02e1684a16f6d9f Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 28 Mar 2025 11:46:40 +0100 Subject: [PATCH 78/86] fix missing links timeseries --- src/egon/data/datasets/scenario_path/scenario_path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 94cde8d7b..f70f5b4e5 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -950,7 +950,7 @@ def import_links(scn: str): # dealing with links time-series for carrier in scn2_link.carrier.unique(): ts = pd.read_sql( - """ + f""" SELECT * FROM grid.egon_etrago_link_timeseries WHERE link_id IN( SELECT link_id FROM grid.egon_etrago_link From ce7e739f7bd354947b06ffa1457abcc82eb801b2 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 28 Mar 2025 15:38:25 +0100 Subject: [PATCH 79/86] skip central_heat_pump links --- src/egon/data/datasets/scenario_path/scenario_path.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index f70f5b4e5..c25fd205c 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -949,6 +949,8 @@ def import_links(scn: str): # dealing with links time-series for carrier in scn2_link.carrier.unique(): + if carrier == "central_heat_pump": + continue ts = pd.read_sql( f""" SELECT * FROM grid.egon_etrago_link_timeseries From b91c024137f18cd45495f5ec2c917b3a415759f1 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 28 Mar 2025 15:40:55 +0100 Subject: [PATCH 80/86] delete unused imports --- src/egon/data/datasets/scenario_path/scenario_path.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index c25fd205c..8c92137bc 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1,15 +1,12 @@ from pathlib import Path -from urllib.request import urlretrieve -import os -import subprocess import numpy as np import pandas as pd import pypsa import geopandas as gpd -from shapely.geometry import Point, LineString +from shapely.geometry import LineString -from egon.data import config, db +from egon.data import db import egon.data.config from egon.data.datasets.pypsaeur import neighbor_reduction, prepared_network From 38f50a4c06362059d1e1386a04d0215bd3a04bb9 Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 28 Mar 2025 15:43:31 +0100 Subject: [PATCH 81/86] using isort --- src/egon/data/datasets/scenario_path/scenario_path.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/scenario_path.py b/src/egon/data/datasets/scenario_path/scenario_path.py index 8c92137bc..404baf2a7 100644 --- a/src/egon/data/datasets/scenario_path/scenario_path.py +++ b/src/egon/data/datasets/scenario_path/scenario_path.py @@ -1,19 +1,18 @@ from pathlib import Path +from shapely.geometry import LineString +import geopandas as gpd import numpy as np import pandas as pd import pypsa -import geopandas as gpd -from shapely.geometry import LineString from egon.data import db -import egon.data.config - from egon.data.datasets.pypsaeur import neighbor_reduction, prepared_network from egon.data.datasets.scenario_parameters import get_sector_parameters from egon.data.datasets.scenario_parameters.parameters import ( annualize_capital_costs, ) +import egon.data.config sources = egon.data.config.datasets()["scenario_path"]["sources"] From 709310fe05521aeaad2738f47f074f88e92bf0ce Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Fri, 28 Mar 2025 15:44:33 +0100 Subject: [PATCH 82/86] using isort --- src/egon/data/datasets/scenario_path/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/scenario_path/__init__.py b/src/egon/data/datasets/scenario_path/__init__.py index 342279875..fbb3ce9e1 100644 --- a/src/egon/data/datasets/scenario_path/__init__.py +++ b/src/egon/data/datasets/scenario_path/__init__.py @@ -1,13 +1,13 @@ from egon.data.datasets import Dataset from egon.data.datasets.scenario_path.scenario_path import ( clean_existing_scn_path_data, - import_network_structure, + import_foreign, import_generators, - import_loads, import_links, + import_loads, + import_network_structure, import_storage_units, import_stores, - import_foreign, ) From b3306baf744dd5ffab783d7a570da547a762044d Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Wed, 2 Apr 2025 14:23:12 +0200 Subject: [PATCH 83/86] using black --- src/egon/data/datasets/pypsaeur/__init__.py | 200 ++++++++++-------- .../scenario_parameters/parameters.py | 152 ++++++------- 2 files changed, 187 insertions(+), 165 deletions(-) diff --git a/src/egon/data/datasets/pypsaeur/__init__.py b/src/egon/data/datasets/pypsaeur/__init__.py index f07a735ae..b231a3d48 100755 --- a/src/egon/data/datasets/pypsaeur/__init__.py +++ b/src/egon/data/datasets/pypsaeur/__init__.py @@ -575,116 +575,128 @@ def combine_decentral_and_rural_heat(network_solved, network_prepared): return network_prepared, network_solved - def combine_residenial_services_heat_load(network_prepared): ###combining loads residential_rural_loads = network_prepared.loads[ network_prepared.loads.carrier.str.contains("residential rural heat") ] - + for i, row in residential_rural_loads.iterrows(): if i in network_prepared.loads_t.p_set.columns: network_prepared.loads_t.p_set[ i.replace("residential", "services") ] += network_prepared.loads_t.p_set[i] network_prepared.mremove("Load", residential_rural_loads.index) - - + target_carriers = ["services rural heat"] rural_heat_loads = network_prepared.loads[ network_prepared.loads.carrier.isin(target_carriers) ] - - #rename carrier - network_prepared.loads.loc[rural_heat_loads.index, "carrier"] = "rural heat" - - #rename bus + + # rename carrier + network_prepared.loads.loc[rural_heat_loads.index, "carrier"] = ( + "rural heat" + ) + + # rename bus network_prepared.loads.loc[rural_heat_loads.index, "bus"] = ( network_prepared.loads.loc[rural_heat_loads.index, "bus"] .str.replace("services rural heat", "rural heat") .str.replace("residential rural heat", "rural heat") ) - - #rename index + + # rename index network_prepared.loads.rename( - index=lambda x: x.replace("services rural heat", "rural heat") - .replace("residential rural heat", "rural heat"), - inplace=True + index=lambda x: x.replace("services rural heat", "rural heat").replace( + "residential rural heat", "rural heat" + ), + inplace=True, ) - - #rename timeseries columns + + # rename timeseries columns network_prepared.loads_t.p_set.columns = ( - network_prepared.loads_t.p_set.columns.str.replace("services rural heat", "rural heat") - .str.replace("residential rural heat", "rural heat") - ) - + network_prepared.loads_t.p_set.columns.str.replace( + "services rural heat", "rural heat" + ).str.replace("residential rural heat", "rural heat") + ) + ####combining heat_pumps - network_prepared.links_t.efficiency.columns = (network_prepared.links_t.efficiency.columns.str.replace("services rural", "rural heat") - .str.replace("residential rural", "rural") - ) - - return network_prepared + network_prepared.links_t.efficiency.columns = ( + network_prepared.links_t.efficiency.columns.str.replace( + "services rural", "rural heat" + ).str.replace("residential rural", "rural") + ) + return network_prepared def import_missing_gens(neighbors, network_solved, scn_name): - carriers_to_keep=['oil', 'lignite', 'coal', 'urban central solid biomass CHP'] + carriers_to_keep = [ + "oil", + "lignite", + "coal", + "urban central solid biomass CHP", + ] marg = margina_cost_missing_gens() - - for carrier in carriers_to_keep: + + for carrier in carriers_to_keep: links = network_solved.links[network_solved.links.carrier == carrier] - links_neighbor = links[ - links['bus1'].isin(neighbors.index) - ] - marg_cost= marg[scn_name][carrier] + links_neighbor = links[links["bus1"].isin(neighbors.index)] + marg_cost = marg[scn_name][carrier] print(marg_cost) for idx, link in links_neighbor.iterrows(): - if carrier == 'urban central solid biomass CHP': - network_solved.add("Generator", f"gen_{idx}_electrical", - bus=link.bus1, - p_nom_opt= link.p_nom_opt*link.efficiency, + if carrier == "urban central solid biomass CHP": + network_solved.add( + "Generator", + f"gen_{idx}_electrical", + bus=link.bus1, + p_nom_opt=link.p_nom_opt * link.efficiency, carrier="central_biomass_CHP", - marginal_cost=marg_cost - ) - network_solved.add("Generator", f"gen_{idx}_heat", + marginal_cost=marg_cost, + ) + network_solved.add( + "Generator", + f"gen_{idx}_heat", bus=link.bus2, - p_nom_opt=link.p_nom_opt*link.efficiency2, + p_nom_opt=link.p_nom_opt * link.efficiency2, carrier="central_biomass_CHP_heat", - ) - else: - network_solved.add("Generator", f"gen_{idx}", + ) + else: + network_solved.add( + "Generator", + f"gen_{idx}", bus=link.bus1, - p_nom_opt=link.p_nom_opt*link.efficiency, + p_nom_opt=link.p_nom_opt * link.efficiency, carrier=carrier, - marginal_cost=marg_cost - ) + marginal_cost=marg_cost, + ) return network_solved - + + def margina_cost_missing_gens(): marginal_costs = { - "powerd2025": { - "oil": 164.90901098901102, - "lignite": 67.38601398601398, - "coal": 76.07459207459208, - "urban central solid biomass CHP": 39.69634478996181 - }, - "powerd2030": { - "oil": 169.8246153846154, - "lignite": 86.11148018648018, - "coal": 88.3854895104895, - "urban central solid biomass CHP": 51.573854337152206 - }, - "powerd2035": { - "oil": 174.7402197802198, - "lignite": 104.834, - "coal": 100.7, - "urban central solid biomass CHP": 63.451363884342605 - } - } - - return marginal_costs + "powerd2025": { + "oil": 164.90901098901102, + "lignite": 67.38601398601398, + "coal": 76.07459207459208, + "urban central solid biomass CHP": 39.69634478996181, + }, + "powerd2030": { + "oil": 169.8246153846154, + "lignite": 86.11148018648018, + "coal": 88.3854895104895, + "urban central solid biomass CHP": 51.573854337152206, + }, + "powerd2035": { + "oil": 174.7402197802198, + "lignite": 104.834, + "coal": 100.7, + "urban central solid biomass CHP": 63.451363884342605, + }, + } + return marginal_costs def neighbor_reduction(scn_name, year=2045): @@ -716,7 +728,7 @@ def neighbor_reduction(scn_name, year=2045): network_solved.buses = network_solved.buses.drop( network_solved.buses.loc[foreign_buses.index].index ) - + # Set country tag for all buses network_solved.buses.country = network_solved.buses.index.str[:2] neighbors = network_solved.buses[network_solved.buses.country != "DE"] @@ -725,12 +737,13 @@ def neighbor_reduction(scn_name, year=2045): db.next_etrago_id("bus") + neighbors.reset_index().index ) - #keep links that are connected to an central EU-bus(solid biomass, oil, llignite, coal) - #transform them to generators - if scn_name !='eGon100RE': - network_solved = import_missing_gens(neighbors, network_solved, scn_name) + # keep links that are connected to an central EU-bus(solid biomass, oil, llignite, coal) + # transform them to generators + if scn_name != "eGon100RE": + network_solved = import_missing_gens( + neighbors, network_solved, scn_name + ) - # Add H2 demand of Fischer-Tropsch process and methanolisation # to industrial H2 demands industrial_hydrogen = network_prepared.loads.loc[ @@ -745,7 +758,7 @@ def neighbor_reduction(scn_name, year=2045): .mul(network_solved.snapshot_weightings.generators, axis=0) .sum() ) - + methanolisation = ( network_solved.links_t.p0[ network_solved.links.loc[ @@ -1027,11 +1040,13 @@ def neighbor_reduction(scn_name, year=2045): # loads # imported from prenetwork in 1h-resolution - - #adjusting index-, bus- and carrier-name for loads for powerd2025 - if scn_name=='powerd2025': - network_prepared = combine_residenial_services_heat_load(network_prepared) - + + # adjusting index-, bus- and carrier-name for loads for powerd2025 + if scn_name == "powerd2025": + network_prepared = combine_residenial_services_heat_load( + network_prepared + ) + neighbor_loads = network_prepared.loads[ network_prepared.loads.bus.isin(neighbors.index) ] @@ -1171,7 +1186,7 @@ def neighbor_reduction(scn_name, year=2045): index=True, index_label="bus_id", ) - + # prepare and write neighboring crossborder lines to etrago tables def lines_to_etrago(neighbor_lines=neighbor_lines, scn=scn_name): neighbor_lines["scn_name"] = scn @@ -1209,9 +1224,9 @@ def lines_to_etrago(neighbor_lines=neighbor_lines, scn=scn_name): .set_crs(4326) ) - neighbor_lines["lifetime"] = get_sector_parameters("electricity", 'eGon100RE')[ - "lifetime" - ]["ac_ehv_overhead_line"] + neighbor_lines["lifetime"] = get_sector_parameters( + "electricity", "eGon100RE" + )["lifetime"]["ac_ehv_overhead_line"] neighbor_lines.to_postgis( "egon_etrago_line", @@ -1393,7 +1408,7 @@ def links_to_etrago(neighbor_links, scn=scn_name, extendable=True): neighbor_links = neighbor_links[ ~neighbor_links.carrier.isin(excluded_carriers) ] - + # Combine CHP_CC and CHP chp_cc = neighbor_links[ neighbor_links.carrier == "urban central gas CHP CC" @@ -1407,22 +1422,24 @@ def links_to_etrago(neighbor_links, scn=scn_name, extendable=True): neighbor_links.Link == row.Link.replace("CHP CC", "CHP"), "p_nom" ] += row.p_nom neighbor_links.drop(index, inplace=True) - + # Combine heat pumps # Like in Germany, there are air heat pumps in central heat grids # and ground heat pumps in rural areas - - #consider aso urban decentral air heat pumps in rural_heat_pumps + + # consider aso urban decentral air heat pumps in rural_heat_pumps mask = neighbor_links["Link"].str.contains("urban decentral air", na=False) - neighbor_links.loc[mask, "Link"] = neighbor_links.loc[mask, "Link"].str.replace("urban decentral", "rural") + neighbor_links.loc[mask, "Link"] = neighbor_links.loc[ + mask, "Link" + ].str.replace("urban decentral", "rural") rural_air = neighbor_links[neighbor_links.carrier == "rural air heat pump"] - + for index, row in rural_air.iterrows(): neighbor_links.loc[ neighbor_links.Link == row.Link.replace("air", "ground"), "p_nom_opt", ] += row.p_nom_opt - + neighbor_links.loc[ neighbor_links.Link == row.Link.replace("air", "ground"), "p_nom" ] += row.p_nom @@ -1547,7 +1564,6 @@ def links_to_etrago(neighbor_links, scn=scn_name, extendable=True): columns=["Load"], errors="ignore", ) - neighbor_loads.to_sql( "egon_etrago_load", diff --git a/src/egon/data/datasets/scenario_parameters/parameters.py b/src/egon/data/datasets/scenario_parameters/parameters.py index 36fb30bce..6162a0838 100755 --- a/src/egon/data/datasets/scenario_parameters/parameters.py +++ b/src/egon/data/datasets/scenario_parameters/parameters.py @@ -130,15 +130,15 @@ def global_settings(scenario): "weather_year": 2011, "population_year": 2019, "fuel_costs": { # TYNDP 2020, data for 2020 (https://2020.entsos-tyndp-scenarios.eu/fuel-commodities-and-carbon-prices/) - "oil": 12.9*3.6, # [EUR/MWh] - "gas": 5.6*3.6, # [EUR/MWh] - "coal": 3.0*3.6, # [EUR/MWh] - "lignite": 1.1*3.6, # [EUR/MWh] - "nuclear": 0.47*3.6, # [EUR/MWh] + "oil": 12.9 * 3.6, # [EUR/MWh] + "gas": 5.6 * 3.6, # [EUR/MWh] + "coal": 3.0 * 3.6, # [EUR/MWh] + "lignite": 1.1 * 3.6, # [EUR/MWh] + "nuclear": 0.47 * 3.6, # [EUR/MWh] "biomass": read_costs(read_csv(2020), "biomass", "fuel"), }, - "co2_costs": 24.7, # [EUR/t_CO2], source: - #https://de.statista.com/statistik/daten/studie/1304069/umfrage/preisentwicklung-von-co2-emissionsrechten-in-eu/ + "co2_costs": 24.7, # [EUR/t_CO2], source: + # https://de.statista.com/statistik/daten/studie/1304069/umfrage/preisentwicklung-von-co2-emissionsrechten-in-eu/ "co2_emissions": { # Netzentwicklungsplan Strom 2035, Version 2021, 1. Entwurf, p. 40, table 8 "waste": 0.165, # [t_CO2/MW_th] "lignite": 0.393, # [t_CO2/MW_th] @@ -325,11 +325,11 @@ def electricity(scenario): / read_costs(costs, "oil", "efficiency"), "other_non_renewable": global_settings(scenario)["fuel_costs"][ "gas" - ] / read_costs(costs, "OCGT", "efficiency") + ] + / read_costs(costs, "OCGT", "efficiency") + global_settings(scenario)["co2_costs"] - * global_settings(scenario)["co2_emissions"][ - "other_non_renewable" - ] / read_costs(costs, "OCGT", "efficiency"), + * global_settings(scenario)["co2_emissions"]["other_non_renewable"] + / read_costs(costs, "OCGT", "efficiency"), "lignite": global_settings(scenario)["fuel_costs"]["lignite"] / read_costs(costs, "lignite", "efficiency") + read_costs(costs, "lignite", "VOM") @@ -501,11 +501,11 @@ def electricity(scenario): / read_costs(costs, "oil", "efficiency"), "other_non_renewable": global_settings(scenario)["fuel_costs"][ "gas" - ] / read_costs(costs, "OCGT", "efficiency") + ] + / read_costs(costs, "OCGT", "efficiency") + global_settings(scenario)["co2_costs"] - * global_settings(scenario)["co2_emissions"][ - "other_non_renewable" - ] / read_costs(costs, "OCGT", "efficiency"), + * global_settings(scenario)["co2_emissions"]["other_non_renewable"] + / read_costs(costs, "OCGT", "efficiency"), "lignite": global_settings(scenario)["fuel_costs"]["lignite"] / read_costs(costs, "lignite", "efficiency") + read_costs(costs, "lignite", "VOM") @@ -684,11 +684,11 @@ def electricity(scenario): / read_costs(costs, "oil", "efficiency"), "other_non_renewable": global_settings(scenario)["fuel_costs"][ "gas" - ] / read_costs(costs, "OCGT", "efficiency") + ] + / read_costs(costs, "OCGT", "efficiency") + global_settings(scenario)["co2_costs"] - * global_settings(scenario)["co2_emissions"][ - "other_non_renewable" - ] / read_costs(costs, "OCGT", "efficiency"), + * global_settings(scenario)["co2_emissions"]["other_non_renewable"] + / read_costs(costs, "OCGT", "efficiency"), "lignite": global_settings(scenario)["fuel_costs"]["lignite"] / read_costs(costs, "lignite", "efficiency") + read_costs(costs, "lignite", "VOM") @@ -746,24 +746,26 @@ def gas(scenario): "main_gas_carrier": "CH4", "H2_feedin_volumetric_fraction": 0.15, } - + # Insert effciencies in p.u. parameters["efficiency"] = { - "power_to_H2": 0.6805, #source: project internal assumption Fraunhofer ISE + "power_to_H2": 0.6805, # source: project internal assumption Fraunhofer ISE "H2_to_power": read_costs(costs, "fuel cell", "efficiency"), "CH4_to_H2": read_costs(costs, "SMR", "efficiency"), "H2_feedin": 1, "H2_to_CH4": read_costs(costs, "methanation", "efficiency"), "OCGT": read_costs(costs, "OCGT", "efficiency"), - "power_to_Heat": 0.2, #overall efficiency (20% electrical Input converted into waste-heat); source: project internal assumption Fraunhofer ISE - "power_to_O2": 0.04, #O2-transfer efficiency; source: Sayed Sadat, Modeling Regional Utilization of the electrolysers Co-Products Oxygen and Heat in Germany, 2024 + "power_to_Heat": 0.2, # overall efficiency (20% electrical Input converted into waste-heat); source: project internal assumption Fraunhofer ISE + "power_to_O2": 0.04, # O2-transfer efficiency; source: Sayed Sadat, Modeling Regional Utilization of the electrolysers Co-Products Oxygen and Heat in Germany, 2024 } - + # Insert overnight investment costs parameters["overnight_cost"] = { - "power_to_H2_system": 452_000, #[EUR/MW] source: project internal assumption Fraunhofer ISE - "power_to_H2_stack": 0.21 * 452_000, #[EUR/MW] source: project internal assumption Fraunhofer ISE - "power_to_H2_OPEX": 0.03 * 452_000, #[EUR/MW/a] 3% of CAPEX, source: project internal assumption Fraunhofer ISE + "power_to_H2_system": 452_000, # [EUR/MW] source: project internal assumption Fraunhofer ISE + "power_to_H2_stack": 0.21 + * 452_000, # [EUR/MW] source: project internal assumption Fraunhofer ISE + "power_to_H2_OPEX": 0.03 + * 452_000, # [EUR/MW/a] 3% of CAPEX, source: project internal assumption Fraunhofer ISE "H2_to_power": read_costs(costs, "fuel cell", "investment"), "CH4_to_H2": read_costs(costs, "SMR", "investment"), "H2_to_CH4": read_costs(costs, "methanation", "investment"), @@ -774,13 +776,15 @@ def gas(scenario): "H2_overground": read_costs( costs, "hydrogen storage tank incl. compressor", "investment" ), - "H2_pipeline": read_costs(costs, "H2 (g) pipeline", "investment"), # [EUR/MW/km] - "Heat_exchanger": 25_000, # [EUR/MW_th] cost assumption for one additional heat_exchanger; source: project internal cost assumption by Fraunhofer ISE - "Heat_pipeline": 400_000, # [EUR/MW/km]; average value for DN100-pipeline; source: L. Zimmermann, MODELLIERUNG DER ABWÄRMENUTZUNG VON ELEKTROLYSEUREN IN DEUTSCHLAND FÜR EINE TECHNO - ÖKONOMISCHE OPTIMIERUNG EINES SEKTOR - GEKOPPELTEN ENERGIESYSTEM, 2024 - "O2_components": 5000, # [EUR] ; source: Sayed Sadat, Modeling Regional Utilization of the electrolysers Co-Products Oxygen and Heat in Germany, 2024 + "H2_pipeline": read_costs( + costs, "H2 (g) pipeline", "investment" + ), # [EUR/MW/km] + "Heat_exchanger": 25_000, # [EUR/MW_th] cost assumption for one additional heat_exchanger; source: project internal cost assumption by Fraunhofer ISE + "Heat_pipeline": 400_000, # [EUR/MW/km]; average value for DN100-pipeline; source: L. Zimmermann, MODELLIERUNG DER ABWÄRMENUTZUNG VON ELEKTROLYSEUREN IN DEUTSCHLAND FÜR EINE TECHNO - ÖKONOMISCHE OPTIMIERUNG EINES SEKTOR - GEKOPPELTEN ENERGIESYSTEM, 2024 + "O2_components": 5000, # [EUR] ; source: Sayed Sadat, Modeling Regional Utilization of the electrolysers Co-Products Oxygen and Heat in Germany, 2024 } - - #overnight_costs for O2_pipeinecosts related to pipeline_diameter + + # overnight_costs for O2_pipeinecosts related to pipeline_diameter parameters["O2_pipeline_costs"] = { 0.5: 500_000, # EUR/km 0.4: 450_000, # EUR/km @@ -788,12 +792,12 @@ def gas(scenario): 0.2: 350_000, # EUR/km 0.0: 300_000, # EUR/km (costs for any other pipeline diameter) } - + # Insert lifetime parameters["lifetime"] = { "power_to_H2_system": 25, # source: project internal assumption Fraunhofer ISE - "power_to_H2_stack": 15, #85000 hours ~ 15 years; source: project internal assumption Fraunhofer ISE - "power_to_H2_OPEX": 1, #given as OPEX/year + "power_to_H2_stack": 15, # 85000 hours ~ 15 years; source: project internal assumption Fraunhofer ISE + "power_to_H2_OPEX": 1, # given as OPEX/year "H2_to_power": read_costs(costs, "fuel cell", "lifetime"), "CH4_to_H2": read_costs(costs, "SMR", "lifetime"), "H2_to_CH4": read_costs(costs, "methanation", "lifetime"), @@ -804,15 +808,15 @@ def gas(scenario): "H2_overground": read_costs( costs, "hydrogen storage tank incl. compressor", "lifetime" ), - "H2_pipeline": read_costs(costs, "H2 (g) pipeline", "lifetime"), - "Heat_exchanger": 20, # assumption based on lifetime heat_exchanger; source: E. van der Roest, R. Bol, T. Fens und A. van Wijk, „Utilisation of waste heat from PEM electrolysers - Unlocking local optimisation, 2023 + "H2_pipeline": read_costs(costs, "H2 (g) pipeline", "lifetime"), + "Heat_exchanger": 20, # assumption based on lifetime heat_exchanger; source: E. van der Roest, R. Bol, T. Fens und A. van Wijk, „Utilisation of waste heat from PEM electrolysers - Unlocking local optimisation, 2023 "Heat_pipeline": 20, - "O2_components": 25, # source: Sayed Sadat, Modeling Regional Utilization of the electrolysers Co-Products Oxygen and Heat in Germany, 2024 + "O2_components": 25, # source: Sayed Sadat, Modeling Regional Utilization of the electrolysers Co-Products Oxygen and Heat in Germany, 2024 } # Insert annualized capital costs parameters["capital_cost"] = {} - parameters["O2_capital_cost"]= {} + parameters["O2_capital_cost"] = {} for comp in parameters["overnight_cost"].keys(): parameters["capital_cost"][comp] = annualize_capital_costs( @@ -820,14 +824,14 @@ def gas(scenario): parameters["lifetime"][comp], global_settings("eGon2035")["interest_rate"], ) - + for diameter in parameters["O2_pipeline_costs"].keys(): parameters["O2_capital_cost"][diameter] = annualize_capital_costs( parameters["O2_pipeline_costs"][diameter], parameters["lifetime"]["O2_components"], global_settings("eGon2035")["interest_rate"], ) - + parameters["marginal_cost"] = { "CH4": global_settings(scenario)["fuel_costs"]["gas"] + global_settings(scenario)["co2_costs"] @@ -859,8 +863,8 @@ def gas(scenario): "CH4_to_H2": read_costs(costs, "SMR", "efficiency"), "H2_to_CH4": read_costs(costs, "methanation", "efficiency"), "OCGT": read_costs(costs, "OCGT", "efficiency"), - "power_to_Heat": 0.2, # source: project internal assumption Fraunhofer ISE - "power_to_O2": 0.015, # source: Sayed Sadat, Modeling Regional Utilization of the electrolysers Co-Products Oxygen and Heat in Germany, 2024 + "power_to_Heat": 0.2, # source: project internal assumption Fraunhofer ISE + "power_to_O2": 0.015, # source: Sayed Sadat, Modeling Regional Utilization of the electrolysers Co-Products Oxygen and Heat in Germany, 2024 } # Insert FOM in % @@ -871,24 +875,25 @@ def gas(scenario): "H2_overground": read_costs( costs, "hydrogen storage tank incl. compressor", "FOM" ), - "power_to_H2_system": 3, #3% of CAPEX, source: project internal assumption Fraunhofer ISE - "power_to_H2_stack": 3, #3% of CAPEX source: project internal assumption Fraunhofer ISE + "power_to_H2_system": 3, # 3% of CAPEX, source: project internal assumption Fraunhofer ISE + "power_to_H2_stack": 3, # 3% of CAPEX source: project internal assumption Fraunhofer ISE "H2_to_power": read_costs(costs, "fuel cell", "FOM"), "CH4_to_H2": read_costs(costs, "SMR", "FOM"), "H2_to_CH4": read_costs(costs, "methanation", "FOM"), - "H2_pipeline": 3, # 3% of CAPEX - "Heat_exchanger": 3, # 3% of CAPEX - "Heat_pipeline": 3, # 3% of CAPEX - "O2_components": 3, # 3% of CAPEX + "H2_pipeline": 3, # 3% of CAPEX + "Heat_exchanger": 3, # 3% of CAPEX + "Heat_pipeline": 3, # 3% of CAPEX + "O2_components": 3, # 3% of CAPEX "H2_pipeline_retrofit": read_costs( costs, "H2 (g) pipeline repurposed", "FOM" ), } - + # Insert overnight investment costs parameters["overnight_cost"] = { - "power_to_H2_system": 357_000, #[EUR/MW] source: project internal assumption Fraunhofer ISE - "power_to_H2_stack": 0.21 * 357_000, #[EUR/MW] source: project internal assumption Fraunhofer ISE + "power_to_H2_system": 357_000, # [EUR/MW] source: project internal assumption Fraunhofer ISE + "power_to_H2_stack": 0.21 + * 357_000, # [EUR/MW] source: project internal assumption Fraunhofer ISE "H2_to_power": read_costs(costs, "fuel cell", "investment"), "CH4_to_H2": read_costs(costs, "SMR", "investment"), "H2_to_CH4": read_costs(costs, "methanation", "investment"), @@ -898,16 +903,18 @@ def gas(scenario): "H2_overground": read_costs( costs, "hydrogen storage tank incl. compressor", "investment" ), - "H2_pipeline": read_costs(costs, "H2 (g) pipeline", "investment"), # [EUR/MW/km] + "H2_pipeline": read_costs( + costs, "H2 (g) pipeline", "investment" + ), # [EUR/MW/km] "H2_pipeline_retrofit": read_costs( costs, "H2 (g) pipeline repurposed", "FOM" ), - "Heat_exchanger": 25_000, # [EUR/MW_th] cost assumption for one additional heat_exchanger; source: project internal cost assumption by Fraunhofer ISE - "Heat_pipeline": 400_000, # [EUR/MW/km]; average value for DN100-pipeline; source: L. Zimmermann, MODELLIERUNG DER ABWÄRMENUTZUNG VON ELEKTROLYSEUREN IN DEUTSCHLAND FÜR EINE TECHNO - ÖKONOMISCHE OPTIMIERUNG EINES SEKTOR - GEKOPPELTEN ENERGIESYSTEM, 2024 - "O2_components": 5000, # [EUR] ; source toDO: ask sayed + "Heat_exchanger": 25_000, # [EUR/MW_th] cost assumption for one additional heat_exchanger; source: project internal cost assumption by Fraunhofer ISE + "Heat_pipeline": 400_000, # [EUR/MW/km]; average value for DN100-pipeline; source: L. Zimmermann, MODELLIERUNG DER ABWÄRMENUTZUNG VON ELEKTROLYSEUREN IN DEUTSCHLAND FÜR EINE TECHNO - ÖKONOMISCHE OPTIMIERUNG EINES SEKTOR - GEKOPPELTEN ENERGIESYSTEM, 2024 + "O2_components": 5000, # [EUR] ; source toDO: ask sayed } - - #overnight_costs for O2_pipeinecosts related to pipeline_diameter + + # overnight_costs for O2_pipeinecosts related to pipeline_diameter parameters["O2_pipeline_costs"] = { 0.5: 500_000, # EUR/km 0.4: 450_000, # EUR/km @@ -919,7 +926,7 @@ def gas(scenario): # Insert lifetime parameters["lifetime"] = { "power_to_H2_system": 30, # source: project internal assumption Fraunhofer ISE - "power_to_H2_stack": 20, #110_000 hours ~ 20 years; source: project internal assumption Fraunhofer ISE + "power_to_H2_stack": 20, # 110_000 hours ~ 20 years; source: project internal assumption Fraunhofer ISE "H2_to_power": read_costs(costs, "fuel cell", "lifetime"), "CH4_to_H2": read_costs(costs, "SMR", "lifetime"), "H2_to_CH4": read_costs(costs, "methanation", "lifetime"), @@ -930,19 +937,19 @@ def gas(scenario): "H2_overground": read_costs( costs, "hydrogen storage tank incl. compressor", "lifetime" ), - "H2_pipeline": read_costs(costs, "H2 (g) pipeline", "lifetime"), + "H2_pipeline": read_costs(costs, "H2 (g) pipeline", "lifetime"), "H2_pipeline_retrofit": read_costs( costs, "H2 (g) pipeline repurposed", "lifetime" ), - "Heat_exchanger": 20, # assumption based on lifetime heat_exchanger; source: E. van der Roest, R. Bol, T. Fens und A. van Wijk, „Utilisation of waste heat from PEM electrolysers - Unlocking local optimisation, 2023 + "Heat_exchanger": 20, # assumption based on lifetime heat_exchanger; source: E. van der Roest, R. Bol, T. Fens und A. van Wijk, „Utilisation of waste heat from PEM electrolysers - Unlocking local optimisation, 2023 "Heat_pipeline": 20, - "O2_components": 25, # source toDO: ask sayed + "O2_components": 25, # source toDO: ask sayed } # Insert costs parameters["capital_cost"] = {} parameters["O2_capital_cost"] = {} - + for comp in parameters["overnight_cost"].keys(): parameters["capital_cost"][comp] = annualize_capital_costs( parameters["overnight_cost"][comp], @@ -951,7 +958,7 @@ def gas(scenario): ) + parameters["overnight_cost"][comp] * ( parameters["FOM"][comp] / 100 ) - + for comp in ["H2_to_power", "H2_to_CH4"]: parameters["capital_cost"][comp] = ( annualize_capital_costs( @@ -969,7 +976,7 @@ def gas(scenario): parameters["lifetime"]["O2_components"], interest_rate, ) - + parameters["marginal_cost"] = { "OCGT": read_costs(costs, "OCGT", "VOM"), "biogas": read_costs(costs, "biogas", "fuel"), @@ -1207,14 +1214,13 @@ def heat(scenario): "DE_demand_residential_MWh": 536692489.8152325 * 0.71542, # [MWh], source: pypsa-eur run from 2024/12/23: # total heat demand muliplied by residential share from resources/pop_weighted_heat_totals - "DE_demand_service_MWh": 536692489.8152325 * (1-0.71542), + "DE_demand_service_MWh": 536692489.8152325 * (1 - 0.71542), # [MWh], source: pypsa-eur run from 2024/12/23: # total heat demand muliplied by service share from resources/pop_weighted_heat_totals "DE_district_heating_share": 0.42311285313808533, - # [%], source: pypsa-eur run from 2024/12/23 + # [%], source: pypsa-eur run from 2024/12/23 } - parameters["marginal_cost"] = { "central_heat_pump": read_costs( costs, "central air-sourced heat pump", "VOM" @@ -1223,8 +1229,8 @@ def heat(scenario): "central_gas_boiler": read_costs( costs, "central gas boiler", "VOM" ), - "rural_biomass_boiler": 51, # considering an efficiency of 0.8 - "rural_oil_boiler": 67.2, # considering an efficiency of 0.9 + "rural_biomass_boiler": 51, # considering an efficiency of 0.8 + "rural_oil_boiler": 67.2, # considering an efficiency of 0.9 "central_resistive_heater": read_costs( costs, "central resistive heater", "VOM" ), @@ -1313,8 +1319,8 @@ def heat(scenario): "central_gas_boiler": read_costs( costs, "central gas boiler", "VOM" ), - "rural_biomass_boiler": 10, # considering an efficiency of 0.8 - "rural_oil_boiler": 59.7, # considering an efficiency of 0.9 + "rural_biomass_boiler": 10, # considering an efficiency of 0.8 + "rural_oil_boiler": 59.7, # considering an efficiency of 0.9 "central_resistive_heater": read_costs( costs, "central resistive heater", "VOM" ), From 7c07862fa3d78a6bc6834faaca34207b3e0f36bb Mon Sep 17 00:00:00 2001 From: CarlosEpia Date: Wed, 9 Apr 2025 10:53:10 +0200 Subject: [PATCH 84/86] add info for rural_biomass_boiler and rural_oil_boiler costs --- .../data/datasets/scenario_parameters/parameters.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/egon/data/datasets/scenario_parameters/parameters.py b/src/egon/data/datasets/scenario_parameters/parameters.py index 6162a0838..41db7c5b0 100755 --- a/src/egon/data/datasets/scenario_parameters/parameters.py +++ b/src/egon/data/datasets/scenario_parameters/parameters.py @@ -1,5 +1,4 @@ -"""The module containing all parameters for the scenario table -""" +"""The module containing all parameters for the scenario table""" import pandas as pd @@ -1229,8 +1228,8 @@ def heat(scenario): "central_gas_boiler": read_costs( costs, "central gas boiler", "VOM" ), - "rural_biomass_boiler": 51, # considering an efficiency of 0.8 - "rural_oil_boiler": 67.2, # considering an efficiency of 0.9 + "rural_biomass_boiler": 51, # fuel cost / eff. eff = 0.8 from pypsaeur. fuel cost from global_settings + "rural_oil_boiler": 67.2, # CO2 cost + fuel cost / eff. eff = 0.9 from pypsaeur. fuel cost from global_settings "central_resistive_heater": read_costs( costs, "central resistive heater", "VOM" ), @@ -1319,8 +1318,8 @@ def heat(scenario): "central_gas_boiler": read_costs( costs, "central gas boiler", "VOM" ), - "rural_biomass_boiler": 10, # considering an efficiency of 0.8 - "rural_oil_boiler": 59.7, # considering an efficiency of 0.9 + "rural_biomass_boiler": 10, # fuel cost / eff. eff = 0.8 from pypsaeur. fuel cost from global_settings + "rural_oil_boiler": 59.7, # CO2 cost + fuel cost / eff. eff = 0.9 from pypsaeur. fuel cost from global_settings "central_resistive_heater": read_costs( costs, "central resistive heater", "VOM" ), From 64c67f0afb401da4e214b37e064adb93d8a40571 Mon Sep 17 00:00:00 2001 From: Lennart Zimmermann Date: Thu, 10 Apr 2025 09:40:59 +0100 Subject: [PATCH 85/86] Implement source of marginal_cost for missing foreign generators --- src/egon/data/datasets/pypsaeur/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/egon/data/datasets/pypsaeur/__init__.py b/src/egon/data/datasets/pypsaeur/__init__.py index b231a3d48..6be9a7dc6 100755 --- a/src/egon/data/datasets/pypsaeur/__init__.py +++ b/src/egon/data/datasets/pypsaeur/__init__.py @@ -675,6 +675,16 @@ def import_missing_gens(neighbors, network_solved, scn_name): def margina_cost_missing_gens(): + """ + Define marginal_cost for foreign generators by interpolating + marginal_costs defined in the parameters.py . The values + include Fuel-costs, VOM and CO2-costs. + + Returns + ------- + marginal_costs : dict + + """ marginal_costs = { "powerd2025": { "oil": 164.90901098901102, From df55fe3c037aef0c03a0b9b88470f97b0b2bf519 Mon Sep 17 00:00:00 2001 From: Lennart Zimmermann Date: Wed, 16 Apr 2025 08:20:44 +0100 Subject: [PATCH 86/86] add source for 02-parameters --- src/egon/data/datasets/scenario_parameters/parameters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/scenario_parameters/parameters.py b/src/egon/data/datasets/scenario_parameters/parameters.py index 41db7c5b0..6cb01ed1c 100755 --- a/src/egon/data/datasets/scenario_parameters/parameters.py +++ b/src/egon/data/datasets/scenario_parameters/parameters.py @@ -910,7 +910,7 @@ def gas(scenario): ), "Heat_exchanger": 25_000, # [EUR/MW_th] cost assumption for one additional heat_exchanger; source: project internal cost assumption by Fraunhofer ISE "Heat_pipeline": 400_000, # [EUR/MW/km]; average value for DN100-pipeline; source: L. Zimmermann, MODELLIERUNG DER ABWÄRMENUTZUNG VON ELEKTROLYSEUREN IN DEUTSCHLAND FÜR EINE TECHNO - ÖKONOMISCHE OPTIMIERUNG EINES SEKTOR - GEKOPPELTEN ENERGIESYSTEM, 2024 - "O2_components": 5000, # [EUR] ; source toDO: ask sayed + "O2_components": 5000, # [EUR] ; source: Sayed Sadat, Modeling Regional Utilization of the electrolysers Co-Products Oxygen and Heat in Germany, 2024 } # overnight_costs for O2_pipeinecosts related to pipeline_diameter @@ -920,7 +920,7 @@ def gas(scenario): 0.3: 400_000, # EUR/km 0.2: 350_000, # EUR/km 0: 300_000, # EUR/km (costs for any other pipeline diameter) - } + } # source: Sayed Sadat, Modeling Regional Utilization of the electrolysers Co-Products Oxygen and Heat in Germany, 2024 # Insert lifetime parameters["lifetime"] = { @@ -942,7 +942,7 @@ def gas(scenario): ), "Heat_exchanger": 20, # assumption based on lifetime heat_exchanger; source: E. van der Roest, R. Bol, T. Fens und A. van Wijk, „Utilisation of waste heat from PEM electrolysers - Unlocking local optimisation, 2023 "Heat_pipeline": 20, - "O2_components": 25, # source toDO: ask sayed + "O2_components": 25, # source: Sayed Sadat, Modeling Regional Utilization of the electrolysers Co-Products Oxygen and Heat in Germany, 2024 } # Insert costs