Features/#856 cleanup results folder

etrago/appl.py

@@ -89,7 +89,7 @@
     "scn_extension": None,  # None or array of extension scenarios
     # Export options:
     "lpfile": False,  # save pyomo's lp file: False or /path/to/lpfile.lp
-    "csv_export": "results",  # save results as csv: False or /path/tofolder
+    "export_results_path": "results",  # save results as csv: False or /path/tofolder
     # Settings:
     "extendable": {
         "extendable_components": [
@@ -135,14 +135,12 @@
             "active": True,  # choose if clustering is activated
             "cluster_within_focus": False,  # False for very low clustering within focus region
             "n_clusters": 30,  # total number of resulting AC nodes
-            "k_elec_busmap": False,  # False or path/to/busmap.csv
         },
         "gas_grids": {
             "active": True,  # choose if clustering is activated
             "cluster_within_focus": False,  #  False for very low clustering within focus region
             "n_clusters_ch4": 15,  # total number of resulting CH4 nodes
             "n_clusters_h2": 15,  # total number of resulting H2 nodes
-            "k_ch4_busmap": False,  # False or path/to/ch4_busmap.csv
         },
     },
     "spatial_disaggregation": None,  # None or 'uniform'
@@ -290,9 +288,9 @@ def run_etrago(args, json_path):
     lpfile : bool or str
         State if and where you want to save pyomo's lp file. Options:
         False or '/path/tofile.lp'. Default: False.
-    csv_export : bool or str
-        State if and where you want to save results as csv files. Options:
-        False or '/path/tofolder'. Default: False.
+    export_results_path : bool or str
+        State if and where you want to save results as csv and .nc files.
+        Options: False or '/path/tofolder'. Default: False.
 
     extendable : dict
         Choose components you want to optimize and set upper bounds for grid

etrago/args.json

@@ -30,7 +30,7 @@
     "scn_name": "eGon2035",
     "scn_extension": null,
     "lpfile": false,
-    "csv_export": "results",
+    "export_results_path": "results",
     "extendable": {
         "extendable_components": [
             "as_in_db"
@@ -83,15 +83,13 @@
         "electricity_grid": {
             "active": true,
             "cluster_within_focus": false,
-            "n_clusters": 30,
-            "k_elec_busmap": false
+            "n_clusters": 30
         },
         "gas_grids": {
             "active": true,
             "cluster_within_focus": false,
             "n_clusters_ch4": 15,
-            "n_clusters_h2": 15,
-            "k_ch4_busmap": false
+            "n_clusters_h2": 15
         }
     },
     "spatial_disaggregation": null,

etrago/cluster/electrical.py

@@ -42,6 +42,7 @@
     from etrago.cluster.spatial import (
         busmap_ehv_clustering,
         drop_nan_values,
+        export_clustering_results,
         focus_weighting,
         group_links,
         kmean_clustering,
@@ -735,6 +736,8 @@ def preprocessing(etrago, apply_on="grid_model"):
         )
 
     settings = etrago.args["network_clustering"]
+    # quick fix while bus map loading is not working
+    settings["k_elec_busmap"] = False
 
     # problem our lines have no v_nom. this is implicitly defined by the
     # connected buses:
@@ -880,7 +883,9 @@ def postprocessing(
     """
     settings = etrago.args["network_clustering"]["electricity_grid"]
     method = etrago.args["network_clustering"]["method"]["algorithm"]
-    num_clusters = settings["n_clusters"]
+
+    # quick fix while bus map loading is not working
+    settings["k_ch4_busmap"] = False
 
     if not settings["k_elec_busmap"]:
         busmap.name = "cluster"
@@ -895,10 +900,6 @@ def postprocessing(
             )
         )
 
-        busmap_elec.to_csv(
-            f"{method}_elecgrid_busmap_{num_clusters}_result.csv"
-        )
-
     else:
         logger.info("Import Busmap for spatial clustering")
         busmap_foreign = pd.read_csv(
@@ -1143,6 +1144,11 @@ def run_spatial_clustering(self):
     -------
     None
     """
+    # quick fix while bus map loading is not working
+    self.args["network_clustering"]["electricity_grid"][
+        "k_elec_busmap"
+    ] = False
+
     if self.args["network_clustering"]["electricity_grid"]["active"]:
         if self.args["spatial_disaggregation"] is not None:
             self.disaggregated_network = self.network.copy()
@@ -1230,3 +1236,6 @@ def run_spatial_clustering(self):
             )
             + self.args["network_clustering"]["method"]["algorithm"]
         )
+
+        if self.args["export_results_path"]:
+            export_clustering_results(self)

etrago/cluster/gas.py

@@ -38,6 +38,7 @@
 if "READTHEDOCS" not in os.environ:
     from etrago.cluster.spatial import (
         drop_nan_values,
+        export_clustering_results,
         focus_weighting,
         group_links,
         kmedoids_dijkstra_clustering,
@@ -453,6 +454,9 @@ def gas_postprocessing(
     settings = etrago.args["network_clustering"]["gas_grids"]
     scn = etrago.args["scn_name"]
 
+    # quick fix while bus map loading is not working
+    settings["k_ch4_busmap"] = False
+
     if apply_on == "grid_model":
         if settings["k_ch4_busmap"] is False:
             if (
@@ -461,11 +465,6 @@ def gas_postprocessing(
             ):
                 busmap.index.name = "bus_id"
                 busmap.name = "cluster"
-                busmap.to_csv(
-                    "kmeans_gasgrid_busmap_"
-                    + str(settings["n_clusters_ch4"])
-                    + "_result.csv"
-                )
 
             else:
                 busmap.name = "cluster"
@@ -478,11 +477,7 @@ def gas_postprocessing(
 
                 export = pd.concat([busmap, busmap_ind], axis=1)
                 export.index.name = "bus_id"
-                export.to_csv(
-                    "kmedoids-dijkstra_gasgrid_busmap_"
-                    + str(settings["n_clusters_ch4"])
-                    + "_result.csv"
-                )
+
         network = etrago.network
     else:
         network = etrago.pre_market_model
@@ -1101,6 +1096,8 @@ def run_spatial_clustering_gas(self):
         "H2_grid" in self.network.buses.carrier.values
     ):
         settings = self.args["network_clustering"]["gas_grids"]
+        # quick fix while bus map loading is not working
+        settings["k_ch4_busmap"] = False
 
         if settings["active"]:
             method = self.args["network_clustering"]["method"]["algorithm"]
@@ -1266,3 +1263,6 @@ def run_spatial_clustering_gas(self):
                         method,
                     )
                 )
+
+        if self.args["export_results_path"]:
+            export_clustering_results(self)

etrago/cluster/spatial.py

@@ -20,6 +20,7 @@
 # File description for read-the-docs
 """spatial.py defines the methods to run spatial clustering on networks."""
 
+import json
 import logging
 import os
 
@@ -1092,3 +1093,15 @@ def drop_nan_values(network):
             (c.attrs.status == "Output") & (c.attrs.varying)
         ].index:
             c.pnl[pnl] = pd.DataFrame(index=network.snapshots)
+
+
+def export_clustering_results(etrago):
+
+    path = etrago.args["export_results_path"]
+
+    with open(os.path.join(path, "busmap.json"), "w") as d:
+        json.dump(etrago.busmap["busmap"], d, indent=4)
+
+    etrago.busmap["orig_network"].export_to_csv_folder(
+        path + "/original_network_topology"
+    )

etrago/disaggregate/spatial.py

@@ -926,6 +926,21 @@ def run_disaggregation(self):
                 )
             )
 
-            if self.args["csv_export"]:
-                path = self.args["csv_export"] + "/disaggregated_network"
-                self.disaggregated_network.export_to_csv_folder(path)
+            if self.args["export_results_path"]:
+                path = (
+                    self.args["export_results_path"]
+                    + "/disaggregated_network.nc"
+                )
+
+                for comp_df in [
+                    self.disaggregated_network.transformers,
+                    self.disaggregated_network.lines,
+                    self.disaggregated_network.links,
+                    self.disaggregated_network.buses,
+                ]:
+                    if "geom" in comp_df.columns:
+                        comp_df.drop(columns=["geom"], inplace=True)
+                    if "topo" in comp_df.columns:
+                        comp_df.drop(columns=["topo"], inplace=True)
+
+                self.disaggregated_network.export_to_netcdf(path)

etrago/disaggregate/temporal.py

@@ -208,7 +208,6 @@ def dispatch_disaggregation(self):
         )
         self.network.stores.e_cyclic = self.network_tsa.stores.e_cyclic
 
-        if self.args["csv_export"]:
-            path = self.args["csv_export"]
-            self.export_to_csv(path)
-            self.export_to_csv(path + "/temporal_disaggregaton")
+        if self.args["export_results_path"]:
+            path = self.args["export_results_path"]
+            self.network.export_to_csv_folder(path + "/temporal_disaggregaton")

etrago/execute/__init__.py

@@ -26,6 +26,7 @@
 
 if "READTHEDOCS" not in os.environ:
     import logging
+    import shutil
     import time
 
     from pypsa.linopf import network_lopf
@@ -202,13 +203,14 @@ def iterate_lopf(
     """
 
     args = etrago.args
-    path = args["csv_export"]
     lp_path = args["lpfile"]
 
-    if args["temporal_disaggregation"]["active"]:
-        if args["csv_export"]:
-            path = path + "/temporally_reduced"
+    if args["export_results_path"]:
+        path = args["export_results_path"] + "/grid_optimization"
+        if not os.path.exists(path):
+            os.makedirs(path, exist_ok=True)
 
+    if args["temporal_disaggregation"]["active"]:
         if args["lpfile"]:
             lp_path = lp_path[0:-3] + "_temporally_reduced.lp"
 
@@ -229,9 +231,20 @@ def iterate_lopf(
             for i in range(1, (1 + n_iter)):
                 run_lopf(etrago, extra_functionality, method)
 
-                if args["csv_export"]:
-                    path_it = path + "/lopf_iteration_" + str(i)
-                    etrago.export_to_csv(path_it)
+                if args["export_results_path"]:
+                    # Store intermediate iterations
+                    if i < n_iter:
+                        path_it = path + "/lopf_iteration_" + str(i)
+                        etrago.network.export_to_csv_folder(path_it)
+                    # Store final result
+                    else:
+                        etrago.network.export_to_csv_folder(path)
+
+                    # Delete previous iteration's results
+                    if i > 1:
+                        path_prev = path + "/lopf_iteration_" + str(i - 1)
+                        if os.path.exists(path_prev):
+                            shutil.rmtree(path_prev)
 
                 if i < n_iter:
                     l_snom_pre, t_snom_pre = update_electrical_parameters(
@@ -263,18 +276,18 @@ def iterate_lopf(
 
                 i += 1
 
-                if args["csv_export"]:
+                if args["export_results_path"]:
                     path_it = path + "/lopf_iteration_" + str(i)
-                    etrago.export_to_csv(path_it)
+                    etrago.network.export_to_csv_folder(path_it)
 
                 if abs(pre - network.objective) <= diff_obj:
                     print("Threshold reached after " + str(i) + " iterations.")
                     break
 
     else:
         run_lopf(etrago, extra_functionality, method)
-        if etrago.args["csv_export"]:
-            etrago.export_to_csv(path)
+        if etrago.args["export_results_path"]:
+            etrago.network.export_to_csv_folder(path)
 
     if args["lpfile"]:
         network.model.write(lp_path)
@@ -322,12 +335,6 @@ def lopf(self):
     z = (y - x) / 60
     logger.info("Time for LOPF [min]: {}".format(round(z, 2)))
 
-    if self.args["csv_export"]:
-        path = self.args["csv_export"]
-        if self.args["temporal_disaggregation"]["active"] is True:
-            path = path + "/temporally_reduced"
-        self.export_to_csv(path)
-
 
 def optimize(self):
     """Run optimization of dispatch and grid and storage expansion based on
@@ -1054,15 +1061,25 @@ def drop_foreign_components(network):
                     foreign_series[comp][attr], comp, attr
                 )
 
-    if args["csv_export"]:
-        path = args["csv_export"] + "/pf_post_lopf"
-        etrago.export_to_csv(path)
+    if args["export_results_path"]:
+        path = args["export_results_path"] + "/non_linear_powerflow"
+        etrago.network.export_to_csv_folder(path)
         pf_solve.to_csv(os.path.join(path, "pf_solution.csv"), index=True)
 
         # Save un-solved disaggregated network
         if args["spatial_disaggregation"]:
-            etrago.disaggregated_network.export_to_csv_folder(
-                args["csv_export"] + "/disaggregated_network"
+            for comp_df in [
+                etrago.disaggregated_network.transformers,
+                etrago.disaggregated_network.lines,
+                etrago.disaggregated_network.links,
+                etrago.disaggregated_network.buses,
+            ]:
+                if "geom" in comp_df.columns:
+                    comp_df.drop(columns=["geom"], inplace=True)
+                if "topo" in comp_df.columns:
+                    comp_df.drop(columns=["topo"], inplace=True)
+            etrago.disaggregated_network.export_to_netcdf(
+                args["export_results_path"] + "/disaggregated_network.nc"
             )
 
     return network

etrago/execute/market_optimization.py

@@ -91,11 +91,13 @@ def market_optimization(self):
         logger.warning("Method type must be either 'pyomo' or 'linopy'")
 
     # Export results of pre-market model
-    if self.args["csv_export"]:
-        path = self.args["csv_export"]
+    if self.args["export_results_path"]:
+        path = self.args["export_results_path"]
         if not os.path.exists(path):
             os.makedirs(path, exist_ok=True)
-        self.pre_market_model.export_to_csv_folder(path + "/pre_market")
+        self.pre_market_model.export_to_csv_folder(
+            path + "/pre_market_optimization"
+        )
     logger.info("Preparing short-term UC market model")
 
     build_shortterm_market_model(self, unit_commitment)
@@ -128,11 +130,11 @@ def market_optimization(self):
     self.args["method"]["formulation"] = method_args
 
     # Export results of market model
-    if self.args["csv_export"]:
-        path = self.args["csv_export"]
+    if self.args["export_results_path"]:
+        path = self.args["export_results_path"]
         if not os.path.exists(path):
             os.makedirs(path, exist_ok=True)
-        self.market_model.export_to_csv_folder(path + "/market")
+        self.market_model.export_to_csv_folder(path + "/market_optimization")
 
 
 def build_market_model(self, unit_commitment=False):