diff --git a/routes/conus_hydrology.py b/routes/conus_hydrology.py index 07b62703..d245b60b 100644 --- a/routes/conus_hydrology.py +++ b/routes/conus_hydrology.py @@ -1,5 +1,6 @@ import asyncio import io +from pprint import pprint import numpy as np import xarray as xr import ast @@ -29,6 +30,7 @@ from csv_functions import create_csv from config import RAS_BASE_URL from . import routes +import statistics coverages = { "stats": ["conus_hydro_segments_stats"], @@ -1002,14 +1004,12 @@ def run_get_conus_hydrology_gauge_info(): return render_template("500/server_error.html"), 500 -@routes.route("/conus_hydrology/hydroviz//") -def fetch_all_hydroviz_route(stream_id, model): +@routes.route("/conus_hydrology/hydroviz/") +def fetch_all_hydroviz_route(stream_id): """ - Function to fetch all data for the hydrology visualization for a given stream ID and model. - Maurer historical baseline data is included in every return. + Function to fetch all data for the hydrology visualization for a given stream ID. Args: stream_id (str): Stream ID for the hydrology data - model (str): Model for projected data, only affects stats for tables. Returns: JSON response with the following top-level keys, which correspond to different sections of the hydroviz webapp and contain only the data necessary to populate those sections. @@ -1017,148 +1017,374 @@ def fetch_all_hydroviz_route(stream_id, model): "hydrograph": ..., "id": ..., "monthly_flow": ..., + "min_max_flow_dates": ..., "name": ..., "stats": ..., "summary": ... } - """ - # Use this to validate the model provided in the URL. - projected_models = [ - "ACCESS1-0", - "BCC-CSM1-1", - "BNU-ESM", - "CCSM4", - "GFDL-ESM2G", - "GFDL-ESM2M", - "IPSL-CM5A-LR", - "IPSL-CM5A-MR", - "MIROC-ESM", - "MIROC-ESM-CHEM", - "MIROC5", - "MRI-CGCM3", - "NorESM1-M", - ] - - if model not in projected_models: - return render_template("400/bad_request.html"), 400 stats_response = run_get_conus_hydrology_stats_data(stream_id) - modeled_response = run_get_conus_hydrology_modeled_climatology(stream_id) + climatology_response = run_get_conus_hydrology_modeled_climatology(stream_id) + + # Projected eras to use for the hydrograph and monthly flow charts. + chart_eras = ["2016-2045", "2046-2075", "2071-2100"] # If either response is an error page, return it. - for response in [stats_response, modeled_response]: + for response in [stats_response, climatology_response]: if isinstance(response, tuple): return response - try: - stats = stats_response.get_json() - modeled = modeled_response.get_json() + # try: + # Fetch all needed data. + stats = stats_response.get_json() + climatology = climatology_response.get_json() + + # Convert static stats to "Maurer plus projected delta". + static_stats = stats["data"]["static"] + maurer_plus_delta_stats = {} + for model in static_stats.keys(): + if model not in maurer_plus_delta_stats: + maurer_plus_delta_stats[model] = {} + for scenario in static_stats[model].keys(): + if scenario in ["historical", "rcp26"]: + continue + if scenario not in maurer_plus_delta_stats[model]: + maurer_plus_delta_stats[model][scenario] = {} + for era in static_stats[model][scenario].keys(): + if era not in maurer_plus_delta_stats[model][scenario]: + maurer_plus_delta_stats[model][scenario][era] = {} + # print("##### KEYS #######") + # pprint(static_stats[model][scenario][era].keys()) + for stat in static_stats[model][scenario][era].keys(): + maurer_historical = static_stats["Maurer"]["historical"][ + "1976-2005" + ][stat] + modeled_historical = static_stats[model]["historical"]["1976-2005"][ + stat + ] + modeled_projected = static_stats[model][scenario][era][stat] + projected_delta = modeled_projected - modeled_historical + maurer_plus_delta = round(maurer_historical + projected_delta, 3) + + ### Temporary fallback to raw projected data, not Maurer + delta method. + ### TODO: Figure out how to make the Maurer + delta method work without unrealistic values. + # # These stats represent days per year, so cannot exceed 366. + # if ( + # stat + # in [ + # "dh15", + # "dl16", + # "lf1", + # "ra8", + # "spr_ord", + # "sum_ord", + # "th1", + # "tl1", + # ] + # and maurer_plus_delta > 366 + # ): + # maurer_plus_delta = 366 + + # # No stat can be negative except for ra3. + # if stat != "ra3" and maurer_plus_delta < 0: + # maurer_plus_delta = 0 + # elif stat == "ra3" and maurer_plus_delta > 0: + # maurer_plus_delta = 0 + + # maurer_plus_delta_stats[model][scenario][era][ + # stat + # ] = maurer_plus_delta + maurer_plus_delta_stats[model][scenario][era][ + stat + ] = modeled_projected + + # pprint(maurer_plus_delta_stats) + + # Convert static climatology to "Maurer plus projected delta". + static_climatology = climatology["data"]["static"] + maurer_plus_delta_climatology = {} + for model in static_climatology.keys(): + if model not in maurer_plus_delta_climatology: + maurer_plus_delta_climatology[model] = {} + for scenario in static_climatology[model].keys(): + if scenario in ["historical", "rcp26"]: + continue + if scenario not in maurer_plus_delta_climatology[model]: + maurer_plus_delta_climatology[model][scenario] = {} + for era in static_climatology[model][scenario].keys(): + if era not in maurer_plus_delta_climatology[model][scenario]: + maurer_plus_delta_climatology[model][scenario][era] = [] + for i in range(len(static_climatology[model][scenario][era])): + doy_stats = {} + for stat in static_climatology[model][scenario][era][i].keys(): + maurer_historical = static_climatology["Maurer"]["historical"][ + "1976-2005" + ][i][stat] + modeled_historical = static_climatology[model]["historical"][ + "1976-2005" + ][i][stat] + modeled_projected = static_climatology[model][scenario][era][i][ + stat + ] + projected_delta = modeled_projected - modeled_historical + maurer_plus_delta = round( + maurer_historical + projected_delta, 3 + ) + + ### Temporary fallback to raw projected data, not Maurer + delta method. + ### TODO: Figure out how to make the Maurer + delta method work without unrealistic values. + # doy_stats[stat] = maurer_plus_delta + doy_stats[stat] = modeled_projected + maurer_plus_delta_climatology[model][scenario][era].append( + doy_stats + ) + + ########## Populate arrays for hydrograph. ########## + + hydrograph = { + "historical": {}, + "projected": {}, + } - # The hydrograph and monthly mean flow charts use this scenario and era. - scenario = "rcp85" - era = "2046-2075" + historical_climatology = climatology["data"]["static"]["Maurer"]["historical"][ + "1976-2005" + ] + hydrograph_historical = { + "doy_min": list(map(lambda x: x["doy_min"], historical_climatology)), + "doy_mean": list(map(lambda x: x["doy_mean"], historical_climatology)), + "doy_max": list(map(lambda x: x["doy_max"], historical_climatology)), + } + hydrograph["historical"] = hydrograph_historical + + # Get min/mean/max values for each DOY across all projected models. + for era in chart_eras: + if era not in hydrograph["projected"]: + hydrograph["projected"][era] = {} + for scenario in ["rcp45", "rcp60", "rcp85"]: + if scenario not in hydrograph["projected"][era]: + hydrograph["projected"][era][scenario] = { + "doy_min_min": [], + "doy_mean_min": [], + "doy_mean_mean": [], + "doy_mean_max": [], + "doy_max_max": [], + } + for i in range(366): + doy_mins = [] + doy_means = [] + doy_maxes = [] + for model_dict in maurer_plus_delta_climatology.values(): + if scenario not in model_dict: + continue + doy_mins.append(model_dict[scenario][era][i]["doy_min"]) + doy_means.append(model_dict[scenario][era][i]["doy_mean"]) + doy_maxes.append(model_dict[scenario][era][i]["doy_max"]) - hydrograph_min = [] - hydrograph_mean = [] - hydrograph_max = [] + hydrograph["projected"][era][scenario]["doy_min_min"].append( + round(min(doy_mins), 3) + ) + hydrograph["projected"][era][scenario]["doy_mean_min"].append( + round(min(doy_means), 3) + ) + hydrograph["projected"][era][scenario]["doy_mean_mean"].append( + round(statistics.mean(doy_means), 3) + ) + hydrograph["projected"][era][scenario]["doy_mean_max"].append( + round(max(doy_means), 3) + ) + hydrograph["projected"][era][scenario]["doy_max_max"].append( + round(max(doy_maxes), 3) + ) - doy_min = None - doy_max = None - doy_mean = None + ########## Populate arrays for monthly modeled flow rate chart. ########## - hydrograph_historical = {} - historical = modeled["data"]["static"]["Maurer"]["historical"]["1976-2005"] + monthly_flow = { + "historical": {}, + "projected": {}, + } - hydrograph_historical["doy_min"] = list( - map(lambda x: round(x["doy_min"]), historical) - ) - hydrograph_historical["doy_mean"] = list( - map(lambda x: x["doy_mean"], historical) - ) - hydrograph_historical["doy_max"] = list( - map(lambda x: round(x["doy_max"]), historical) - ) + monthly_flow_keys = [ + "ma12", + "ma13", + "ma14", + "ma15", + "ma16", + "ma17", + "ma18", + "ma19", + "ma20", + "ma21", + "ma22", + "ma23", + ] - del modeled["data"]["static"]["Maurer"] - - # Get min/mean/max values for each DOY across all projected models. - for i in range(366): - unset = True - for model_dict in modeled["data"]["static"].values(): - if unset: - doy_min = model_dict[scenario][era][i]["doy_min"] - doy_max = model_dict[scenario][era][i]["doy_max"] - doy_mean = model_dict[scenario][era][i]["doy_mean"] - unset = False + # Get historical data for each month. + for key in monthly_flow_keys: + monthly_flow["historical"][key] = stats["data"]["static"]["Maurer"][ + "historical" + ]["1976-2005"][key] + + # Populate lists of values for each month for each projected model. + # These values will be used to generate box plots in the webapp. + for era in chart_eras: + if era not in monthly_flow["projected"]: + monthly_flow["projected"][era] = {} + for scenario in ["rcp45", "rcp60", "rcp85"]: + if scenario not in monthly_flow["projected"][era]: + monthly_flow["projected"][era][scenario] = {} + for model_stats in maurer_plus_delta_stats.values(): + # print("########### model_stats ############") + # pprint(model_stats) + if scenario not in model_stats: continue - else: - if model_dict[scenario][era][i]["doy_min"] < doy_min: - doy_min = model_dict[scenario][era][i]["doy_min"] - if model_dict[scenario][era][i]["doy_max"] > doy_max: - doy_max = model_dict[scenario][era][i]["doy_max"] - doy_mean += model_dict[scenario][era][i]["doy_mean"] - - hydrograph_min.append(round(doy_min)) - hydrograph_mean.append(round(doy_mean / len(modeled["data"]["static"]), 3)) - hydrograph_max.append(round(doy_max)) - - monthly_flow_keys = [ - "ma12", - "ma13", - "ma14", - "ma15", - "ma16", - "ma17", - "ma18", - "ma19", - "ma20", - "ma21", - "ma22", - "ma23", - ] - - monthly_flow = {"historical": {}, "projected": {}} - - # Get historical data for each month. - for key in monthly_flow_keys: - monthly_flow["historical"][key] = stats["data"]["static"]["Maurer"][ - "historical" - ]["1976-2005"][key] - - # Populate lists of values for each month for each projected model. - # These values will be used to generate box plots in the webapp. - for model_key, model_stats in stats["data"]["static"].items(): - # Skip Maurer, which is used only for historical baselines. - if model_key == "Maurer": - continue - for key in monthly_flow_keys: - if key not in monthly_flow["projected"]: - monthly_flow["projected"][key] = [] - monthly_flow["projected"][key].append(model_stats[scenario][era][key]) - - # Stats for the tables in the webapp, only for the requested model. - table_stats = { - "historical": stats["data"]["static"]["Maurer"]["historical"], - "projected": stats["data"]["static"][model][scenario], - } - - response = { - "hydrograph": { - "historical": hydrograph_historical, - "projected": { - "doy_min": hydrograph_min, - "doy_mean": hydrograph_mean, - "doy_max": hydrograph_max, - }, + # print("##############################") + # pprint(model_stats[scenario][era]) + for key in monthly_flow_keys: + if key not in monthly_flow["projected"][era][scenario]: + monthly_flow["projected"][era][scenario][key] = [] + monthly_flow["projected"][era][scenario][key].append( + model_stats[scenario][era][key] + ) + + ########## Populate arrays for max flow date chart. ########## + + min_max_flow_dates = { + "historical": { + "min": { + "flow": None, + "date": None, }, - "id": stats["id"], - "name": stats["name"], - "monthly_flow": monthly_flow, - "stats": table_stats, - "summary": stats["summary"], - } - return jsonify(response) + "max": { + "flow": None, + "date": None, + }, + }, + "projected": {}, + } - except Exception as exc: - return render_template("500/server_error.html"), 500 + # Get historical data for min and max flow date. + min_max_flow_dates["historical"]["min"]["flow"] = stats["data"]["static"]["Maurer"][ + "historical" + ]["1976-2005"]["dl1"] + min_max_flow_dates["historical"]["min"]["date"] = stats["data"]["static"]["Maurer"][ + "historical" + ]["1976-2005"]["tl1"] + min_max_flow_dates["historical"]["max"]["flow"] = stats["data"]["static"]["Maurer"][ + "historical" + ]["1976-2005"]["dh1"] + min_max_flow_dates["historical"]["max"]["date"] = stats["data"]["static"]["Maurer"][ + "historical" + ]["1976-2005"]["th1"] + + # Populate lists of values for max flow date for each projected model. + for era in chart_eras: + if era not in min_max_flow_dates["projected"]: + min_max_flow_dates["projected"][era] = {} + for scenario in ["rcp45", "rcp60", "rcp85"]: + if scenario not in min_max_flow_dates["projected"][era]: + min_max_flow_dates["projected"][era][scenario] = { + "min": { + "flow": [], + "date": [], + }, + "max": { + "flow": [], + "date": [], + }, + } + for model_stats in maurer_plus_delta_stats.values(): + if scenario not in model_stats: + continue + + min_max_flow_dates["projected"][era][scenario]["min"]["flow"].append( + model_stats[scenario][era]["dl1"] + ) + min_max_flow_dates["projected"][era][scenario]["min"]["date"].append( + model_stats[scenario][era]["tl1"] + ) + min_max_flow_dates["projected"][era][scenario]["max"]["flow"].append( + model_stats[scenario][era]["dh1"] + ) + min_max_flow_dates["projected"][era][scenario]["max"]["date"].append( + model_stats[scenario][era]["th1"] + ) + + ########## Calculate stats for the stats table. ########## + + table_stats = { + "historical": {}, + "projected": {}, + } + + table_stats["historical"] = stats["data"]["static"]["Maurer"]["historical"] + + all_stats = {} + mid_stats = {} + for era in chart_eras: + for model_stats in maurer_plus_delta_stats.values(): + for scenario in model_stats.keys(): + if scenario in ["historical", "rcp26"]: + continue + if era not in model_stats[scenario]: + continue + for stat in model_stats[scenario][era].keys(): + if era not in all_stats: + all_stats[era] = {} + if stat not in all_stats[era]: + all_stats[era][stat] = [] + all_stats[era][stat].append(model_stats[scenario][era][stat]) + if scenario == "rcp60": + if era not in mid_stats: + mid_stats[era] = {} + if stat not in mid_stats[era]: + mid_stats[era][stat] = [] + mid_stats[era][stat].append(model_stats["rcp60"][era][stat]) + + for era in all_stats.keys(): + if era not in table_stats["projected"]: + table_stats["projected"][era] = {} + for stat in all_stats[era].keys(): + all_values = all_stats[era][stat] + mid_values = mid_stats[era][stat] + if "min" not in table_stats["projected"][era]: + table_stats["projected"][era]["min"] = {} + if "max" not in table_stats["projected"][era]: + table_stats["projected"][era]["max"] = {} + if "mid" not in table_stats["projected"][era]: + table_stats["projected"][era]["mid"] = {} + table_stats["projected"][era]["min"][stat] = round(min(all_values), 3) + table_stats["projected"][era]["max"][stat] = round(max(all_values), 3) + table_stats["projected"][era]["mid"][stat] = round( + statistics.mean(mid_values), 3 + ) + + gauge_id = None + h8_outlet = False + huc8 = None + + gdf = asyncio.run(get_features(stream_id)) + if not isinstance(gdf, tuple): + if gdf.loc[0].h8_outlet == 1: + h8_outlet = True + if gdf.loc[0].GAUGE_ID != "NA": + gauge_id = gdf.loc[0].GAUGE_ID + huc8 = gdf.loc[0].huc8 + + response = { + "gauge_id": gauge_id, + "h8_outlet": h8_outlet, + "huc8": huc8, + "hydrograph": hydrograph, + "id": stats["id"], + "name": stats["name"], + "monthly_flow": monthly_flow, + "min_max_flow_dates": min_max_flow_dates, + "stats": table_stats, + "summary": stats["summary"], + } + + return jsonify(response) + + # except Exception as exc: + # return render_template("500/server_error.html"), 500