From acbe61c8d31c675b81e09c34ddf18f72529cbce4 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Tue, 21 Oct 2025 11:35:20 -0800 Subject: [PATCH 01/30] initial commit, set up new route in app and add new WCPS functions + allowed params --- application.py | 18 +++- generate_requests.py | 87 +++++++++++++++++++ routes/__init__.py | 1 + routes/dynamic_indicators.py | 57 ++++++++++++ .../documentation/dynamic_indicators.html | 4 + 5 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 routes/dynamic_indicators.py create mode 100644 templates/documentation/dynamic_indicators.html diff --git a/application.py b/application.py index 8e5d5663..b4eb3624 100644 --- a/application.py +++ b/application.py @@ -12,10 +12,8 @@ # Configure logging to emit to stdout logging.basicConfig( level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(sys.stdout) - ] + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + handlers=[logging.StreamHandler(sys.stdout)], ) # Elastic Beanstalk wants `application` to be present. @@ -116,6 +114,18 @@ class QueryParamsSchema(Schema): required=False, ) + # Make sure "n" parameter is string number between -1000 and 1000 + n = fields.Str( + validate=lambda str: str.isdigit() and -1000 <= int(str) <= 1000, + required=False, + ) + + # Make sure "units" parameter is "in", "mm", "F", or "C" + units = fields.Str( + validate=lambda str: str in ["in", "mm", "F", "C"], + required=False, + ) + schema = QueryParamsSchema() errors = schema.validate(request.args) if errors: diff --git a/generate_requests.py b/generate_requests.py index 0e8d8e2b..1aac7dca 100644 --- a/generate_requests.py +++ b/generate_requests.py @@ -193,3 +193,90 @@ def generate_wcps_describe_coverage_str(cov_id): """ query_str = f'for $c in ({cov_id}) return describe($c, "application/json", "outputType=GeneralGridCoverage")' return quote(query_str) + + +def construct_get_annual_mmm_stat_wcps_query_string( + coverage, + operator, + start_year, + end_year, + x_coord, + y_coord, + format="application/json", +): + """ + Construct a WCPS query string to compute annual min, mean, or max statistics over a time range. + + Args: + coverage (str): The coverage identifier. + operator (str): The statistical operation to perform ('min', 'max', 'mean'). + start_year (int): The starting year of the time range. + end_year (int): The ending year of the time range. + x_coord (float or str): The x-coordinate for the point query. + y_coord (float or str): The y-coordinate for the point query. + format (str): The desired output format (default is "application/json"). + Returns: + query_string (str): The constructed WCPS query string. + """ + # convert inputs to strings if not already + x_coord = str(x_coord) + y_coord = str(y_coord) + + query_string = f""" + for $cov in ({coverage}) + let $start_year := "{start_year}", + $end_year := "{end_year}", + $x_coord := {x_coord}, + $y_coord := {y_coord} + return encode( + coverage result + over $pt t($start_year : $end_year) + values {operator} ( $cov[x($x_coord), y($y_coord), ansi($pt : $pt)] ) + , "{format}") + """ + return query_string + + +def construct_count_annual_days_above_or_below_threshold_wcps_query_string( + coverage, + operator, + threshold, + start_year, + end_year, + x_coord, + y_coord, + format="application/json", +): + """ + Construct a WCPS query string to count annual days above or below a specified threshold over a time range. + Args: + coverage (str): The coverage identifier. + operator (str): The comparison operator ('>' for above threshold, '<' for below threshold). + threshold (float): The threshold value for comparison. + start_year (int): The starting year of the time range. + end_year (int): The ending year of the time range. + x_coord (float or str): The x-coordinate for the point query. + y_coord (float or str): The y-coordinate for the point query. + format (str): The desired output format (default is "application/json"). + Returns: + query_string (str): The constructed WCPS query string. + """ + # convert inputs to strings if not already + threshold = str(threshold) + x_coord = str(x_coord) + y_coord = str(y_coord) + + query_string = f""" + for $cov in ({coverage}) + let $threshold := {threshold}, + $start_year := "{start_year}", + $end_year := "{end_year}", + $x_coord := {x_coord}, + $y_coord := {y_coord} + return encode( + coverage result + over $pt t($start_year : $end_year) + values ( sum($cov[x($x_coord), y($y_coord), ansi($pt : $pt)] {operator} $threshold) ) + , "{format}") + """ + return query_string diff --git a/routes/__init__.py b/routes/__init__.py index 268fdf7c..25857783 100644 --- a/routes/__init__.py +++ b/routes/__init__.py @@ -38,3 +38,4 @@ def enforce_site_offline(): from .cmip6 import * from .places import * from .era5wrf import * +from .dynamic_indicators import * diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py new file mode 100644 index 00000000..138f433e --- /dev/null +++ b/routes/dynamic_indicators.py @@ -0,0 +1,57 @@ +import asyncio +import ast +import logging +import xarray as xr +import cftime +import io +import numpy as np +from flask import Blueprint, render_template, request +import datetime + +# local imports +from generate_urls import generate_wcs_query_url +from generate_requests import generate_wcs_getcov_str +from fetch_data import fetch_data, describe_via_wcps +from validate_request import ( + latlon_is_numeric_and_in_geodetic_range, + validate_year, +) + +# TODO: for additional postprocessing or csv output, uncomment these imports and add code +# from postprocessing import postprocess, prune_nulls_with_max_intensity +# from csv_functions import create_csv + +from . import routes + +logger = logging.getLogger(__name__) + +cmip6_api = Blueprint("dynamic_indicators_api", __name__) + +coverages = { + "pr": [ + "cmip6_downscaled_pr_5ModelAvg_historical_wcs", + "cmip6_downscaled_pr_5ModelAvg_ssp126_wcs", + "cmip6_downscaled_pr_5ModelAvg_ssp245_wcs", + "cmip6_downscaled_pr_5ModelAvg_ssp370_wcs", + "cmip6_downscaled_pr_5ModelAvg_ssp585_wcs", + ], + "tasmin": [ + "cmip6_downscaled_tasmin_5ModelAvg_historical_wcs", + "cmip6_downscaled_tasmin_5ModelAvg_ssp126_wcs", + "cmip6_downscaled_tasmin_5ModelAvg_ssp245_wcs", + "cmip6_downscaled_tasmin_5ModelAvg_ssp370_wcs", + "cmip6_downscaled_tasmin_5ModelAvg_ssp585_wcs", + ], + "tasmax": [ + "cmip6_downscaled_tasmax_5ModelAvg_historical_wcs", + "cmip6_downscaled_tasmax_5ModelAvg_ssp126_wcs", + "cmip6_downscaled_tasmax_5ModelAvg_ssp245_wcs", + "cmip6_downscaled_tasmax_5ModelAvg_ssp370_wcs", + "cmip6_downscaled_tasmax_5ModelAvg_ssp585_wcs", + ], +} + +time_domains = { + "historical": (1965, 2014), + "projected": (2015, 2100), +} diff --git a/templates/documentation/dynamic_indicators.html b/templates/documentation/dynamic_indicators.html new file mode 100644 index 00000000..7a9a070f --- /dev/null +++ b/templates/documentation/dynamic_indicators.html @@ -0,0 +1,4 @@ +{% extends 'base.html' %} {% block content %} +

Dynamically Generated Indicators from Downscaled CMIP6

+ +{% endblock %} \ No newline at end of file From c17a5b9fa961c61970918f51f7925bee5e8cce66 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Tue, 21 Oct 2025 14:48:34 -0800 Subject: [PATCH 02/30] sketch all routes --- generate_requests.py | 3 +- routes/dynamic_indicators.py | 80 +++++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/generate_requests.py b/generate_requests.py index 1aac7dca..767fb324 100644 --- a/generate_requests.py +++ b/generate_requests.py @@ -206,10 +206,11 @@ def construct_get_annual_mmm_stat_wcps_query_string( ): """ Construct a WCPS query string to compute annual min, mean, or max statistics over a time range. + Note that using an empty string in the operator will return all annual values without aggregation. Args: coverage (str): The coverage identifier. - operator (str): The statistical operation to perform ('min', 'max', 'mean'). + operator (str): The statistical operation to perform ('min', 'max', 'mean', or '' to return all values). start_year (int): The starting year of the time range. end_year (int): The ending year of the time range. x_coord (float or str): The x-coordinate for the point query. diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 138f433e..566f8b25 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -10,8 +10,10 @@ # local imports from generate_urls import generate_wcs_query_url -from generate_requests import generate_wcs_getcov_str -from fetch_data import fetch_data, describe_via_wcps +from generate_requests import ( + construct_count_annual_days_above_or_below_threshold_wcps_query_string, + construct_get_annual_mmm_stat_wcps_query_string, +) from validate_request import ( latlon_is_numeric_and_in_geodetic_range, validate_year, @@ -55,3 +57,77 @@ "historical": (1965, 2014), "projected": (2015, 2100), } + + +@routes.route( + "/dynamic_indicators/count_days_above//////" +) +def count_days_above(threshold, units, variable, lat, lon): + # tasmax coverage + # usage: .../dynamic_indicators/count_days_above/25/C/tasmax/64.5/-147.5/ + # usage: .../dynamic_indicators/count_days_above/5/mm/pr/64.5/-147.5/ + + # sample URL for days above 25C + url = generate_wcs_query_url( + "ProcessCoverages&query=" + + construct_count_annual_days_above_or_below_threshold_wcps_query_string( + "cmip6_downscaled_tasmax_5ModelAvg_ssp585_wcs", + 25, + ">", + 2000, + 2010, + 350000, + 1700000, + ) + ) + + return None + + +@routes.route("/dynamic_indicators/count_days_below/////") +def count_days_below(threshold, units, lat, lon): + # tasmin coverage + # usage: .../dynamic_indicators/count_days_below/-30/C/64.5/-147.5/ + return None + + +@routes.route("/dynamic_indicators/stat//////") +def get_annual_stat(stat, variable, units, lat, lon): + # pr, tasmax, or tasmin coverage + # usage: .../dynamic_indicators/stat/max/pr/mm/64.5/-147.5/ + # usage: .../dynamic_indicators/stat/min/tasmin/C/64.5/-147.5/ + + # sample URL for maxmimum daily precip + url = generate_wcs_query_url( + "ProcessCoverages&query=" + + construct_get_annual_mmm_stat_wcps_query_string( + "cmip6_downscaled_prx_5ModelAvg_ssp585_wcs", + "max", + 2000, + 2010, + 350000, + 1700000, + ) + ) + return None + + +@routes.route("/dynamic_indicators/rank//////") +def get_annual_stat(position, direction, variable, units, lat, lon): + # pr, tasmax, or tasmin coverage + # usage: .../dynamic_indicators/rank/6/highest/pr/64.5/-147.5/ + # usage: .../dynamic_indicators/rank/6/lowest/tasmin/64.5/-147.5/ + + # sample URL for all values - postprocess this for ranking + url = generate_wcs_query_url( + "ProcessCoverages&query=" + + construct_get_annual_mmm_stat_wcps_query_string( + "cmip6_downscaled_prx_5ModelAvg_ssp585_wcs", + "", + 2000, + 2010, + 350000, + 1700000, + ) + ) + return None From 195e54e64ea9ba2e8f031425fc1ce4f2c663bd8f Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Tue, 21 Oct 2025 15:13:19 -0800 Subject: [PATCH 03/30] begin validation routines --- routes/dynamic_indicators.py | 123 +++++++++++++++++++++++++++-------- 1 file changed, 96 insertions(+), 27 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 566f8b25..79a5d694 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -9,6 +9,7 @@ import datetime # local imports +from fetch_data import fetch_data from generate_urls import generate_wcs_query_url from generate_requests import ( construct_count_annual_days_above_or_below_threshold_wcps_query_string, @@ -17,6 +18,7 @@ from validate_request import ( latlon_is_numeric_and_in_geodetic_range, validate_year, + project_latlon, ) # TODO: for additional postprocessing or csv output, uncomment these imports and add code @@ -59,40 +61,103 @@ } +def validate_latlon_and_reproject_to_epsg_3338(lat, lon): + lat = float(lat) + lon = float(lon) + if not latlon_is_numeric_and_in_geodetic_range(lat, lon): + raise ValueError("Latitude and/or longitude are out of range or not numeric.") + + # TODO: add step to validate geographic lat/lon with geotiff + + lat, lon = project_latlon(lat, lon, from_epsg=4326, to_epsg=3338) + return lat, lon + + +def validate_operator(operator): + if operator not in ["above", "below"]: + raise ValueError("Operator must be 'above' or 'below'.") + if operator == "above": + operator = ">" + else: + operator = "<" + return operator + + +def validate_units_threshold_and_variable(units, variable): + if variable in ["tasmax", "tasmin"]: + if units not in ["C", "F"]: + raise ValueError("Units for temperature must be 'C' or 'F'.") + if units == "F": + threshold = (float(threshold) - 32) * 5.0 / 9.0 + else: + threshold = float(threshold) + elif variable == "pr": + if units not in ["mm", "in"]: + raise ValueError("Units for precipitation must be 'mm' or 'in'.") + if units == "in": + threshold = float(threshold) * 25.4 + else: + threshold = float(threshold) + else: + raise ValueError("Variable must be 'tasmax', 'tasmin', or 'pr'.") + return units, threshold + + @routes.route( - "/dynamic_indicators/count_days_above//////" + "/dynamic_indicators/count_days/////////" ) -def count_days_above(threshold, units, variable, lat, lon): +def count_days_above( + operator, threshold, units, variable, lat, lon, start_year, end_year +): # tasmax coverage - # usage: .../dynamic_indicators/count_days_above/25/C/tasmax/64.5/-147.5/ - # usage: .../dynamic_indicators/count_days_above/5/mm/pr/64.5/-147.5/ - - # sample URL for days above 25C - url = generate_wcs_query_url( - "ProcessCoverages&query=" - + construct_count_annual_days_above_or_below_threshold_wcps_query_string( - "cmip6_downscaled_tasmax_5ModelAvg_ssp585_wcs", - 25, - ">", - 2000, - 2010, - 350000, - 1700000, + # usage: .../dynamic_indicators/count_days/above/25/C/tasmax/64.5/-147.5/2000/2050 + # usage: .../dynamic_indicators/count_days/above/5/mm/pr/64.5/-147.5/2000/2050/ + + # Validate request params + lat, lon = validate_latlon_and_reproject_to_epsg_3338(lat, lon) + operator = validate_operator(operator) + units = validate_units_threshold_and_variable(threshold, units, variable) + validate_year(start_year) + validate_year(end_year) + + # TODO: parse years and list appropriate coverages, for example: + # year_ranges = [(2000,2014), (2015,2050), (2015,2050), (2015,2050), (2015,2050)] + # coverages = ["cmip6_downscaled_pr_5ModelAvg_historical_wcs", + # "cmip6_downscaled_pr_5ModelAvg_ssp126_wcs", + # "cmip6_downscaled_pr_5ModelAvg_ssp245_wcs", + # "cmip6_downscaled_pr_5ModelAvg_ssp370_wcs", + # "cmip6_downscaled_pr_5ModelAvg_ssp585_wcs",] + year_ranges = [] + coverages = [] + + urls = [] + + for coverage, year_range in zip(coverages, year_ranges): + url = generate_wcs_query_url( + "ProcessCoverages&query=" + + construct_count_annual_days_above_or_below_threshold_wcps_query_string( + coverage, + threshold, + operator, + year_range[0], + year_range[1], + lon, + lat, + ) ) - ) + urls.append(url) - return None + data = fetch_data[urls] + # TODO: postprocess the results as needed -@routes.route("/dynamic_indicators/count_days_below/////") -def count_days_below(threshold, units, lat, lon): - # tasmin coverage - # usage: .../dynamic_indicators/count_days_below/-30/C/64.5/-147.5/ - return None + return data -@routes.route("/dynamic_indicators/stat//////") -def get_annual_stat(stat, variable, units, lat, lon): +@routes.route( + "/dynamic_indicators/stat////////" +) +def get_annual_stat(stat, variable, units, lat, lon, start_year, end_year): # pr, tasmax, or tasmin coverage # usage: .../dynamic_indicators/stat/max/pr/mm/64.5/-147.5/ # usage: .../dynamic_indicators/stat/min/tasmin/C/64.5/-147.5/ @@ -112,8 +177,12 @@ def get_annual_stat(stat, variable, units, lat, lon): return None -@routes.route("/dynamic_indicators/rank//////") -def get_annual_stat(position, direction, variable, units, lat, lon): +@routes.route( + "/dynamic_indicators/rank////////" +) +def get_annual_stat( + position, direction, variable, units, lat, lon, start_year, end_year +): # pr, tasmax, or tasmin coverage # usage: .../dynamic_indicators/rank/6/highest/pr/64.5/-147.5/ # usage: .../dynamic_indicators/rank/6/lowest/tasmin/64.5/-147.5/ From ccb3ff908d1c11c32ce7bf120fddf44d5038da4d Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Wed, 22 Oct 2025 08:10:58 -0800 Subject: [PATCH 04/30] add async fetch count_days function --- routes/dynamic_indicators.py | 96 +++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 79a5d694..8e433e60 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -1,12 +1,6 @@ import asyncio -import ast import logging -import xarray as xr -import cftime -import io -import numpy as np from flask import Blueprint, render_template, request -import datetime # local imports from fetch_data import fetch_data @@ -103,35 +97,44 @@ def validate_units_threshold_and_variable(units, variable): return units, threshold -@routes.route( - "/dynamic_indicators/count_days/////////" -) -def count_days_above( - operator, threshold, units, variable, lat, lon, start_year, end_year -): - # tasmax coverage - # usage: .../dynamic_indicators/count_days/above/25/C/tasmax/64.5/-147.5/2000/2050 - # usage: .../dynamic_indicators/count_days/above/5/mm/pr/64.5/-147.5/2000/2050/ +def validate_start_and_end_years(start_year, end_year): + if not (validate_year(start_year) and validate_year(end_year)): + raise ValueError("Start year and/or end year are invalid.") + if start_year >= end_year: + raise ValueError("Start year must be before end year.") + return start_year, end_year - # Validate request params - lat, lon = validate_latlon_and_reproject_to_epsg_3338(lat, lon) - operator = validate_operator(operator) - units = validate_units_threshold_and_variable(threshold, units, variable) - validate_year(start_year) - validate_year(end_year) - - # TODO: parse years and list appropriate coverages, for example: - # year_ranges = [(2000,2014), (2015,2050), (2015,2050), (2015,2050), (2015,2050)] - # coverages = ["cmip6_downscaled_pr_5ModelAvg_historical_wcs", - # "cmip6_downscaled_pr_5ModelAvg_ssp126_wcs", - # "cmip6_downscaled_pr_5ModelAvg_ssp245_wcs", - # "cmip6_downscaled_pr_5ModelAvg_ssp370_wcs", - # "cmip6_downscaled_pr_5ModelAvg_ssp585_wcs",] + +def build_year_and_coverage_lists_for_iteration( + start_year, end_year, variable, time_domains, coverages +): + # if years span historical and projected, need to split into two ranges year_ranges = [] coverages = [] - - urls = [] - + historical_range = time_domains["historical"] + projected_range = time_domains["projected"] + + if start_year < historical_range[1] and end_year > historical_range[0]: + # overlaps historical + hist_start = max(start_year, historical_range[0]) + hist_end = min(end_year, historical_range[1]) + year_ranges.append((hist_start, hist_end)) + coverages.append(coverages[variable][0]) + if start_year < projected_range[1] and end_year > projected_range[0]: + # overlaps projected + proj_start = max(start_year, projected_range[0]) + proj_end = min(end_year, projected_range[1]) + year_ranges.append((proj_start, proj_end)) + # for projected, use all SSPs + for ssp_coverage in coverages[variable][1:]: + year_ranges.append((proj_start, proj_end)) + coverages.append(ssp_coverage) + + return year_ranges, coverages + + +async def fetch_count_days_data(coverages, year_ranges, threshold, operator, lon, lat): + tasks = [] for coverage, year_range in zip(coverages, year_ranges): url = generate_wcs_query_url( "ProcessCoverages&query=" @@ -145,9 +148,32 @@ def count_days_above( lat, ) ) - urls.append(url) + tasks.append(fetch_data[url]) + data = await asyncio.gather(*tasks) + return data - data = fetch_data[urls] + +@routes.route( + "/dynamic_indicators/count_days/////////" +) +def count_days(operator, threshold, units, variable, lat, lon, start_year, end_year): + # pr, tasmin, or tasmax coverage + # usage: .../dynamic_indicators/count_days/above/25/C/tasmax/64.5/-147.5/2000/2050 + # usage: .../dynamic_indicators/count_days/above/5/mm/pr/64.5/-147.5/2000/2050/ + + # Validate request params + lat, lon = validate_latlon_and_reproject_to_epsg_3338(lat, lon) + operator = validate_operator(operator) + units = validate_units_threshold_and_variable(threshold, units, variable) + start_year, end_year = validate_start_and_end_years(start_year, end_year) + + year_ranges, coverages = build_year_and_coverage_lists_for_iteration( + start_year, end_year, variable, time_domains, coverages + ) + + data = asyncio.run( + fetch_count_days_data(coverages, year_ranges, threshold, operator, lon, lat) + ) # TODO: postprocess the results as needed @@ -180,7 +206,7 @@ def get_annual_stat(stat, variable, units, lat, lon, start_year, end_year): @routes.route( "/dynamic_indicators/rank////////" ) -def get_annual_stat( +def get_annual_rank( position, direction, variable, units, lat, lon, start_year, end_year ): # pr, tasmax, or tasmin coverage From cd866e03ef3f72ff20a0c5d3cc3f635295df1eff Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Wed, 22 Oct 2025 11:18:08 -0800 Subject: [PATCH 05/30] working count_days route --- generate_requests.py | 57 ++++++------ routes/dynamic_indicators.py | 168 +++++++++++++++++++++++++++-------- 2 files changed, 163 insertions(+), 62 deletions(-) diff --git a/generate_requests.py b/generate_requests.py index 767fb324..34b15f33 100644 --- a/generate_requests.py +++ b/generate_requests.py @@ -223,18 +223,23 @@ def construct_get_annual_mmm_stat_wcps_query_string( x_coord = str(x_coord) y_coord = str(y_coord) - query_string = f""" - for $cov in ({coverage}) - let $start_year := "{start_year}", - $end_year := "{end_year}", - $x_coord := {x_coord}, - $y_coord := {y_coord} - return encode( - coverage result - over $pt t($start_year : $end_year) - values {operator} ( $cov[x($x_coord), y($y_coord), ansi($pt : $pt)] ) - , "{format}") - """ + query_string = quote( + ( + f"for $cov in ({coverage}) " + f'let $start_year := "{start_year}", ' + f'$end_year := "{end_year}", ' + f"$x_coord := {x_coord}, " + f"$y_coord := {y_coord} " + f"return encode( " + f"coverage result " + f"over $pt t($start_year : $end_year) " + f"values {operator} ( $cov[x($x_coord), y($y_coord), ansi($pt : $pt)] ) " + f', "{format}")' + ) + ) + + print(query_string) + return query_string @@ -267,17 +272,19 @@ def construct_count_annual_days_above_or_below_threshold_wcps_query_string( x_coord = str(x_coord) y_coord = str(y_coord) - query_string = f""" - for $cov in ({coverage}) - let $threshold := {threshold}, - $start_year := "{start_year}", - $end_year := "{end_year}", - $x_coord := {x_coord}, - $y_coord := {y_coord} - return encode( - coverage result - over $pt t($start_year : $end_year) - values ( sum($cov[x($x_coord), y($y_coord), ansi($pt : $pt)] {operator} $threshold) ) - , "{format}") - """ + query_string = quote( + ( + f"for $cov in ({coverage}) " + f"let $threshold := {threshold}, " + f'$start_year := "{start_year}", ' + f'$end_year := "{end_year}", ' + f"$x_coord := {x_coord}, " + f"$y_coord := {y_coord} " + f"return encode( " + f"coverage result " + f"over $pt t($start_year : $end_year) " + f"values ( sum($cov[x($x_coord), y($y_coord), ansi($pt : $pt)] {operator} $threshold) ) " + f', "{format}")' + ) + ) return query_string diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 8e433e60..13e0f0fc 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -25,7 +25,7 @@ cmip6_api = Blueprint("dynamic_indicators_api", __name__) -coverages = { +all_coverages = { "pr": [ "cmip6_downscaled_pr_5ModelAvg_historical_wcs", "cmip6_downscaled_pr_5ModelAvg_ssp126_wcs", @@ -50,8 +50,14 @@ } time_domains = { - "historical": (1965, 2014), - "projected": (2015, 2100), + "historical": ( + 1966, + 2014, + ), # actual data starts at 1965, but we are having an issue with noon time stamps at lower bound + "projected": ( + 2016, + 2100, + ), # actual data starts at 2015, but we are having an issue with noon time stamps at lower bound } @@ -63,8 +69,9 @@ def validate_latlon_and_reproject_to_epsg_3338(lat, lon): # TODO: add step to validate geographic lat/lon with geotiff - lat, lon = project_latlon(lat, lon, from_epsg=4326, to_epsg=3338) - return lat, lon + lon, lat = project_latlon(lat, lon, dst_crs=3338) + + return lon, lat def validate_operator(operator): @@ -77,7 +84,7 @@ def validate_operator(operator): return operator -def validate_units_threshold_and_variable(units, variable): +def validate_units_threshold_and_variable(units, threshold, variable): if variable in ["tasmax", "tasmin"]: if units not in ["C", "F"]: raise ValueError("Units for temperature must be 'C' or 'F'.") @@ -94,23 +101,19 @@ def validate_units_threshold_and_variable(units, variable): threshold = float(threshold) else: raise ValueError("Variable must be 'tasmax', 'tasmin', or 'pr'.") - return units, threshold + # TODO: validate threshold ranges based on variable if needed + # ie, temperature thresholds within reasonable limits, no negative precip allowed, etc. -def validate_start_and_end_years(start_year, end_year): - if not (validate_year(start_year) and validate_year(end_year)): - raise ValueError("Start year and/or end year are invalid.") - if start_year >= end_year: - raise ValueError("Start year must be before end year.") - return start_year, end_year + return units, threshold def build_year_and_coverage_lists_for_iteration( - start_year, end_year, variable, time_domains, coverages + start_year, end_year, variable, time_domains, all_coverages ): # if years span historical and projected, need to split into two ranges year_ranges = [] - coverages = [] + var_coverages = [] historical_range = time_domains["historical"] projected_range = time_domains["projected"] @@ -119,65 +122,156 @@ def build_year_and_coverage_lists_for_iteration( hist_start = max(start_year, historical_range[0]) hist_end = min(end_year, historical_range[1]) year_ranges.append((hist_start, hist_end)) - coverages.append(coverages[variable][0]) + var_coverages.append(all_coverages[variable][0]) if start_year < projected_range[1] and end_year > projected_range[0]: # overlaps projected proj_start = max(start_year, projected_range[0]) proj_end = min(end_year, projected_range[1]) year_ranges.append((proj_start, proj_end)) # for projected, use all SSPs - for ssp_coverage in coverages[variable][1:]: + for ssp_coverage in all_coverages[variable][1:]: year_ranges.append((proj_start, proj_end)) - coverages.append(ssp_coverage) + var_coverages.append(ssp_coverage) - return year_ranges, coverages + return year_ranges, var_coverages -async def fetch_count_days_data(coverages, year_ranges, threshold, operator, lon, lat): +async def fetch_count_days_data( + var_coverages, year_ranges, threshold, operator, lon, lat +): tasks = [] - for coverage, year_range in zip(coverages, year_ranges): + for coverage, year_range in zip(var_coverages, year_ranges): url = generate_wcs_query_url( "ProcessCoverages&query=" + construct_count_annual_days_above_or_below_threshold_wcps_query_string( coverage, - threshold, operator, + threshold, year_range[0], year_range[1], - lon, - lat, + x_coord=lon, + y_coord=lat, ) ) - tasks.append(fetch_data[url]) + + tasks.append(fetch_data([url])) data = await asyncio.gather(*tasks) + return data +def postprocess_count_days(data, start_year, end_year): + # if the year range spans historical and projected, our data will be a list of 5 lists: + # the first has day counts for each historical year, and the rest have day counts for each projected year for each SSP + # if the year range is only historical or only projected, our data will be a list of 1-4 lists accordingly + + # we want to create dictionary for output, with the following structure: + # { + # "historical": { + # "data": {"2000": 45, "2001": 50, ...}, + # "summary": {"min": 30, "max": 60, "mean": 45.5}, + # }, + # "projected": { + # "ssp126": { + # "data": {"2020": 55, "2021": 60, ...}, + # "summary": {"min": 40, "max": 70, "mean": 55.5}, + # }, + # "ssp245": { + # "data": {"2020": 50, "2021": 55, ...}, + # "summary": {"min": 35, "max": 65, "mean": 50.5}, + # }, + # ... + # } + # } + + start_year = int(start_year) + end_year = int(end_year) + + result = {} + current_index = 0 + if ( + start_year < time_domains["historical"][1] + and end_year > time_domains["historical"][0] + ): + # historical data present + hist_data = data[current_index] + hist_years = list( + range( + max(start_year, time_domains["historical"][0]), + min(end_year, time_domains["historical"][1]) + 1, + ) + ) + hist_day_counts = {str(year): hist_data[i] for i, year in enumerate(hist_years)} + result["historical"] = { + "data": hist_day_counts, + "summary": { + "min": min(hist_day_counts.values()), + "max": max(hist_day_counts.values()), + "mean": sum(hist_day_counts.values()) / len(hist_day_counts), + }, + } + current_index += 1 + if ( + start_year < time_domains["projected"][1] + and end_year > time_domains["projected"][0] + ): + # projected data present + result["projected"] = {} + ssp_names = ["ssp126", "ssp245", "ssp370", "ssp585"] + proj_years = list( + range( + max(start_year, time_domains["projected"][0]), + min(end_year, time_domains["projected"][1]) + 1, + ) + ) + for ssp in ssp_names: + proj_data = data[current_index] + proj_day_counts = { + str(year): proj_data[i] for i, year in enumerate(proj_years) + } + result["projected"][ssp] = { + "data": proj_day_counts, + "summary": { + "min": min(proj_day_counts.values()), + "max": max(proj_day_counts.values()), + "mean": sum(proj_day_counts.values()) / len(proj_day_counts), + }, + } + current_index += 1 + return result + + @routes.route( "/dynamic_indicators/count_days/////////" ) def count_days(operator, threshold, units, variable, lat, lon, start_year, end_year): - # pr, tasmin, or tasmax coverage - # usage: .../dynamic_indicators/count_days/above/25/C/tasmax/64.5/-147.5/2000/2050 - # usage: .../dynamic_indicators/count_days/above/5/mm/pr/64.5/-147.5/2000/2050/ + # example usage: + # http://127.0.0.1:5000/dynamic_indicators/count_days/above/25/C/tasmax/64.5/-147.5/2000/2030/ + # http://127.0.0.1:5000/dynamic_indicators/count_days/above/5/mm/pr/64.5/-147.5/2000/2030/ # Validate request params - lat, lon = validate_latlon_and_reproject_to_epsg_3338(lat, lon) - operator = validate_operator(operator) - units = validate_units_threshold_and_variable(threshold, units, variable) - start_year, end_year = validate_start_and_end_years(start_year, end_year) + try: + lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon) + operator = validate_operator(operator) + units, threshold = validate_units_threshold_and_variable( + units, threshold, variable + ) + validate_year(start_year, end_year) + except ValueError as e: + return {"error": str(e)}, 400 - year_ranges, coverages = build_year_and_coverage_lists_for_iteration( - start_year, end_year, variable, time_domains, coverages + # build lists for iteration + year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( + int(start_year), int(end_year), variable, time_domains, all_coverages ) data = asyncio.run( - fetch_count_days_data(coverages, year_ranges, threshold, operator, lon, lat) + fetch_count_days_data(var_coverages, year_ranges, threshold, operator, lon, lat) ) - # TODO: postprocess the results as needed + result = postprocess_count_days(data, start_year, end_year) - return data + return result @routes.route( From 46b62488dc4b8f8e59f296e70d03c97400ff2e70 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Wed, 22 Oct 2025 12:59:28 -0800 Subject: [PATCH 06/30] finish fetching and post-processing functions --- routes/dynamic_indicators.py | 315 +++++++++++++++++++++++++++++------ 1 file changed, 268 insertions(+), 47 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 13e0f0fc..54dc3699 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -53,11 +53,11 @@ "historical": ( 1966, 2014, - ), # actual data starts at 1965, but we are having an issue with noon time stamps at lower bound + ), # actual data starts at 1965, but we are having an issue with noon time stamps at lower bound! Ask JP for more details "projected": ( 2016, 2100, - ), # actual data starts at 2015, but we are having an issue with noon time stamps at lower bound + ), # actual data starts at 2015, but we are having an issue with noon time stamps at lower bound! Ask JP for more details } @@ -88,17 +88,19 @@ def validate_units_threshold_and_variable(units, threshold, variable): if variable in ["tasmax", "tasmin"]: if units not in ["C", "F"]: raise ValueError("Units for temperature must be 'C' or 'F'.") - if units == "F": - threshold = (float(threshold) - 32) * 5.0 / 9.0 - else: - threshold = float(threshold) + if threshold is not None: + if units == "F": + threshold = (float(threshold) - 32) * 5.0 / 9.0 + else: + threshold = float(threshold) elif variable == "pr": if units not in ["mm", "in"]: raise ValueError("Units for precipitation must be 'mm' or 'in'.") - if units == "in": - threshold = float(threshold) * 25.4 - else: - threshold = float(threshold) + if threshold is not None: + if units == "in": + threshold = float(threshold) * 25.4 + else: + threshold = float(threshold) else: raise ValueError("Variable must be 'tasmax', 'tasmin', or 'pr'.") @@ -108,6 +110,27 @@ def validate_units_threshold_and_variable(units, threshold, variable): return units, threshold +def validate_stat(stat): + if stat not in ["max", "min", "mean", "sum"]: + raise ValueError("Stat must be 'max', 'min', 'mean', or 'sum'.") + if stat == "mean": + stat = "avg" # rasdaman uses 'avg' instead of 'mean' in WCPS queries + return stat + + +def validate_rank_position_and_direction(position, direction): + # position must be between 1 and 365, and direction must be "highest" or "lowest" + try: + position = int(position) + if position < 1 or position > 365: + raise ValueError + except ValueError: + raise ValueError("Position must be an integer between 1 and 365.") + if direction not in ["highest", "lowest"]: + raise ValueError("Direction must be 'highest' or 'lowest'.") + return position, direction + + def build_year_and_coverage_lists_for_iteration( start_year, end_year, variable, time_domains, all_coverages ): @@ -241,20 +264,194 @@ def postprocess_count_days(data, start_year, end_year): return result +async def fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat): + tasks = [] + for coverage, year_range in zip(var_coverages, year_ranges): + url = generate_wcs_query_url( + "ProcessCoverages&query=" + + construct_get_annual_mmm_stat_wcps_query_string( + coverage, + stat, + year_range[0], + year_range[1], + x_coord=lon, + y_coord=lat, + ) + ) + tasks.append(fetch_data([url])) + data = await asyncio.gather(*tasks) + + return data + + +def postprocess_annual_stat(data, start_year, end_year, units): + # data in / out is similar to postprocess_count_days + # define conversion functions + def mm_to_inches(mm): + return mm / 25.4 + + def c_to_f(c): + return (c * 9 / 5) + 32 + + # Determine if and how to convert + if units == "in": + convert = mm_to_inches + elif units == "F": + convert = c_to_f + else: + convert = lambda x: x # No conversion + + start_year = int(start_year) + end_year = int(end_year) + result = {} + current_index = 0 + + if ( + start_year < time_domains["historical"][1] + and end_year > time_domains["historical"][0] + ): + # historical data present + hist_data = data[current_index] + hist_years = list( + range( + max(start_year, time_domains["historical"][0]), + min(end_year, time_domains["historical"][1]) + 1, + ) + ) + hist_stats = { + str(year): convert(hist_data[i]) for i, year in enumerate(hist_years) + } + result["historical"] = { + "data": hist_stats, + "summary": { + "min": min(hist_stats.values()), + "max": max(hist_stats.values()), + "mean": sum(hist_stats.values()) / len(hist_stats), + }, + } + current_index += 1 + + if ( + start_year < time_domains["projected"][1] + and end_year > time_domains["projected"][0] + ): + # projected data present + result["projected"] = {} + ssp_names = ["ssp126", "ssp245", "ssp370", "ssp585"] + proj_years = list( + range( + max(start_year, time_domains["projected"][0]), + min(end_year, time_domains["projected"][1]) + 1, + ) + ) + for ssp in ssp_names: + proj_data = data[current_index] + proj_stats = { + str(year): convert(proj_data[i]) for i, year in enumerate(proj_years) + } + result["projected"][ssp] = { + "data": proj_stats, + "summary": { + "min": min(proj_stats.values()), + "max": max(proj_stats.values()), + "mean": sum(proj_stats.values()) / len(proj_stats), + }, + } + current_index += 1 + + return result + + +def postprocess_annual_rank(data, start_year, end_year, position, direction): + # data in / out is similar to postprocess_annual_stat + start_year = int(start_year) + end_year = int(end_year) + result = {} + current_index = 0 + + if ( + start_year < time_domains["historical"][1] + and end_year > time_domains["historical"][0] + ): + # historical data present + hist_data = data[current_index] + hist_years = list( + range( + max(start_year, time_domains["historical"][0]), + min(end_year, time_domains["historical"][1]) + 1, + ) + ) + hist_ranks = {} + for i, year in enumerate(hist_years): + sorted_values = sorted(hist_data[i]) + if direction == "highest": + rank_value = sorted_values[-position] + else: + rank_value = sorted_values[position - 1] + hist_ranks[str(year)] = rank_value + result["historical"] = { + "data": hist_ranks, + "summary": { + "min": min(hist_ranks.values()), + "max": max(hist_ranks.values()), + "mean": sum(hist_ranks.values()) / len(hist_ranks), + }, + } + + current_index += 1 + + if ( + start_year < time_domains["projected"][1] + and end_year > time_domains["projected"][0] + ): + # projected data present + result["projected"] = {} + ssp_names = ["ssp126", "ssp245", "ssp370", "ssp585"] + proj_years = list( + range( + max(start_year, time_domains["projected"][0]), + min(end_year, time_domains["projected"][1]) + 1, + ) + ) + for ssp in ssp_names: + proj_data = data[current_index] + proj_ranks = {} + for i, year in enumerate(proj_years): + sorted_values = sorted(proj_data[i]) + if direction == "highest": + rank_value = sorted_values[-position] + else: + rank_value = sorted_values[position - 1] + proj_ranks[str(year)] = rank_value + result["projected"][ssp] = { + "data": proj_ranks, + "summary": { + "min": min(proj_ranks.values()), + "max": max(proj_ranks.values()), + "mean": sum(proj_ranks.values()) / len(proj_ranks), + }, + } + current_index += 1 + + return result + + @routes.route( "/dynamic_indicators/count_days/////////" ) def count_days(operator, threshold, units, variable, lat, lon, start_year, end_year): # example usage: - # http://127.0.0.1:5000/dynamic_indicators/count_days/above/25/C/tasmax/64.5/-147.5/2000/2030/ - # http://127.0.0.1:5000/dynamic_indicators/count_days/above/5/mm/pr/64.5/-147.5/2000/2030/ + # http://127.0.0.1:5000/dynamic_indicators/count_days/above/25/C/tasmax/64.5/-147.5/2000/2030/ ->>> can recreate the "summer days" indicator + # http://127.0.0.1:5000/dynamic_indicators/count_days/below/-30/C/tasmax/64.5/-147.5/2000/2030/ ->>> can recreate the "deep winter days" indicator + # http://127.0.0.1:5000/dynamic_indicators/count_days/above/10/mm/pr/64.5/-147.5/2000/2030/ ->>> can recreate the "days above 10mm precip" indicator + # http://127.0.0.1:5000/dynamic_indicators/count_days/above/1/mm/pr/64.5/-147.5/2000/2030/ ->>> can recreate the "wet days" indicator # Validate request params try: lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon) operator = validate_operator(operator) units, threshold = validate_units_threshold_and_variable( - units, threshold, variable + units=units, threshold=threshold, variable=variable ) validate_year(start_year, end_year) except ValueError as e: @@ -278,45 +475,69 @@ def count_days(operator, threshold, units, variable, lat, lon, start_year, end_y "/dynamic_indicators/stat////////" ) def get_annual_stat(stat, variable, units, lat, lon, start_year, end_year): - # pr, tasmax, or tasmin coverage - # usage: .../dynamic_indicators/stat/max/pr/mm/64.5/-147.5/ - # usage: .../dynamic_indicators/stat/min/tasmin/C/64.5/-147.5/ - - # sample URL for maxmimum daily precip - url = generate_wcs_query_url( - "ProcessCoverages&query=" - + construct_get_annual_mmm_stat_wcps_query_string( - "cmip6_downscaled_prx_5ModelAvg_ssp585_wcs", - "max", - 2000, - 2010, - 350000, - 1700000, + # example usage: + # http://127.0.0.1:5000/dynamic_indicators/stat/max/pr/mm/64.5/-147.5/2000/2030 ->>> can recreate the "maxmimum one day precip" indicator + # http://127.0.0.1:5000/dynamic_indicators/stat/min/tasmin/C/64.5/-147.5/2000/2030 ->>> coldest day per year + # http://127.0.0.1:5000/dynamic_indicators/stat/max/tasmax/C/64.5/-147.5/2000/2030 ->>> hottest day per year + # http://127.0.0.1:5000/dynamic_indicators/stat/sum/pr/mm/64.5/-147.5/2000/2030/ ->>> total annual precipitation (NOTE: summary section of return will show mean annual precip over the year range) + # http://127.0.0.1:5000/dynamic_indicators/stat/mean/pr/mm/64.5/-147.5/2000/2030/ ->>> mean daily precipitation (NOTE: this is not a common mean statistic for precip - avg amount of precip per day over the year) + + # Validate request params + try: + stat = validate_stat(stat) + lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon) + units, _threshold = validate_units_threshold_and_variable( + units=units, threshold=None, variable=variable ) + validate_year(start_year, end_year) + except ValueError as e: + return {"error": str(e)}, 400 + + # build lists for iteration + year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( + int(start_year), int(end_year), variable, time_domains, all_coverages ) - return None + + data = asyncio.run( + fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat) + ) + + result = postprocess_annual_stat(data, start_year, end_year, units) + + return result @routes.route( "/dynamic_indicators/rank////////" ) -def get_annual_rank( - position, direction, variable, units, lat, lon, start_year, end_year -): - # pr, tasmax, or tasmin coverage - # usage: .../dynamic_indicators/rank/6/highest/pr/64.5/-147.5/ - # usage: .../dynamic_indicators/rank/6/lowest/tasmin/64.5/-147.5/ - - # sample URL for all values - postprocess this for ranking - url = generate_wcs_query_url( - "ProcessCoverages&query=" - + construct_get_annual_mmm_stat_wcps_query_string( - "cmip6_downscaled_prx_5ModelAvg_ssp585_wcs", - "", - 2000, - 2010, - 350000, - 1700000, - ) +def get_annual_rank(position, direction, variable, lat, lon, start_year, end_year): + # example usage: + # http://127.0.0.1:5000/dynamic_indicators/rank/6/highest/tasmax/64.5/-147.5/2000/2030/ ->>> can recreate "hot day threshold" indicator + # http://127.0.0.1:5000/dynamic_indicators/rank/6/lowest/tasmin/64.5/-147.5/2000/2030/ ->>> can recreate "cold day threshold" indicators + + # Validate request params + try: + position, direction = validate_rank_position_and_direction(position, direction) + lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon) + validate_year(start_year, end_year) + except ValueError as e: + return {"error": str(e)}, 400 + if variable not in ["tasmax", "tasmin", "pr"]: + raise ValueError("Variable must be 'tasmax', 'tasmin', or 'pr'.") + + # build lists for iteration + year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( + int(start_year), int(end_year), variable, time_domains, all_coverages + ) + + stat = ( + "" # omitting stat will force a return of all values, which we need for ranking + ) + + data = asyncio.run( + fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat) ) - return None + + result = postprocess_annual_rank(data, start_year, end_year, position, direction) + + return result From f0c876245a792072fafa473da9fa3a5dcc6a451c Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Wed, 22 Oct 2025 13:09:53 -0800 Subject: [PATCH 07/30] replace errors with templates --- routes/dynamic_indicators.py | 84 ++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 36 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 54dc3699..44f12405 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -65,7 +65,7 @@ def validate_latlon_and_reproject_to_epsg_3338(lat, lon): lat = float(lat) lon = float(lon) if not latlon_is_numeric_and_in_geodetic_range(lat, lon): - raise ValueError("Latitude and/or longitude are out of range or not numeric.") + return render_template("400/bad_request.html"), 400 # TODO: add step to validate geographic lat/lon with geotiff @@ -76,7 +76,7 @@ def validate_latlon_and_reproject_to_epsg_3338(lat, lon): def validate_operator(operator): if operator not in ["above", "below"]: - raise ValueError("Operator must be 'above' or 'below'.") + return render_template("400/bad_request.html"), 400 if operator == "above": operator = ">" else: @@ -95,14 +95,14 @@ def validate_units_threshold_and_variable(units, threshold, variable): threshold = float(threshold) elif variable == "pr": if units not in ["mm", "in"]: - raise ValueError("Units for precipitation must be 'mm' or 'in'.") + return render_template("400/bad_request.html"), 400 if threshold is not None: if units == "in": threshold = float(threshold) * 25.4 else: threshold = float(threshold) else: - raise ValueError("Variable must be 'tasmax', 'tasmin', or 'pr'.") + return render_template("400/bad_request.html"), 400 # TODO: validate threshold ranges based on variable if needed # ie, temperature thresholds within reasonable limits, no negative precip allowed, etc. @@ -112,7 +112,7 @@ def validate_units_threshold_and_variable(units, threshold, variable): def validate_stat(stat): if stat not in ["max", "min", "mean", "sum"]: - raise ValueError("Stat must be 'max', 'min', 'mean', or 'sum'.") + return render_template("400/bad_request.html"), 400 if stat == "mean": stat = "avg" # rasdaman uses 'avg' instead of 'mean' in WCPS queries return stat @@ -125,9 +125,9 @@ def validate_rank_position_and_direction(position, direction): if position < 1 or position > 365: raise ValueError except ValueError: - raise ValueError("Position must be an integer between 1 and 365.") + return render_template("400/bad_request.html"), 400 if direction not in ["highest", "lowest"]: - raise ValueError("Direction must be 'highest' or 'lowest'.") + return render_template("400/bad_request.html"), 400 return position, direction @@ -454,21 +454,26 @@ def count_days(operator, threshold, units, variable, lat, lon, start_year, end_y units=units, threshold=threshold, variable=variable ) validate_year(start_year, end_year) - except ValueError as e: - return {"error": str(e)}, 400 + except: + return render_template("400/bad_request.html"), 400 # build lists for iteration year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( int(start_year), int(end_year), variable, time_domains, all_coverages ) - data = asyncio.run( - fetch_count_days_data(var_coverages, year_ranges, threshold, operator, lon, lat) - ) - - result = postprocess_count_days(data, start_year, end_year) - - return result + try: + data = asyncio.run( + fetch_count_days_data( + var_coverages, year_ranges, threshold, operator, lon, lat + ) + ) + result = postprocess_count_days(data, start_year, end_year) + return result + except Exception as exc: + if hasattr(exc, "status") and exc.status == 404: + return render_template("404/no_data.html"), 404 + return render_template("500/server_error.html"), 500 @routes.route( @@ -490,21 +495,24 @@ def get_annual_stat(stat, variable, units, lat, lon, start_year, end_year): units=units, threshold=None, variable=variable ) validate_year(start_year, end_year) - except ValueError as e: - return {"error": str(e)}, 400 + except: + return render_template("400/bad_request.html"), 400 # build lists for iteration year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( int(start_year), int(end_year), variable, time_domains, all_coverages ) - data = asyncio.run( - fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat) - ) - - result = postprocess_annual_stat(data, start_year, end_year, units) - - return result + try: + data = asyncio.run( + fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat) + ) + result = postprocess_annual_stat(data, start_year, end_year, units) + return result + except Exception as exc: + if hasattr(exc, "status") and exc.status == 404: + return render_template("404/no_data.html"), 404 + return render_template("500/server_error.html"), 500 @routes.route( @@ -520,10 +528,10 @@ def get_annual_rank(position, direction, variable, lat, lon, start_year, end_yea position, direction = validate_rank_position_and_direction(position, direction) lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon) validate_year(start_year, end_year) - except ValueError as e: - return {"error": str(e)}, 400 + except: + return render_template("400/bad_request.html"), 400 if variable not in ["tasmax", "tasmin", "pr"]: - raise ValueError("Variable must be 'tasmax', 'tasmin', or 'pr'.") + return render_template("400/bad_request.html"), 400 # build lists for iteration year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( @@ -533,11 +541,15 @@ def get_annual_rank(position, direction, variable, lat, lon, start_year, end_yea stat = ( "" # omitting stat will force a return of all values, which we need for ranking ) - - data = asyncio.run( - fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat) - ) - - result = postprocess_annual_rank(data, start_year, end_year, position, direction) - - return result + try: + data = asyncio.run( + fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat) + ) + result = postprocess_annual_rank( + data, start_year, end_year, position, direction + ) + return result + except Exception as exc: + if hasattr(exc, "status") and exc.status == 404: + return render_template("404/no_data.html"), 404 + return render_template("500/server_error.html"), 500 From 062e484fde8a01ec812185295d8f2daf260c3426 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Wed, 22 Oct 2025 13:40:19 -0800 Subject: [PATCH 08/30] add more notes --- routes/dynamic_indicators.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 44f12405..67ae2f30 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -53,11 +53,11 @@ "historical": ( 1966, 2014, - ), # actual data starts at 1965, but we are having an issue with noon time stamps at lower bound! Ask JP for more details + ), # NOTE: actual data starts at 1965, but we are having an issue with noon time stamps at lower bound! Ask JP for more details "projected": ( 2016, 2100, - ), # actual data starts at 2015, but we are having an issue with noon time stamps at lower bound! Ask JP for more details + ), # NOTE: actual data starts at 2015, but we are having an issue with noon time stamps at lower bound! Ask JP for more details } @@ -114,7 +114,7 @@ def validate_stat(stat): if stat not in ["max", "min", "mean", "sum"]: return render_template("400/bad_request.html"), 400 if stat == "mean": - stat = "avg" # rasdaman uses 'avg' instead of 'mean' in WCPS queries + stat = "avg" # NOTE: rasdaman uses 'avg' instead of 'mean' in WCPS queries return stat @@ -293,13 +293,12 @@ def mm_to_inches(mm): def c_to_f(c): return (c * 9 / 5) + 32 - # Determine if and how to convert if units == "in": convert = mm_to_inches elif units == "F": convert = c_to_f else: - convert = lambda x: x # No conversion + convert = lambda x: x start_year = int(start_year) end_year = int(end_year) @@ -538,9 +537,7 @@ def get_annual_rank(position, direction, variable, lat, lon, start_year, end_yea int(start_year), int(end_year), variable, time_domains, all_coverages ) - stat = ( - "" # omitting stat will force a return of all values, which we need for ranking - ) + stat = "" # NOTE: omitting stat will force a return of all values, which we need for ranking try: data = asyncio.run( fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat) From e083a7711877f290ffea40d7d9caacbf455ee416 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Wed, 22 Oct 2025 14:00:02 -0800 Subject: [PATCH 09/30] fix typo --- routes/dynamic_indicators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 67ae2f30..55a8b1be 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -441,7 +441,7 @@ def postprocess_annual_rank(data, start_year, end_year, position, direction): def count_days(operator, threshold, units, variable, lat, lon, start_year, end_year): # example usage: # http://127.0.0.1:5000/dynamic_indicators/count_days/above/25/C/tasmax/64.5/-147.5/2000/2030/ ->>> can recreate the "summer days" indicator - # http://127.0.0.1:5000/dynamic_indicators/count_days/below/-30/C/tasmax/64.5/-147.5/2000/2030/ ->>> can recreate the "deep winter days" indicator + # http://127.0.0.1:5000/dynamic_indicators/count_days/below/-30/C/tasmin/64.5/-147.5/2000/2030/ ->>> can recreate the "deep winter days" indicator # http://127.0.0.1:5000/dynamic_indicators/count_days/above/10/mm/pr/64.5/-147.5/2000/2030/ ->>> can recreate the "days above 10mm precip" indicator # http://127.0.0.1:5000/dynamic_indicators/count_days/above/1/mm/pr/64.5/-147.5/2000/2030/ ->>> can recreate the "wet days" indicator From 665472d86a9fc7ee9e26d1842d6269fd1105a6ce Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Thu, 23 Oct 2025 10:43:26 -0800 Subject: [PATCH 10/30] change coverage IDs to fetch from 6model avg --- routes/dynamic_indicators.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 55a8b1be..2e3d27db 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -27,25 +27,25 @@ all_coverages = { "pr": [ - "cmip6_downscaled_pr_5ModelAvg_historical_wcs", - "cmip6_downscaled_pr_5ModelAvg_ssp126_wcs", - "cmip6_downscaled_pr_5ModelAvg_ssp245_wcs", - "cmip6_downscaled_pr_5ModelAvg_ssp370_wcs", - "cmip6_downscaled_pr_5ModelAvg_ssp585_wcs", + "cmip6_downscaled_pr_6ModelAvg_historical_wcs", + "cmip6_downscaled_pr_6ModelAvg_ssp126_wcs", + "cmip6_downscaled_pr_6ModelAvg_ssp245_wcs", + "cmip6_downscaled_pr_6ModelAvg_ssp370_wcs", + "cmip6_downscaled_pr_6ModelAvg_ssp585_wcs", ], "tasmin": [ - "cmip6_downscaled_tasmin_5ModelAvg_historical_wcs", - "cmip6_downscaled_tasmin_5ModelAvg_ssp126_wcs", - "cmip6_downscaled_tasmin_5ModelAvg_ssp245_wcs", - "cmip6_downscaled_tasmin_5ModelAvg_ssp370_wcs", - "cmip6_downscaled_tasmin_5ModelAvg_ssp585_wcs", + "cmip6_downscaled_tasmin_6ModelAvg_historical_wcs", + "cmip6_downscaled_tasmin_6ModelAvg_ssp126_wcs", + "cmip6_downscaled_tasmin_6ModelAvg_ssp245_wcs", + "cmip6_downscaled_tasmin_6ModelAvg_ssp370_wcs", + "cmip6_downscaled_tasmin_6ModelAvg_ssp585_wcs", ], "tasmax": [ - "cmip6_downscaled_tasmax_5ModelAvg_historical_wcs", - "cmip6_downscaled_tasmax_5ModelAvg_ssp126_wcs", - "cmip6_downscaled_tasmax_5ModelAvg_ssp245_wcs", - "cmip6_downscaled_tasmax_5ModelAvg_ssp370_wcs", - "cmip6_downscaled_tasmax_5ModelAvg_ssp585_wcs", + "cmip6_downscaled_tasmax_6ModelAvg_historical_wcs", + "cmip6_downscaled_tasmax_6ModelAvg_ssp126_wcs", + "cmip6_downscaled_tasmax_6ModelAvg_ssp245_wcs", + "cmip6_downscaled_tasmax_6ModelAvg_ssp370_wcs", + "cmip6_downscaled_tasmax_6ModelAvg_ssp585_wcs", ], } From 764f26150191eb960051c32bbd964a0a9ec04eec Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Wed, 5 Nov 2025 10:13:56 -0900 Subject: [PATCH 11/30] add bbox validation and function documentation --- routes/dynamic_indicators.py | 149 ++++++++++++++++++++++------------- 1 file changed, 96 insertions(+), 53 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 2e3d27db..6a6e6f29 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -3,7 +3,7 @@ from flask import Blueprint, render_template, request # local imports -from fetch_data import fetch_data +from fetch_data import fetch_data, describe_via_wcps from generate_urls import generate_wcs_query_url from generate_requests import ( construct_count_annual_days_above_or_below_threshold_wcps_query_string, @@ -13,6 +13,8 @@ latlon_is_numeric_and_in_geodetic_range, validate_year, project_latlon, + validate_latlon_in_bboxes, + construct_latlon_bbox_from_coverage_bounds, ) # TODO: for additional postprocessing or csv output, uncomment these imports and add code @@ -61,13 +63,41 @@ } -def validate_latlon_and_reproject_to_epsg_3338(lat, lon): +async def get_cmip6_metadata(cov_id): + """Get the coverage metadata and encodings for CMIP6 downscaled daily coverage""" + metadata = await describe_via_wcps(cov_id) + return metadata + + +def validate_latlon_and_reproject_to_epsg_3338(lat, lon, variable): + """Validate lat/lon, then reproject to EPSG:3338""" lat = float(lat) lon = float(lon) if not latlon_is_numeric_and_in_geodetic_range(lat, lon): return render_template("400/bad_request.html"), 400 - # TODO: add step to validate geographic lat/lon with geotiff + var_coverages = all_coverages.get(variable) + + for cov_id in var_coverages: + + metadata = asyncio.run(get_cmip6_metadata(cov_id)) + cmip6_downscaled_bbox = construct_latlon_bbox_from_coverage_bounds(metadata) + within_bounds = validate_latlon_in_bboxes( + lat, lon, [cmip6_downscaled_bbox], [cov_id] + ) + if within_bounds == 404: + return ( + render_template("404/no_data.html"), + 404, + ) + if within_bounds == 422: + return ( + render_template( + "422/invalid_latlon_outside_coverage.html", + bboxes=[cmip6_downscaled_bbox], + ), + 422, + ) lon, lat = project_latlon(lat, lon, dst_crs=3338) @@ -75,6 +105,7 @@ def validate_latlon_and_reproject_to_epsg_3338(lat, lon): def validate_operator(operator): + """Validate operator is 'above' or 'below' and convert to '>' or '<'""" if operator not in ["above", "below"]: return render_template("400/bad_request.html"), 400 if operator == "above": @@ -85,6 +116,7 @@ def validate_operator(operator): def validate_units_threshold_and_variable(units, threshold, variable): + """Validate units and threshold based on variable type. Convert threshold to standard units if needed.""" if variable in ["tasmax", "tasmin"]: if units not in ["C", "F"]: raise ValueError("Units for temperature must be 'C' or 'F'.") @@ -104,13 +136,15 @@ def validate_units_threshold_and_variable(units, threshold, variable): else: return render_template("400/bad_request.html"), 400 - # TODO: validate threshold ranges based on variable if needed - # ie, temperature thresholds within reasonable limits, no negative precip allowed, etc. + # precipitation thresholds should be >= 0 + if variable == "pr" and threshold is not None and threshold < 0: + return render_template("400/bad_request.html"), 400 return units, threshold def validate_stat(stat): + """Validate that stat is one of 'max', 'min', 'mean', or 'sum'.""" if stat not in ["max", "min", "mean", "sum"]: return render_template("400/bad_request.html"), 400 if stat == "mean": @@ -119,7 +153,7 @@ def validate_stat(stat): def validate_rank_position_and_direction(position, direction): - # position must be between 1 and 365, and direction must be "highest" or "lowest" + """Validate rank position and direction. Position must be between 1 and 365, and direction must be 'highest' or 'lowest'.""" try: position = int(position) if position < 1 or position > 365: @@ -134,7 +168,7 @@ def validate_rank_position_and_direction(position, direction): def build_year_and_coverage_lists_for_iteration( start_year, end_year, variable, time_domains, all_coverages ): - # if years span historical and projected, need to split into two ranges + """Build lists of year ranges and variable coverages for iteration based on start and end years. If years span historical and projected, need to split into two ranges.""" year_ranges = [] var_coverages = [] historical_range = time_domains["historical"] @@ -162,6 +196,7 @@ def build_year_and_coverage_lists_for_iteration( async def fetch_count_days_data( var_coverages, year_ranges, threshold, operator, lon, lat ): + """Fetch count of days above or below threshold for given variable coverages and year ranges.""" tasks = [] for coverage, year_range in zip(var_coverages, year_ranges): url = generate_wcs_query_url( @@ -184,28 +219,30 @@ async def fetch_count_days_data( def postprocess_count_days(data, start_year, end_year): - # if the year range spans historical and projected, our data will be a list of 5 lists: - # the first has day counts for each historical year, and the rest have day counts for each projected year for each SSP - # if the year range is only historical or only projected, our data will be a list of 1-4 lists accordingly - - # we want to create dictionary for output, with the following structure: - # { - # "historical": { - # "data": {"2000": 45, "2001": 50, ...}, - # "summary": {"min": 30, "max": 60, "mean": 45.5}, - # }, - # "projected": { - # "ssp126": { - # "data": {"2020": 55, "2021": 60, ...}, - # "summary": {"min": 40, "max": 70, "mean": 55.5}, - # }, - # "ssp245": { - # "data": {"2020": 50, "2021": 55, ...}, - # "summary": {"min": 35, "max": 65, "mean": 50.5}, - # }, - # ... - # } - # } + """Postprocess count days data into structured dictionary output. + If the year range spans historical and projected, our data will be a list of 5 lists: + The first has day counts for each historical year, and the rest have day counts for each projected year for each SSP. + If the year range is only historical or only projected, our data will be a list of 1-4 lists accordingly. + + We want to create dictionary for output, with the following structure: + { + "historical": { + "data": {"2000": 45, "2001": 50, ...}, + "summary": {"min": 30, "max": 60, "mean": 45.5}, + }, + "projected": { + "ssp126": { + "data": {"2020": 55, "2021": 60, ...}, + "summary": {"min": 40, "max": 70, "mean": 55.5}, + }, + "ssp245": { + "data": {"2020": 50, "2021": 55, ...}, + "summary": {"min": 35, "max": 65, "mean": 50.5}, + }, + ... + } + } + """ start_year = int(start_year) end_year = int(end_year) @@ -265,6 +302,7 @@ def postprocess_count_days(data, start_year, end_year): async def fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat): + """Fetch annual statistic data for given variable coverages and year ranges.""" tasks = [] for coverage, year_range in zip(var_coverages, year_ranges): url = generate_wcs_query_url( @@ -285,7 +323,8 @@ async def fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat): def postprocess_annual_stat(data, start_year, end_year, units): - # data in / out is similar to postprocess_count_days + """Postprocess annual statistic data into structured dictionary output.""" + # define conversion functions def mm_to_inches(mm): return mm / 25.4 @@ -362,7 +401,7 @@ def c_to_f(c): def postprocess_annual_rank(data, start_year, end_year, position, direction): - # data in / out is similar to postprocess_annual_stat + """Postprocess annual rank data into structured dictionary output.""" start_year = int(start_year) end_year = int(end_year) result = {} @@ -439,19 +478,21 @@ def postprocess_annual_rank(data, start_year, end_year, position, direction): "/dynamic_indicators/count_days/////////" ) def count_days(operator, threshold, units, variable, lat, lon, start_year, end_year): - # example usage: - # http://127.0.0.1:5000/dynamic_indicators/count_days/above/25/C/tasmax/64.5/-147.5/2000/2030/ ->>> can recreate the "summer days" indicator - # http://127.0.0.1:5000/dynamic_indicators/count_days/below/-30/C/tasmin/64.5/-147.5/2000/2030/ ->>> can recreate the "deep winter days" indicator - # http://127.0.0.1:5000/dynamic_indicators/count_days/above/10/mm/pr/64.5/-147.5/2000/2030/ ->>> can recreate the "days above 10mm precip" indicator - # http://127.0.0.1:5000/dynamic_indicators/count_days/above/1/mm/pr/64.5/-147.5/2000/2030/ ->>> can recreate the "wet days" indicator - + """Count the number of days above or below a threshold for a given variable and location over a specified year range. + + Example usage: + - http://127.0.0.1:5000/dynamic_indicators/count_days/above/25/C/tasmax/64.5/-147.5/2000/2030/ ->>> can recreate the "summer days" indicator + - http://127.0.0.1:5000/dynamic_indicators/count_days/below/-30/C/tasmin/64.5/-147.5/2000/2030/ ->>> can recreate the "deep winter days" indicator + - http://127.0.0.1:5000/dynamic_indicators/count_days/above/10/mm/pr/64.5/-147.5/2000/2030/ ->>> can recreate the "days above 10mm precip" indicator + - http://127.0.0.1:5000/dynamic_indicators/count_days/above/1/mm/pr/64.5/-147.5/2000/2030/ ->>> can recreate the "wet days" indicator + """ # Validate request params try: - lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon) operator = validate_operator(operator) units, threshold = validate_units_threshold_and_variable( units=units, threshold=threshold, variable=variable ) + lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon, variable) validate_year(start_year, end_year) except: return render_template("400/bad_request.html"), 400 @@ -479,20 +520,21 @@ def count_days(operator, threshold, units, variable, lat, lon, start_year, end_y "/dynamic_indicators/stat////////" ) def get_annual_stat(stat, variable, units, lat, lon, start_year, end_year): - # example usage: - # http://127.0.0.1:5000/dynamic_indicators/stat/max/pr/mm/64.5/-147.5/2000/2030 ->>> can recreate the "maxmimum one day precip" indicator - # http://127.0.0.1:5000/dynamic_indicators/stat/min/tasmin/C/64.5/-147.5/2000/2030 ->>> coldest day per year - # http://127.0.0.1:5000/dynamic_indicators/stat/max/tasmax/C/64.5/-147.5/2000/2030 ->>> hottest day per year - # http://127.0.0.1:5000/dynamic_indicators/stat/sum/pr/mm/64.5/-147.5/2000/2030/ ->>> total annual precipitation (NOTE: summary section of return will show mean annual precip over the year range) - # http://127.0.0.1:5000/dynamic_indicators/stat/mean/pr/mm/64.5/-147.5/2000/2030/ ->>> mean daily precipitation (NOTE: this is not a common mean statistic for precip - avg amount of precip per day over the year) - + """Get annual statistic (max, min, mean, sum) for a given variable and location over a specified year range. + Example usage: + - http://127.0.0.1:5000/dynamic_indicators/stat/max/pr/mm/64.5/-147.5/2000/2030 ->>> can recreate the "maxmimum one day precip" indicator + - http://127.0.0.1:5000/dynamic_indicators/stat/min/tasmin/C/64.5/-147.5/2000/2030 ->>> coldest day per year + - http://127.0.0.1:5000/dynamic_indicators/stat/max/tasmax/C/64.5/-147.5/2000/2030 ->>> hottest day per year + - http://127.0.0.1:5000/dynamic_indicators/stat/sum/pr/mm/64.5/-147.5/2000/2030/ ->>> total annual precipitation (NOTE: summary section of return will show mean annual precip over the year range) + - http://127.0.0.1:5000/dynamic_indicators/stat/mean/pr/mm/64.5/-147.5/2000/2030/ ->>> mean daily precipitation (NOTE: this is not a common mean statistic for precip - avg amount of precip per day over the year) + """ # Validate request params try: stat = validate_stat(stat) - lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon) units, _threshold = validate_units_threshold_and_variable( units=units, threshold=None, variable=variable ) + lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon, variable) validate_year(start_year, end_year) except: return render_template("400/bad_request.html"), 400 @@ -518,19 +560,20 @@ def get_annual_stat(stat, variable, units, lat, lon, start_year, end_year): "/dynamic_indicators/rank////////" ) def get_annual_rank(position, direction, variable, lat, lon, start_year, end_year): - # example usage: - # http://127.0.0.1:5000/dynamic_indicators/rank/6/highest/tasmax/64.5/-147.5/2000/2030/ ->>> can recreate "hot day threshold" indicator - # http://127.0.0.1:5000/dynamic_indicators/rank/6/lowest/tasmin/64.5/-147.5/2000/2030/ ->>> can recreate "cold day threshold" indicators - + """Get annual rank value (e.g., 6th highest, 10th lowest) for a given variable and location over a specified year range. + Example usage: + - http://127.0.0.1:5000/dynamic_indicators/rank/6/highest/tasmax/64.5/-147.5/2000/2030/ ->>> can recreate "hot day threshold" indicator + - http://127.0.0.1:5000/dynamic_indicators/rank/6/lowest/tasmin/64.5/-147.5/2000/2030/ ->>> can recreate "cold day threshold" indicators + """ # Validate request params + if variable not in ["tasmax", "tasmin", "pr"]: + return render_template("400/bad_request.html"), 400 try: position, direction = validate_rank_position_and_direction(position, direction) - lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon) + lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon, variable) validate_year(start_year, end_year) except: return render_template("400/bad_request.html"), 400 - if variable not in ["tasmax", "tasmin", "pr"]: - return render_template("400/bad_request.html"), 400 # build lists for iteration year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( From eeaf452d353b06a71011ad137da8da1c27a4bf47 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Wed, 5 Nov 2025 10:28:40 -0900 Subject: [PATCH 12/30] add point to route url --- routes/dynamic_indicators.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 6a6e6f29..1beedde6 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -475,16 +475,16 @@ def postprocess_annual_rank(data, start_year, end_year, position, direction): @routes.route( - "/dynamic_indicators/count_days/////////" + "/dynamic_indicators/count_days/////point/////" ) def count_days(operator, threshold, units, variable, lat, lon, start_year, end_year): """Count the number of days above or below a threshold for a given variable and location over a specified year range. Example usage: - - http://127.0.0.1:5000/dynamic_indicators/count_days/above/25/C/tasmax/64.5/-147.5/2000/2030/ ->>> can recreate the "summer days" indicator - - http://127.0.0.1:5000/dynamic_indicators/count_days/below/-30/C/tasmin/64.5/-147.5/2000/2030/ ->>> can recreate the "deep winter days" indicator - - http://127.0.0.1:5000/dynamic_indicators/count_days/above/10/mm/pr/64.5/-147.5/2000/2030/ ->>> can recreate the "days above 10mm precip" indicator - - http://127.0.0.1:5000/dynamic_indicators/count_days/above/1/mm/pr/64.5/-147.5/2000/2030/ ->>> can recreate the "wet days" indicator + - http://127.0.0.1:5000/dynamic_indicators/count_days/above/25/C/tasmax/point/64.5/-147.5/2000/2030/ ->>> can recreate the "summer days" indicator + - http://127.0.0.1:5000/dynamic_indicators/count_days/below/-30/C/tasmin/point/64.5/-147.5/2000/2030/ ->>> can recreate the "deep winter days" indicator + - http://127.0.0.1:5000/dynamic_indicators/count_days/above/10/mm/pr/point/64.5/-147.5/2000/2030/ ->>> can recreate the "days above 10mm precip" indicator + - http://127.0.0.1:5000/dynamic_indicators/count_days/above/1/mm/pr/point/64.5/-147.5/2000/2030/ ->>> can recreate the "wet days" indicator """ # Validate request params try: @@ -517,16 +517,16 @@ def count_days(operator, threshold, units, variable, lat, lon, start_year, end_y @routes.route( - "/dynamic_indicators/stat////////" + "/dynamic_indicators/stat////point/////" ) def get_annual_stat(stat, variable, units, lat, lon, start_year, end_year): """Get annual statistic (max, min, mean, sum) for a given variable and location over a specified year range. Example usage: - - http://127.0.0.1:5000/dynamic_indicators/stat/max/pr/mm/64.5/-147.5/2000/2030 ->>> can recreate the "maxmimum one day precip" indicator - - http://127.0.0.1:5000/dynamic_indicators/stat/min/tasmin/C/64.5/-147.5/2000/2030 ->>> coldest day per year - - http://127.0.0.1:5000/dynamic_indicators/stat/max/tasmax/C/64.5/-147.5/2000/2030 ->>> hottest day per year - - http://127.0.0.1:5000/dynamic_indicators/stat/sum/pr/mm/64.5/-147.5/2000/2030/ ->>> total annual precipitation (NOTE: summary section of return will show mean annual precip over the year range) - - http://127.0.0.1:5000/dynamic_indicators/stat/mean/pr/mm/64.5/-147.5/2000/2030/ ->>> mean daily precipitation (NOTE: this is not a common mean statistic for precip - avg amount of precip per day over the year) + - http://127.0.0.1:5000/dynamic_indicators/stat/max/pr/mm/point/64.5/-147.5/2000/2030 ->>> can recreate the "maxmimum one day precip" indicator + - http://127.0.0.1:5000/dynamic_indicators/stat/min/tasmin/C/point/64.5/-147.5/2000/2030 ->>> coldest day per year + - http://127.0.0.1:5000/dynamic_indicators/stat/max/tasmax/C/point/64.5/-147.5/2000/2030 ->>> hottest day per year + - http://127.0.0.1:5000/dynamic_indicators/stat/sum/pr/mm/point/64.5/-147.5/2000/2030/ ->>> total annual precipitation (NOTE: summary section of return will show mean annual precip over the year range) + - http://127.0.0.1:5000/dynamic_indicators/stat/mean/pr/mm/point/64.5/-147.5/2000/2030/ ->>> mean daily precipitation (NOTE: this is not a common mean statistic for precip - avg amount of precip per day over the year) """ # Validate request params try: @@ -557,13 +557,13 @@ def get_annual_stat(stat, variable, units, lat, lon, start_year, end_year): @routes.route( - "/dynamic_indicators/rank////////" + "/dynamic_indicators/rank////point/////" ) def get_annual_rank(position, direction, variable, lat, lon, start_year, end_year): """Get annual rank value (e.g., 6th highest, 10th lowest) for a given variable and location over a specified year range. Example usage: - - http://127.0.0.1:5000/dynamic_indicators/rank/6/highest/tasmax/64.5/-147.5/2000/2030/ ->>> can recreate "hot day threshold" indicator - - http://127.0.0.1:5000/dynamic_indicators/rank/6/lowest/tasmin/64.5/-147.5/2000/2030/ ->>> can recreate "cold day threshold" indicators + - http://127.0.0.1:5000/dynamic_indicators/rank/6/highest/tasmax/point/64.5/-147.5/2000/2030/ ->>> can recreate "hot day threshold" indicator + - http://127.0.0.1:5000/dynamic_indicators/rank/6/lowest/tasmin/point/64.5/-147.5/2000/2030/ ->>> can recreate "cold day threshold" indicators """ # Validate request params if variable not in ["tasmax", "tasmin", "pr"]: From 8e95345ff07d604a42f06e150bdaa1b89e0c5eb6 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Wed, 5 Nov 2025 11:02:08 -0900 Subject: [PATCH 13/30] set up area fetching functions --- routes/dynamic_indicators.py | 171 ++++++++++++++++++++++++++++++++++- 1 file changed, 168 insertions(+), 3 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 1beedde6..fc09ce8e 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -218,6 +218,17 @@ async def fetch_count_days_data( return data +async def fetch_count_days_area_data( + var_coverages, year_ranges, threshold, operator, place_id +): + """Fetch count of days above or below threshold for given variable coverages and year ranges over an area (zonal mean).""" + + # TODO: implement area fetching logic + data = None + + return data + + def postprocess_count_days(data, start_year, end_year): """Postprocess count days data into structured dictionary output. If the year range spans historical and projected, our data will be a list of 5 lists: @@ -322,6 +333,14 @@ async def fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat): return data +async def fetch_annual_stat_area_data(var_coverages, year_ranges, stat, place_id): + """Fetch annual statistic data for given variable coverages and year ranges over an area (zonal mean).""" + + # TODO: implement area fetching logic + data = None + return data + + def postprocess_annual_stat(data, start_year, end_year, units): """Postprocess annual statistic data into structured dictionary output.""" @@ -400,6 +419,14 @@ def c_to_f(c): return result +async def fetch_annual_rank_area_data(var_coverages, year_ranges, stat, place_id): + """Fetch annual statistic data for given variable coverages and year ranges over an area (zonal mean).""" + + # TODO: implement area fetching logic + data = None + return data + + def postprocess_annual_rank(data, start_year, end_year, position, direction): """Postprocess annual rank data into structured dictionary output.""" start_year = int(start_year) @@ -474,10 +501,15 @@ def postprocess_annual_rank(data, start_year, end_year, position, direction): return result +###### POINT QUERIES ###### + + @routes.route( "/dynamic_indicators/count_days/////point/////" ) -def count_days(operator, threshold, units, variable, lat, lon, start_year, end_year): +def count_days_point( + operator, threshold, units, variable, lat, lon, start_year, end_year +): """Count the number of days above or below a threshold for a given variable and location over a specified year range. Example usage: @@ -519,7 +551,7 @@ def count_days(operator, threshold, units, variable, lat, lon, start_year, end_y @routes.route( "/dynamic_indicators/stat////point/////" ) -def get_annual_stat(stat, variable, units, lat, lon, start_year, end_year): +def get_annual_stat_point(stat, variable, units, lat, lon, start_year, end_year): """Get annual statistic (max, min, mean, sum) for a given variable and location over a specified year range. Example usage: - http://127.0.0.1:5000/dynamic_indicators/stat/max/pr/mm/point/64.5/-147.5/2000/2030 ->>> can recreate the "maxmimum one day precip" indicator @@ -559,7 +591,9 @@ def get_annual_stat(stat, variable, units, lat, lon, start_year, end_year): @routes.route( "/dynamic_indicators/rank////point/////" ) -def get_annual_rank(position, direction, variable, lat, lon, start_year, end_year): +def get_annual_rank_point( + position, direction, variable, lat, lon, start_year, end_year +): """Get annual rank value (e.g., 6th highest, 10th lowest) for a given variable and location over a specified year range. Example usage: - http://127.0.0.1:5000/dynamic_indicators/rank/6/highest/tasmax/point/64.5/-147.5/2000/2030/ ->>> can recreate "hot day threshold" indicator @@ -593,3 +627,134 @@ def get_annual_rank(position, direction, variable, lat, lon, start_year, end_yea if hasattr(exc, "status") and exc.status == 404: return render_template("404/no_data.html"), 404 return render_template("500/server_error.html"), 500 + + +###### AREA QUERIES ###### + + +@routes.route( + "/dynamic_indicators/count_days/////area////" +) +def count_days_area( + operator, threshold, units, variable, place_id, start_year, end_year +): + """Count the number of days above or below a threshold for a given variable and specified year range, and compute zonal mean over area. + + Example usage: + - http://127.0.0.1:5000/dynamic_indicators/count_days/above/25/C/tasmax/area/1908030609/2000/2030/ ->>> can recreate the "summer days" indicator + - http://127.0.0.1:5000/dynamic_indicators/count_days/below/-30/C/tasmin/area/1908030609/2000/2030/ ->>> can recreate the "deep winter days" indicator + - http://127.0.0.1:5000/dynamic_indicators/count_days/above/10/mm/pr/area/1908030609/2000/2030/ ->>> can recreate the "days above 10mm precip" indicator + - http://127.0.0.1:5000/dynamic_indicators/count_days/above/1/mm/pr/area/1908030609/2000/2030/ ->>> can recreate the "wet days" indicator + """ + # Validate request params + try: + operator = validate_operator(operator) + units, threshold = validate_units_threshold_and_variable( + units=units, threshold=threshold, variable=variable + ) + validate_year(start_year, end_year) + + # TODO: validate the place_id + + except: + return render_template("400/bad_request.html"), 400 + + # build lists for iteration + year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( + int(start_year), int(end_year), variable, time_domains, all_coverages + ) + + try: + data = asyncio.run( + fetch_count_days_area_data( + var_coverages, year_ranges, threshold, operator, place_id + ) + ) + result = postprocess_count_days(data, start_year, end_year) + return result + except Exception as exc: + if hasattr(exc, "status") and exc.status == 404: + return render_template("404/no_data.html"), 404 + return render_template("500/server_error.html"), 500 + + +@routes.route( + "/dynamic_indicators/stat////area////" +) +def get_annual_stat_area(stat, variable, units, place_id, start_year, end_year): + """Get annual statistic (max, min, mean, sum) for a given variable over a specified year range, and compute zonal mean over area. + Example usage: + - http://127.0.0.1:5000/dynamic_indicators/stat/max/pr/mm/area/1908030609/2000/2030 ->>> can recreate the "maxmimum one day precip" indicator + - http://127.0.0.1:5000/dynamic_indicators/stat/min/tasmin/C/area/1908030609/2000/2030 ->>> coldest day per year + - http://127.0.0.1:5000/dynamic_indicators/stat/max/tasmax/C/area/1908030609/2000/2030 ->>> hottest day per year + - http://127.0.0.1:5000/dynamic_indicators/stat/sum/pr/mm/area/1908030609/2000/2030/ ->>> total annual precipitation (NOTE: summary section of return will show mean annual precip over the year range) + - http://127.0.0.1:5000/dynamic_indicators/stat/mean/pr/mm/area/1908030609/2000/2030/ ->>> mean daily precipitation (NOTE: this is not a common mean statistic for precip - avg amount of precip per day over the year) + """ + # Validate request params + try: + stat = validate_stat(stat) + units, _threshold = validate_units_threshold_and_variable( + units=units, threshold=None, variable=variable + ) + validate_year(start_year, end_year) + + # TODO: validate the place_id + + except: + return render_template("400/bad_request.html"), 400 + + # build lists for iteration + year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( + int(start_year), int(end_year), variable, time_domains, all_coverages + ) + + try: + data = asyncio.run( + fetch_annual_stat_area_data(var_coverages, year_ranges, stat, place_id) + ) + result = postprocess_annual_stat(data, start_year, end_year, units) + return result + except Exception as exc: + if hasattr(exc, "status") and exc.status == 404: + return render_template("404/no_data.html"), 404 + return render_template("500/server_error.html"), 500 + + +@routes.route( + "/dynamic_indicators/rank////area////" +) +def get_annual_rank_area(position, direction, variable, place_id, start_year, end_year): + """Get annual rank value (e.g., 6th highest, 10th lowest) for a given variable over a specified year range, and compute zonal mean over area. + Example usage: + - http://127.0.0.1:5000/dynamic_indicators/rank/6/highest/tasmax/area/1908030609/2000/2030/ ->>> can recreate "hot day threshold" indicator + - http://127.0.0.1:5000/dynamic_indicators/rank/6/lowest/tasmin/area/1908030609/2000/2030/ ->>> can recreate "cold day threshold" indicators + """ + # Validate request params + if variable not in ["tasmax", "tasmin", "pr"]: + return render_template("400/bad_request.html"), 400 + try: + position, direction = validate_rank_position_and_direction(position, direction) + validate_year(start_year, end_year) + + # TODO: validate the place_id + + except: + return render_template("400/bad_request.html"), 400 + + # build lists for iteration + year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( + int(start_year), int(end_year), variable, time_domains, all_coverages + ) + + try: + data = asyncio.run( + fetch_annual_rank_area_data(var_coverages, year_ranges, place_id) + ) + result = postprocess_annual_rank( + data, start_year, end_year, position, direction + ) + return result + except Exception as exc: + if hasattr(exc, "status") and exc.status == 404: + return render_template("404/no_data.html"), 404 + return render_template("500/server_error.html"), 500 From 24f4d7f1b8fd05410164dfb705e230000d26556b Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Wed, 5 Nov 2025 11:47:14 -0900 Subject: [PATCH 14/30] bring in zonal stats functions --- routes/dynamic_indicators.py | 128 +++++++++++++++++++++++++++++------ 1 file changed, 106 insertions(+), 22 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index fc09ce8e..3e43383e 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -3,11 +3,12 @@ from flask import Blueprint, render_template, request # local imports -from fetch_data import fetch_data, describe_via_wcps +from fetch_data import fetch_data, describe_via_wcps, get_poly, fetch_bbox_netcdf from generate_urls import generate_wcs_query_url from generate_requests import ( construct_count_annual_days_above_or_below_threshold_wcps_query_string, construct_get_annual_mmm_stat_wcps_query_string, + generate_netcdf_wcs_getcov_str, ) from validate_request import ( latlon_is_numeric_and_in_geodetic_range, @@ -15,6 +16,13 @@ project_latlon, validate_latlon_in_bboxes, construct_latlon_bbox_from_coverage_bounds, + validate_var_id, +) +from zonal_stats import ( + get_scale_factor, + rasterize_polygon, + interpolate, + calculate_zonal_means_vectorized, ) # TODO: for additional postprocessing or csv output, uncomment these imports and add code @@ -165,6 +173,17 @@ def validate_rank_position_and_direction(position, direction): return position, direction +def validate_place_id(place_id): + poly_type = validate_var_id(place_id) + if type(poly_type) is tuple: + return poly_type + try: + polygon = get_poly(place_id) + except: + return render_template("422/invalid_area.html"), 422 + return polygon + + def build_year_and_coverage_lists_for_iteration( start_year, end_year, variable, time_domains, all_coverages ): @@ -193,6 +212,34 @@ def build_year_and_coverage_lists_for_iteration( return year_ranges, var_coverages +def calculate_indicators_zonal_stats(polygon, dataset, variable): + """Calculate zonal mean statistics for the given polygon and dataset.""" + # get scale factor once, not per variable or time slice! + ds = dataset + spatial_resolution = ds.rio.resolution() + grid_cell_area_m2 = abs(spatial_resolution[0]) * abs(spatial_resolution[1]) + polygon_area_m2 = polygon.area + scale_factor = get_scale_factor(grid_cell_area_m2, polygon_area_m2) + + # create an initial array for the basis of polygon rasterization + # why? polygon rasterization bogs down hard when doing it in the loop + da_i = interpolate( + ds.isel(ansi=0), variable, "X", "Y", scale_factor, method="nearest" + ) + + rasterized_polygon_array = rasterize_polygon(da_i, "X", "Y", polygon) + + da_i_3d = interpolate(ds, variable, "X", "Y", scale_factor, method="nearest") + # calculate zonal stats for the entire time series + # rename the ansi dimensions to "time" so this function will work + da_i_3d = da_i_3d.rename({"ansi": "time"}) + time_series_means = calculate_zonal_means_vectorized( + da_i_3d, rasterized_polygon_array, "X", "Y" + ) + + return time_series_means + + async def fetch_count_days_data( var_coverages, year_ranges, threshold, operator, lon, lat ): @@ -218,15 +265,46 @@ async def fetch_count_days_data( return data +async def fetch_area_data(var_coverages, year_ranges, polygon): + """ + Make an async request for CMIP6 downscaled daily data for provided coverage within a specified polygon. + Args: + var_coverages (list): list of coverage IDs + year_ranges (tuple): (start_year, end_year) + polygon (shapely.geometry.Polygon): polygon geometry + Returns: + list of data results within the specified polygon + """ + urls = [] + for cov_id, year_range in zip(var_coverages, year_ranges): + # Generate WCS GetCoverage request string for the polygon bounds + wcs_str = generate_netcdf_wcs_getcov_str(polygon.total_bounds, cov_id=cov_id) + # add time range to WCS string + wcs_str += f"&SUBSET=ansi({year_range[0]}-01-01T00:00:00Z,{year_range[1]}-12-31T23:59:59Z)" + # Generate the URL for the WCS query + url = generate_wcs_query_url(wcs_str) + urls.append(url) + # Fetch the data and add to a list of area datasets (one dataset per coverage) + area_dataset_list = await fetch_bbox_netcdf([urls]) + + return area_dataset_list + + async def fetch_count_days_area_data( - var_coverages, year_ranges, threshold, operator, place_id + variable, var_coverages, year_ranges, threshold, operator, polygon ): - """Fetch count of days above or below threshold for given variable coverages and year ranges over an area (zonal mean).""" + """Fetch daily data covering a polygon area, and count days above or below threshold for given variable coverages and year range. + Compute zonal mean of the counts over the area.""" - # TODO: implement area fetching logic - data = None + area_dataset_list = await fetch_area_data(var_coverages, year_ranges, polygon) - return data + area_daily_means_list = [] + for dataset in area_dataset_list: + area_daily_means_list.append( + calculate_indicators_zonal_stats(polygon, dataset, variable) + ) + + return area_daily_means_list def postprocess_count_days(data, start_year, end_year): @@ -333,8 +411,11 @@ async def fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat): return data -async def fetch_annual_stat_area_data(var_coverages, year_ranges, stat, place_id): - """Fetch annual statistic data for given variable coverages and year ranges over an area (zonal mean).""" +async def fetch_annual_stat_area_data( + variable, var_coverages, year_ranges, stat, polygon +): + """Fetch daily data covering a polygon area, and calculate stats for given variable coverages and year range. + Compute zonal mean of the stats over the area.""" # TODO: implement area fetching logic data = None @@ -419,8 +500,12 @@ def c_to_f(c): return result -async def fetch_annual_rank_area_data(var_coverages, year_ranges, stat, place_id): - """Fetch annual statistic data for given variable coverages and year ranges over an area (zonal mean).""" +async def fetch_annual_rank_area_data( + variable, var_coverages, year_ranges, stat, polygon +): + """Fetch daily data covering a polygon area, and rank values for given variable coverages and year range. + # TODO: inspect zonal stats logic here! >>> Compute zonal mean of the ranks over the area. + """ # TODO: implement area fetching logic data = None @@ -653,9 +738,7 @@ def count_days_area( units=units, threshold=threshold, variable=variable ) validate_year(start_year, end_year) - - # TODO: validate the place_id - + polygon = validate_place_id(place_id) except: return render_template("400/bad_request.html"), 400 @@ -667,9 +750,12 @@ def count_days_area( try: data = asyncio.run( fetch_count_days_area_data( - var_coverages, year_ranges, threshold, operator, place_id + variable, var_coverages, year_ranges, threshold, operator, polygon ) ) + + print(data) + result = postprocess_count_days(data, start_year, end_year) return result except Exception as exc: @@ -697,9 +783,7 @@ def get_annual_stat_area(stat, variable, units, place_id, start_year, end_year): units=units, threshold=None, variable=variable ) validate_year(start_year, end_year) - - # TODO: validate the place_id - + polygon = validate_place_id(place_id) except: return render_template("400/bad_request.html"), 400 @@ -710,7 +794,9 @@ def get_annual_stat_area(stat, variable, units, place_id, start_year, end_year): try: data = asyncio.run( - fetch_annual_stat_area_data(var_coverages, year_ranges, stat, place_id) + fetch_annual_stat_area_data( + variable, var_coverages, year_ranges, stat, polygon + ) ) result = postprocess_annual_stat(data, start_year, end_year, units) return result @@ -735,9 +821,7 @@ def get_annual_rank_area(position, direction, variable, place_id, start_year, en try: position, direction = validate_rank_position_and_direction(position, direction) validate_year(start_year, end_year) - - # TODO: validate the place_id - + polygon = validate_place_id(place_id) except: return render_template("400/bad_request.html"), 400 @@ -748,7 +832,7 @@ def get_annual_rank_area(position, direction, variable, place_id, start_year, en try: data = asyncio.run( - fetch_annual_rank_area_data(var_coverages, year_ranges, place_id) + fetch_annual_rank_area_data(variable, var_coverages, year_ranges, polygon) ) result = postprocess_annual_rank( data, start_year, end_year, position, direction From 47a87cd1a0821f289ba6f728c95c97c84bb383d1 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Wed, 5 Nov 2025 13:48:56 -0900 Subject: [PATCH 15/30] finish area query for count days --- routes/dynamic_indicators.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 3e43383e..097a7663 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -3,7 +3,7 @@ from flask import Blueprint, render_template, request # local imports -from fetch_data import fetch_data, describe_via_wcps, get_poly, fetch_bbox_netcdf +from fetch_data import fetch_data, describe_via_wcps, get_poly, fetch_bbox_netcdf_list from generate_urls import generate_wcs_query_url from generate_requests import ( construct_count_annual_days_above_or_below_threshold_wcps_query_string, @@ -278,14 +278,14 @@ async def fetch_area_data(var_coverages, year_ranges, polygon): urls = [] for cov_id, year_range in zip(var_coverages, year_ranges): # Generate WCS GetCoverage request string for the polygon bounds - wcs_str = generate_netcdf_wcs_getcov_str(polygon.total_bounds, cov_id=cov_id) + wcs_str = generate_netcdf_wcs_getcov_str(polygon.total_bounds, cov_id) # add time range to WCS string - wcs_str += f"&SUBSET=ansi({year_range[0]}-01-01T00:00:00Z,{year_range[1]}-12-31T23:59:59Z)" + wcs_str += f'&SUBSET=ansi("{year_range[0]}-01-01","{year_range[1]}-12-31")' # Generate the URL for the WCS query url = generate_wcs_query_url(wcs_str) urls.append(url) # Fetch the data and add to a list of area datasets (one dataset per coverage) - area_dataset_list = await fetch_bbox_netcdf([urls]) + area_dataset_list = await fetch_bbox_netcdf_list(urls) return area_dataset_list @@ -304,7 +304,27 @@ async def fetch_count_days_area_data( calculate_indicators_zonal_stats(polygon, dataset, variable) ) - return area_daily_means_list + # area_daily_means is a list, where each item corresponds to a coverage/year range combo + # and each item is a list of daily means for each day in that range + # to count days above or below threshold for each year, we need to iterate through each item in area_daily_means_list + # and divide each list into chunks of 365 (no leap years) to count days above/below threshold per year + # we will output a list of lists, where each sublist contains the counts for each year in the corresponding year range + + data = [] + + for i, daily_means in enumerate(area_daily_means_list): + num_years_in_range = year_ranges[i][1] - year_ranges[i][0] + 1 + yearly_counts = [] + for year_idx in range(num_years_in_range): + year_daily_means = daily_means[year_idx * 365 : (year_idx + 1) * 365] + if operator == ">": + count = sum(1 for day_mean in year_daily_means if day_mean > threshold) + else: + count = sum(1 for day_mean in year_daily_means if day_mean < threshold) + yearly_counts.append(count) + data.append(yearly_counts) + + return data def postprocess_count_days(data, start_year, end_year): From df9c7eb21c09c19bb34b18c78b0570c1e847f0f0 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Thu, 6 Nov 2025 11:35:31 -0900 Subject: [PATCH 16/30] finish area queries for stat and rank endpoints --- routes/dynamic_indicators.py | 128 +++++++++++++++++++++-------------- 1 file changed, 77 insertions(+), 51 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 097a7663..3dcf536f 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -265,7 +265,9 @@ async def fetch_count_days_data( return data -async def fetch_area_data(var_coverages, year_ranges, polygon): +async def fetch_area_data_and_calculate_zonal_stats( + variable, var_coverages, year_ranges, polygon +): """ Make an async request for CMIP6 downscaled daily data for provided coverage within a specified polygon. Args: @@ -287,22 +289,25 @@ async def fetch_area_data(var_coverages, year_ranges, polygon): # Fetch the data and add to a list of area datasets (one dataset per coverage) area_dataset_list = await fetch_bbox_netcdf_list(urls) - return area_dataset_list + # Calculate zonal stats for each dataset + area_daily_means_list = [] + for dataset in area_dataset_list: + area_daily_means_list.append( + calculate_indicators_zonal_stats(polygon, dataset, variable) + ) + + return area_daily_means_list async def fetch_count_days_area_data( variable, var_coverages, year_ranges, threshold, operator, polygon ): - """Fetch daily data covering a polygon area, and count days above or below threshold for given variable coverages and year range. - Compute zonal mean of the counts over the area.""" - - area_dataset_list = await fetch_area_data(var_coverages, year_ranges, polygon) + """Fetch daily data covering a polygon area, and calculatee the zonal mean of the area. + Count days above or below threshold for given variable coverages and year range.""" - area_daily_means_list = [] - for dataset in area_dataset_list: - area_daily_means_list.append( - calculate_indicators_zonal_stats(polygon, dataset, variable) - ) + area_daily_means_list = await fetch_area_data_and_calculate_zonal_stats( + variable, var_coverages, year_ranges, polygon + ) # area_daily_means is a list, where each item corresponds to a coverage/year range combo # and each item is a list of daily means for each day in that range @@ -374,9 +379,9 @@ def postprocess_count_days(data, start_year, end_year): result["historical"] = { "data": hist_day_counts, "summary": { - "min": min(hist_day_counts.values()), - "max": max(hist_day_counts.values()), - "mean": sum(hist_day_counts.values()) / len(hist_day_counts), + "min": round(min(hist_day_counts.values()), 2), + "max": round(max(hist_day_counts.values()), 2), + "mean": round(sum(hist_day_counts.values()) / len(hist_day_counts), 2), }, } current_index += 1 @@ -401,9 +406,11 @@ def postprocess_count_days(data, start_year, end_year): result["projected"][ssp] = { "data": proj_day_counts, "summary": { - "min": min(proj_day_counts.values()), - "max": max(proj_day_counts.values()), - "mean": sum(proj_day_counts.values()) / len(proj_day_counts), + "min": round(min(proj_day_counts.values()), 2), + "max": round(max(proj_day_counts.values()), 2), + "mean": round( + sum(proj_day_counts.values()) / len(proj_day_counts), 2 + ), }, } current_index += 1 @@ -434,11 +441,41 @@ async def fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat): async def fetch_annual_stat_area_data( variable, var_coverages, year_ranges, stat, polygon ): - """Fetch daily data covering a polygon area, and calculate stats for given variable coverages and year range. - Compute zonal mean of the stats over the area.""" + """Fetch daily data covering a polygon area, and calculatee the zonal mean of the area. + Calculate requested stat for given variable coverages and year range.""" + + area_daily_means_list = await fetch_area_data_and_calculate_zonal_stats( + variable, var_coverages, year_ranges, polygon + ) + + # area_daily_means is a list, where each item corresponds to a coverage/year range combo + # and each item is a list of daily means for each day in that range + # to get the stat for each year, we need to iterate through each item in area_daily_means_list + # and divide each list into chunks of 365 (no leap years) and apply the stat function to the yearly values + # we will output a list of lists, where each sublist contains the stat for each year in the corresponding year range + + # if no stat is provided, return all daily means (split by year) for ranking purposes + + data = [] + + for i, daily_means in enumerate(area_daily_means_list): + num_years_in_range = year_ranges[i][1] - year_ranges[i][0] + 1 + yearly_stats = [] + for year_idx in range(num_years_in_range): + year_daily_means = daily_means[year_idx * 365 : (year_idx + 1) * 365] + if stat == "max": + stat_value = round(max(year_daily_means), 2) + elif stat == "min": + stat_value = round(min(year_daily_means), 2) + elif stat == "sum": + stat_value = round(sum(year_daily_means), 2) + elif stat == "avg": + stat_value = round(sum(year_daily_means) / len(year_daily_means), 2) + else: + stat_value = [round(value, 2) for value in year_daily_means] + yearly_stats.append(stat_value) + data.append(yearly_stats) - # TODO: implement area fetching logic - data = None return data @@ -447,17 +484,17 @@ def postprocess_annual_stat(data, start_year, end_year, units): # define conversion functions def mm_to_inches(mm): - return mm / 25.4 + return round((mm / 25.4), 2) def c_to_f(c): - return (c * 9 / 5) + 32 + return round(((c * 9 / 5) + 32), 2) if units == "in": convert = mm_to_inches elif units == "F": convert = c_to_f - else: - convert = lambda x: x + else: # round values regardless of units + convert = lambda x: round(x, 2) start_year = int(start_year) end_year = int(end_year) @@ -482,9 +519,9 @@ def c_to_f(c): result["historical"] = { "data": hist_stats, "summary": { - "min": min(hist_stats.values()), - "max": max(hist_stats.values()), - "mean": sum(hist_stats.values()) / len(hist_stats), + "min": round(min(hist_stats.values()), 2), + "max": round(max(hist_stats.values()), 2), + "mean": round(sum(hist_stats.values()) / len(hist_stats), 2), }, } current_index += 1 @@ -510,9 +547,9 @@ def c_to_f(c): result["projected"][ssp] = { "data": proj_stats, "summary": { - "min": min(proj_stats.values()), - "max": max(proj_stats.values()), - "mean": sum(proj_stats.values()) / len(proj_stats), + "min": round(min(proj_stats.values()), 2), + "max": round(max(proj_stats.values()), 2), + "mean": round(sum(proj_stats.values()) / len(proj_stats), 2), }, } current_index += 1 @@ -520,18 +557,6 @@ def c_to_f(c): return result -async def fetch_annual_rank_area_data( - variable, var_coverages, year_ranges, stat, polygon -): - """Fetch daily data covering a polygon area, and rank values for given variable coverages and year range. - # TODO: inspect zonal stats logic here! >>> Compute zonal mean of the ranks over the area. - """ - - # TODO: implement area fetching logic - data = None - return data - - def postprocess_annual_rank(data, start_year, end_year, position, direction): """Postprocess annual rank data into structured dictionary output.""" start_year = int(start_year) @@ -562,9 +587,9 @@ def postprocess_annual_rank(data, start_year, end_year, position, direction): result["historical"] = { "data": hist_ranks, "summary": { - "min": min(hist_ranks.values()), - "max": max(hist_ranks.values()), - "mean": sum(hist_ranks.values()) / len(hist_ranks), + "min": round(min(hist_ranks.values()), 2), + "max": round(max(hist_ranks.values()), 2), + "mean": round(sum(hist_ranks.values()) / len(hist_ranks), 2), }, } @@ -596,9 +621,9 @@ def postprocess_annual_rank(data, start_year, end_year, position, direction): result["projected"][ssp] = { "data": proj_ranks, "summary": { - "min": min(proj_ranks.values()), - "max": max(proj_ranks.values()), - "mean": sum(proj_ranks.values()) / len(proj_ranks), + "min": round(min(proj_ranks.values()), 2), + "max": round(max(proj_ranks.values()), 2), + "mean": round(sum(proj_ranks.values()) / len(proj_ranks), 2), }, } current_index += 1 @@ -849,10 +874,11 @@ def get_annual_rank_area(position, direction, variable, place_id, start_year, en year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( int(start_year), int(end_year), variable, time_domains, all_coverages ) - try: data = asyncio.run( - fetch_annual_rank_area_data(variable, var_coverages, year_ranges, polygon) + fetch_annual_stat_area_data( + variable, var_coverages, year_ranges, "", polygon + ) ) result = postprocess_annual_rank( data, start_year, end_year, position, direction From 2435b8fb2fbc3b5b3d142477d69917542e5ae435 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Thu, 6 Nov 2025 11:36:17 -0900 Subject: [PATCH 17/30] add stat note --- routes/dynamic_indicators.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 3dcf536f..d56aa4d0 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -874,10 +874,13 @@ def get_annual_rank_area(position, direction, variable, place_id, start_year, en year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( int(start_year), int(end_year), variable, time_domains, all_coverages ) + + # stat is empty to force return of all values for ranking + stat = "" try: data = asyncio.run( fetch_annual_stat_area_data( - variable, var_coverages, year_ranges, "", polygon + variable, var_coverages, year_ranges, stat, polygon ) ) result = postprocess_annual_rank( From 4552f90734c16b8e91c38f23101e1a3e824e7800 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Thu, 6 Nov 2025 12:14:56 -0900 Subject: [PATCH 18/30] start documentation --- routes/dynamic_indicators.py | 5 ++ .../documentation/dynamic_indicators.html | 61 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index d56aa4d0..056f91b2 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -631,6 +631,11 @@ def postprocess_annual_rank(data, start_year, end_year, position, direction): return result +@routes.route("/dynamic_indicators/") +def dyanmic_indicators_about(): + return render_template("documentation/dynamic_indicators.html") + + ###### POINT QUERIES ###### diff --git a/templates/documentation/dynamic_indicators.html b/templates/documentation/dynamic_indicators.html index 7a9a070f..137639d9 100644 --- a/templates/documentation/dynamic_indicators.html +++ b/templates/documentation/dynamic_indicators.html @@ -1,4 +1,65 @@ {% extends 'base.html' %} {% block content %}

Dynamically Generated Indicators from Downscaled CMIP6

+

+ This endpoint provides access to dynamically generated indicators derived from + a 6-model average of CMIP6 daily data that have been bias-adjusted and downscaled + using dynamically downscaled ERA5 reference data. The source dataset spans + 1965–2100 at 4km spatial resolution. The spatial domain covers most of Alaska + and parts of western Canada. Variables include daily maximum and minimum temperature + (tasmax and tasmin), as well as daily total precipitation + (pr). +

+ +

Available Operations

+ + + + + + + + + + + + + + + + + + + + + + + + + +
OperationrouteDescription
Count annual days above / below threshold + count_days/above/<value>/<units>/..., + count_days/below/<value>/<units>/... + + Counts number of days per year where values are above or below the provided <value> + given in the provided <units>. For area queries, daily values are weighted by spatial + area before applying the threshold. +
Calculate annual statistic + stat/max/<value>/<units>/..., + stat/min/<value>/<units>/..., + stat/mean/<value>/<units>/..., + stat/sum/<value>/<units>/... + + Calculates the specified annual statistic (max, min, mean, or sum) + for the provided <value> in the provided <units>. For area queries, daily values + are weighted by spatial area before calculating the statistic. +
Rank annual values and get n-position + rank/<position>/<direction>/..., + rank/<position>/<direction>/... + + Ranks values in each year and returns the value at the specified <position> + and <direction> (e.g., 5th highest, 2nd lowest). For area queries, daily values + are weighted by spatial area before ranking. +
+ {% endblock %} \ No newline at end of file From 54575a945db2ede09a5938e7a66b7667f8f634c8 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Thu, 6 Nov 2025 12:34:36 -0900 Subject: [PATCH 19/30] more doc --- .../documentation/dynamic_indicators.html | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/templates/documentation/dynamic_indicators.html b/templates/documentation/dynamic_indicators.html index 137639d9..e8e5602d 100644 --- a/templates/documentation/dynamic_indicators.html +++ b/templates/documentation/dynamic_indicators.html @@ -62,4 +62,100 @@

Available Operations

+ +

Service Endpoints

+ +

Point Query

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
EndpointExample URL
+ Count annual days above 25C in select years + + /dynamic_indicators/count_days/above/25/C/tasmax/point/64.5/-147.5/2000/2030/ +
+ Count annual days below -40F in select years + + /dynamic_indicators/count_days/below/-40/F/tasmin/point/64.5/-147.5/2000/2030/ +
+ Count annual days above 10mm precipitation in select years + + /dynamic_indicators/count_days/above/10/mm/pr/point/64.5/-147.5/2000/2030/ +
+ Get maximum 1-day precipitation in select years + + /dynamic_indicators/stat/max/pr/mm/point/64.5/-147.5/2000/2030/ +
+ Get maximum 1-day temperature in select years + + /dynamic_indicators/stat/max/tasmax/F/point/64.5/-147.5/2000/2030/ +
+ Get 6th coldest day in select years + + /dynamic_indicators/rank/6/lowest/tasmin/64.5/-147.5/2000/2030/ +
+ Ranking queries will return values in metric units (C + for tasmin and tasmax, mm for pr). +
+ +
+ + + {% endblock %} \ No newline at end of file From 621dfdeebe00557eb627f583a04c8ce398fd9c63 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Thu, 6 Nov 2025 12:37:16 -0900 Subject: [PATCH 20/30] fix rank rounding --- routes/dynamic_indicators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 056f91b2..efc2c276 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -583,7 +583,7 @@ def postprocess_annual_rank(data, start_year, end_year, position, direction): rank_value = sorted_values[-position] else: rank_value = sorted_values[position - 1] - hist_ranks[str(year)] = rank_value + hist_ranks[str(year)] = round(rank_value, 2) result["historical"] = { "data": hist_ranks, "summary": { @@ -617,7 +617,7 @@ def postprocess_annual_rank(data, start_year, end_year, position, direction): rank_value = sorted_values[-position] else: rank_value = sorted_values[position - 1] - proj_ranks[str(year)] = rank_value + proj_ranks[str(year)] = round(rank_value, 2) result["projected"][ssp] = { "data": proj_ranks, "summary": { From 20becbf1c3efd3dc989f686e0a8a15ff7e7a3786 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Fri, 7 Nov 2025 09:35:06 -0900 Subject: [PATCH 21/30] remove "projected" key in results and replace with scenario; add example returns to documentation --- application.py | 1 + routes/dynamic_indicators.py | 9 +- .../documentation/dynamic_indicators.html | 184 ++++++++++++++++++ 3 files changed, 188 insertions(+), 6 deletions(-) diff --git a/application.py b/application.py index b4eb3624..48c59eb0 100644 --- a/application.py +++ b/application.py @@ -32,6 +32,7 @@ def get_service_categories(): return [ ("CMIP6", "/cmip6"), ("Climate Indicators", "/indicators"), + ("Climate Indicators, Dynamic", "/dynamic_indicators"), ("Climate Protection from Spruce Beetles", "/beetles"), ("Degree Days", "/degree_days"), ("Digital Elevation Models (DEMs)", "/elevation"), diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index efc2c276..4e7faefb 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -390,7 +390,6 @@ def postprocess_count_days(data, start_year, end_year): and end_year > time_domains["projected"][0] ): # projected data present - result["projected"] = {} ssp_names = ["ssp126", "ssp245", "ssp370", "ssp585"] proj_years = list( range( @@ -403,7 +402,7 @@ def postprocess_count_days(data, start_year, end_year): proj_day_counts = { str(year): proj_data[i] for i, year in enumerate(proj_years) } - result["projected"][ssp] = { + result[ssp] = { "data": proj_day_counts, "summary": { "min": round(min(proj_day_counts.values()), 2), @@ -531,7 +530,6 @@ def c_to_f(c): and end_year > time_domains["projected"][0] ): # projected data present - result["projected"] = {} ssp_names = ["ssp126", "ssp245", "ssp370", "ssp585"] proj_years = list( range( @@ -544,7 +542,7 @@ def c_to_f(c): proj_stats = { str(year): convert(proj_data[i]) for i, year in enumerate(proj_years) } - result["projected"][ssp] = { + result[ssp] = { "data": proj_stats, "summary": { "min": round(min(proj_stats.values()), 2), @@ -600,7 +598,6 @@ def postprocess_annual_rank(data, start_year, end_year, position, direction): and end_year > time_domains["projected"][0] ): # projected data present - result["projected"] = {} ssp_names = ["ssp126", "ssp245", "ssp370", "ssp585"] proj_years = list( range( @@ -618,7 +615,7 @@ def postprocess_annual_rank(data, start_year, end_year, position, direction): else: rank_value = sorted_values[position - 1] proj_ranks[str(year)] = round(rank_value, 2) - result["projected"][ssp] = { + result[ssp] = { "data": proj_ranks, "summary": { "min": round(min(proj_ranks.values()), 2), diff --git a/templates/documentation/dynamic_indicators.html b/templates/documentation/dynamic_indicators.html index e8e5602d..22e4992c 100644 --- a/templates/documentation/dynamic_indicators.html +++ b/templates/documentation/dynamic_indicators.html @@ -156,6 +156,190 @@

Point Query

+

Point Query Output

+ +
Count annual days above / below threshold
+ +

+ Data are returned as JSON keyed hierarchically by scenario. Results + are the annual indicator values, and a summary of the maximum, mean, + and minimum of those values. +

+

+{
+    "historical": {
+        "data": {
+            "2000": 9,
+            "2001": 18,
+            "2002": 18,
+            "2003": 13,
+            "2004": 13,
+            "2005": 8,
+            "2006": 14,
+            "2007": 17,
+            "2008": 17,
+            "2009": 9,
+            "2010": 13,
+            "2011": 16,
+            "2012": 14,
+            "2013": 17,
+            "2014": 7
+            },
+        "summary": {
+            "max": 18,
+            "mean": 13.53,
+            "min": 7
+            }
+        },
+    "ssp126": {
+        "data": {
+            "2016": 17,
+            "2017": 19,
+            "2018": 9,
+            "2019": 15,
+            "2020": 24,
+            "2021": 9,
+            "2022": 9,
+            "2023": 17,
+            "2024": 14,
+            "2025": 13,
+            "2026": 23,
+            "2027": 12,
+            "2028": 12,
+            "2029": 10,
+            "2030": 13
+            },
+        "summary": {
+            "max": 24,
+            "mean": 14.4,
+            "min": 9
+            }
+        },
+    ...
+}
+
+ +
Calculate annual statistic
+ +

+ Data are returned as JSON keyed hierarchically by scenario. Results + are the annual statistic value, and a summary of the maximum, mean, + and minimum of those values. +

+

+{
+    "historical": {
+        "data": {
+            "2000": 9.24,
+            "2001": 13.12,
+            "2002": 12.99,
+            "2003": 9.09,
+            "2004": 7.65,
+            "2005": 8.04,
+            "2006": 17.5,
+            "2007": 9.66,
+            "2008": 18.01,
+            "2009": 12.92,
+            "2010": 9.7,
+            "2011": 8.52,
+            "2012": 9.53,
+            "2013": 9.84,
+            "2014": 8.19
+            },
+        "summary": {
+            "max": 18.01,
+            "mean": 10.93,
+            "min": 7.65
+            }
+        },
+    "ssp126": {
+        "data": {
+            "2016": 8.33,
+            "2017": 20.16,
+            "2018": 8.33,
+            "2019": 10.45,
+            "2020": 9.44,
+            "2021": 13.5,
+            "2022": 11.63,
+            "2023": 11.29,
+            "2024": 8.2,
+            "2025": 11.06,
+            "2026": 17.36,
+            "2027": 14.05,
+            "2028": 10.67,
+            "2029": 8.84,
+            "2030": 7.96
+            },
+        "summary": {
+            "max": 20.16,
+            "mean": 11.42,
+            "min": 7.96
+            }
+        },
+    ...
+}
+
+ +
Rank annual values and get n-position
+ +

+ Data are returned as JSON keyed hierarchically by scenario. Results + are the annual rank value, and a summary of the maximum, mean, + and minimum of those values. +

+

+{
+    "historical": {
+        "data": {
+            "2000": -27.58,
+            "2001": -28.37,
+            "2002": -28.24,
+            "2003": -24.46,
+            "2004": -24.73,
+            "2005": -24.83,
+            "2006": -28.61,
+            "2007": -24.63,
+            "2008": -25.46,
+            "2009": -24.71,
+            "2010": -27.45,
+            "2011": -29.64,
+            "2012": -26.92,
+            "2013": -27.47,
+            "2014": -26.68
+            },
+        "summary": {
+            "max": -24.46,
+            "mean": -26.65,
+            "min": -29.64
+            }
+        },
+    "ssp126": {
+        "data": {
+            "2016": -27.28,
+            "2017": -25.53,
+            "2018": -23.39,
+            "2019": -25.74,
+            "2020": -24.14,
+            "2021": -26.39,
+            "2022": -24.82,
+            "2023": -25.08,
+            "2024": -25.39,
+            "2025": -25.87,
+            "2026": -25.14,
+            "2027": -25.58,
+            "2028": -23.52,
+            "2029": -26.43,
+            "2030": -25.61
+            },
+        "summary": {
+            "max": -23.39,
+            "mean": -25.33,
+            "min": -27.28
+            }
+        },
+    ...
+}
+
 
 
 {% endblock %}
\ No newline at end of file

From be260b43719d216328695d2e54df7ac5a5d76d88 Mon Sep 17 00:00:00 2001
From: joshdpaul 
Date: Fri, 7 Nov 2025 10:14:17 -0900
Subject: [PATCH 22/30] add area query documentation

---
 .../documentation/dynamic_indicators.html     | 109 +++++++++++++++++-
 1 file changed, 108 insertions(+), 1 deletion(-)

diff --git a/templates/documentation/dynamic_indicators.html b/templates/documentation/dynamic_indicators.html
index 22e4992c..9def3bb9 100644
--- a/templates/documentation/dynamic_indicators.html
+++ b/templates/documentation/dynamic_indicators.html
@@ -137,7 +137,7 @@ 
Point Query

         
           /dynamic_indicators/rank/6/lowest/tasmin/64.5/-147.5/2000/2030//dynamic_indicators/rank/6/lowest/tasmin/point/64.5/-147.5/2000/2030/
         
       
@@ -339,7 +339,114 @@ 
Rank annual values and get n-position

         },
     ...
 }
+
+ + +

Area Query

+ +

+ Area queries return zonal statistics aggregated over a specified + polygon area. Indicators are values represent the spatially-weighted average of all 4km × 4km grid + cells within the polygon boundary. + Learn more about what types of places are available for area queries. +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
EndpointExample URL
+ Count annual days above 25C in select years + + /dynamic_indicators/count_days/above/25/C/tasmax/area/1908030609/2000/2030/ +
+ Count annual days below -40F in select years + + /dynamic_indicators/count_days/below/-40/F/tasmin/area/1908030609/2000/2030/ +
+ Count annual days above 10mm precipitation in select years + + /dynamic_indicators/count_days/above/10/mm/pr/area/1908030609/2000/2030/ +
+ Get maximum 1-day precipitation in select years + + /dynamic_indicators/stat/max/pr/mm/area/1908030609/2000/2030/ +
+ Get maximum 1-day temperature in select years + + /dynamic_indicators/stat/max/tasmax/F/area/1908030609/2000/2030/ +
+ Get 6th coldest day in select years + + /dynamic_indicators/rank/6/lowest/tasmin/area/1908030609/2000/2030/ +
+ Ranking queries will return values in metric units (C + for tasmin and tasmax, mm for pr). +
+ +
+ +

Area Query Output

+

+ Area queries return the same JSON structure as point queries, but values + represent spatial averages rather than point measurements. Raw values are + averaged over the specified polygon area before applying the indicator calculation. +

{% endblock %} \ No newline at end of file From fd06a70a074294509e8ef0133840ddd4f3abb38f Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Fri, 7 Nov 2025 10:25:40 -0900 Subject: [PATCH 23/30] finish doc --- .../documentation/dynamic_indicators.html | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/templates/documentation/dynamic_indicators.html b/templates/documentation/dynamic_indicators.html index 9def3bb9..db9bc7ee 100644 --- a/templates/documentation/dynamic_indicators.html +++ b/templates/documentation/dynamic_indicators.html @@ -449,4 +449,33 @@

Area Query Output

+

Source data

+ + + + + + + + + + + + + + + + + +
Metadata & source data accessAcademic reference
+ Publication in progress. For more information, + please + contact us directly. + TBD
+ See this page for a full list of our CMIP6 data sources and suggested + citations. +
+ + + {% endblock %} \ No newline at end of file From b2be9cba47394f5187392a2fb571d6b825313025 Mon Sep 17 00:00:00 2001 From: Josh Paul <99696041+Joshdpaul@users.noreply.github.com> Date: Fri, 7 Nov 2025 10:38:36 -0900 Subject: [PATCH 24/30] Add validation for 'units' parameter in application.py --- application.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/application.py b/application.py index 2cfc1ee1..101afd75 100644 --- a/application.py +++ b/application.py @@ -124,6 +124,9 @@ class QueryParamsSchema(Schema): # Make sure "units" parameter is "in", "mm", "F", or "C" units = fields.Str( validate=lambda str: str in ["in", "mm", "F", "C"], + required=False, + ) + # Make sure "vars" parameter is only letters and commas, and less than # or equal to 100 characters long. def validate_vars(value): From cba06b11581379b9a97720bc60bb73a01415d09f Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Tue, 11 Nov 2025 09:39:56 -0900 Subject: [PATCH 25/30] remove "units" and "n" request params from marshmallow validation --- application.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/application.py b/application.py index 101afd75..1c37c8d8 100644 --- a/application.py +++ b/application.py @@ -115,18 +115,6 @@ class QueryParamsSchema(Schema): required=False, ) - # Make sure "n" parameter is string number between -1000 and 1000 - n = fields.Str( - validate=lambda str: str.isdigit() and -1000 <= int(str) <= 1000, - required=False, - ) - - # Make sure "units" parameter is "in", "mm", "F", or "C" - units = fields.Str( - validate=lambda str: str in ["in", "mm", "F", "C"], - required=False, - ) - # Make sure "vars" parameter is only letters and commas, and less than # or equal to 100 characters long. def validate_vars(value): From 5120e665be1150505cced37b5357929df610b643 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Tue, 11 Nov 2025 09:41:15 -0900 Subject: [PATCH 26/30] rm print statement --- generate_requests.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/generate_requests.py b/generate_requests.py index 34b15f33..9d21ac14 100644 --- a/generate_requests.py +++ b/generate_requests.py @@ -238,8 +238,6 @@ def construct_get_annual_mmm_stat_wcps_query_string( ) ) - print(query_string) - return query_string From 303a9a91f01af62724f0cade1675f29c6bc7912c Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Tue, 11 Nov 2025 10:18:04 -0900 Subject: [PATCH 27/30] improve documentation --- templates/documentation/dynamic_indicators.html | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/templates/documentation/dynamic_indicators.html b/templates/documentation/dynamic_indicators.html index db9bc7ee..c762035d 100644 --- a/templates/documentation/dynamic_indicators.html +++ b/templates/documentation/dynamic_indicators.html @@ -4,7 +4,7 @@

Dynamically Generated Indicators from Downscaled CMIP6

This endpoint provides access to dynamically generated indicators derived from a 6-model average of CMIP6 daily data that have been bias-adjusted and downscaled - using dynamically downscaled ERA5 reference data. The source dataset spans + using dynamically downscaled ERA5 reference data. The source dataset spans 1965–2100 at 4km spatial resolution. The spatial domain covers most of Alaska and parts of western Canada. Variables include daily maximum and minimum temperature (tasmax and tasmin), as well as daily total precipitation @@ -29,8 +29,7 @@

Available Operations

Counts number of days per year where values are above or below the provided <value> - given in the provided <units>. For area queries, daily values are weighted by spatial - area before applying the threshold. + given in the provided <units>. @@ -43,8 +42,7 @@

Available Operations

Calculates the specified annual statistic (max, min, mean, or sum) - for the provided <value> in the provided <units>. For area queries, daily values - are weighted by spatial area before calculating the statistic. + for the provided <value> in the provided <units>. @@ -55,8 +53,7 @@

Available Operations

Ranks values in each year and returns the value at the specified <position> - and <direction> (e.g., 5th highest, 2nd lowest). For area queries, daily values - are weighted by spatial area before ranking. + and <direction> (e.g., 5th highest, 2nd lowest). @@ -346,7 +343,7 @@

Area Query

Area queries return zonal statistics aggregated over a specified - polygon area. Indicators are values represent the spatially-weighted average of all 4km × 4km grid + polygon area. Indicators are calculated from mean daily values representing the spatially-weighted average of all 4km × 4km grid cells within the polygon boundary. Learn more about what types of places are available for area queries.

From a3c52cd9e53cc92deac13d166eeff539bc7441c4 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Tue, 11 Nov 2025 10:19:31 -0900 Subject: [PATCH 28/30] fix wrf link --- templates/documentation/dynamic_indicators.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/documentation/dynamic_indicators.html b/templates/documentation/dynamic_indicators.html index c762035d..ecc3fd77 100644 --- a/templates/documentation/dynamic_indicators.html +++ b/templates/documentation/dynamic_indicators.html @@ -4,7 +4,7 @@

Dynamically Generated Indicators from Downscaled CMIP6

This endpoint provides access to dynamically generated indicators derived from a 6-model average of CMIP6 daily data that have been bias-adjusted and downscaled - using dynamically downscaled ERA5 reference data. The source dataset spans + using dynamically downscaled ERA5 reference data. The source dataset spans 1965–2100 at 4km spatial resolution. The spatial domain covers most of Alaska and parts of western Canada. Variables include daily maximum and minimum temperature (tasmax and tasmin), as well as daily total precipitation From 28968e30bb7606898391f63e6fb940f3e531454d Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Tue, 11 Nov 2025 10:37:12 -0900 Subject: [PATCH 29/30] move validation functions --- routes/dynamic_indicators.py | 33 +++++++-------------------------- validate_request.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 4e7faefb..8870ecb4 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -17,6 +17,9 @@ validate_latlon_in_bboxes, construct_latlon_bbox_from_coverage_bounds, validate_var_id, + validate_operator, + validate_rank_position, + validate_rank_direction, ) from zonal_stats import ( get_scale_factor, @@ -112,17 +115,6 @@ def validate_latlon_and_reproject_to_epsg_3338(lat, lon, variable): return lon, lat -def validate_operator(operator): - """Validate operator is 'above' or 'below' and convert to '>' or '<'""" - if operator not in ["above", "below"]: - return render_template("400/bad_request.html"), 400 - if operator == "above": - operator = ">" - else: - operator = "<" - return operator - - def validate_units_threshold_and_variable(units, threshold, variable): """Validate units and threshold based on variable type. Convert threshold to standard units if needed.""" if variable in ["tasmax", "tasmin"]: @@ -160,19 +152,6 @@ def validate_stat(stat): return stat -def validate_rank_position_and_direction(position, direction): - """Validate rank position and direction. Position must be between 1 and 365, and direction must be 'highest' or 'lowest'.""" - try: - position = int(position) - if position < 1 or position > 365: - raise ValueError - except ValueError: - return render_template("400/bad_request.html"), 400 - if direction not in ["highest", "lowest"]: - return render_template("400/bad_request.html"), 400 - return position, direction - - def validate_place_id(place_id): poly_type = validate_var_id(place_id) if type(poly_type) is tuple: @@ -735,7 +714,8 @@ def get_annual_rank_point( if variable not in ["tasmax", "tasmin", "pr"]: return render_template("400/bad_request.html"), 400 try: - position, direction = validate_rank_position_and_direction(position, direction) + position = validate_rank_position(position) + direction = validate_rank_direction(direction) lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon, variable) validate_year(start_year, end_year) except: @@ -866,7 +846,8 @@ def get_annual_rank_area(position, direction, variable, place_id, start_year, en if variable not in ["tasmax", "tasmin", "pr"]: return render_template("400/bad_request.html"), 400 try: - position, direction = validate_rank_position_and_direction(position, direction) + position = validate_rank_position(position) + direction = validate_rank_direction(direction) validate_year(start_year, end_year) polygon = validate_place_id(place_id) except: diff --git a/validate_request.py b/validate_request.py index 44ac7659..d861eb86 100644 --- a/validate_request.py +++ b/validate_request.py @@ -565,3 +565,32 @@ def get_coverage_crs_str(coverage_metadata): ) return crs.to_string() + + +def validate_operator(operator): + """Validate operator is 'above' or 'below' and convert to '>' or '<'""" + if operator not in ["above", "below"]: + return render_template("400/bad_request.html"), 400 + if operator == "above": + operator = ">" + else: + operator = "<" + return operator + + +def validate_rank_position(position): + """Validate rank position. Position must be between 1 and 365.""" + try: + position = int(position) + if position < 1 or position > 365: + raise ValueError + except ValueError: + return render_template("400/bad_request.html"), 400 + return position + + +def validate_rank_direction(direction): + """Validate rank direction. Direction must be 'highest' or 'lowest'.""" + if direction not in ["highest", "lowest"]: + return render_template("400/bad_request.html"), 400 + return direction From 557d1db82ec6b106676e6376f4515c3c9d8321c8 Mon Sep 17 00:00:00 2001 From: joshdpaul Date: Tue, 11 Nov 2025 15:53:18 -0900 Subject: [PATCH 30/30] refactor validation routines --- routes/dynamic_indicators.py | 261 +++++++++++++++++++---------------- validate_request.py | 42 +++++- 2 files changed, 175 insertions(+), 128 deletions(-) diff --git a/routes/dynamic_indicators.py b/routes/dynamic_indicators.py index 8870ecb4..78594287 100644 --- a/routes/dynamic_indicators.py +++ b/routes/dynamic_indicators.py @@ -20,6 +20,8 @@ validate_operator, validate_rank_position, validate_rank_direction, + validate_stat, + validate_units_threshold_and_variable, ) from zonal_stats import ( get_scale_factor, @@ -80,12 +82,8 @@ async def get_cmip6_metadata(cov_id): return metadata -def validate_latlon_and_reproject_to_epsg_3338(lat, lon, variable): +def validate_latlon_in_coverage_bounds(lat, lon, variable): """Validate lat/lon, then reproject to EPSG:3338""" - lat = float(lat) - lon = float(lon) - if not latlon_is_numeric_and_in_geodetic_range(lat, lon): - return render_template("400/bad_request.html"), 400 var_coverages = all_coverages.get(variable) @@ -109,58 +107,27 @@ def validate_latlon_and_reproject_to_epsg_3338(lat, lon, variable): ), 422, ) + return True - lon, lat = project_latlon(lat, lon, dst_crs=3338) - return lon, lat +def convert_threshold(units, threshold): + """Convert threshold based on units.""" + if units == "F": + threshold = (float(threshold) - 32) * 5.0 / 9.0 + if units == "in": + threshold = float(threshold) * 25.4 + if units in ["C", "mm"]: + threshold = float(threshold) + return threshold -def validate_units_threshold_and_variable(units, threshold, variable): - """Validate units and threshold based on variable type. Convert threshold to standard units if needed.""" - if variable in ["tasmax", "tasmin"]: - if units not in ["C", "F"]: - raise ValueError("Units for temperature must be 'C' or 'F'.") - if threshold is not None: - if units == "F": - threshold = (float(threshold) - 32) * 5.0 / 9.0 - else: - threshold = float(threshold) - elif variable == "pr": - if units not in ["mm", "in"]: - return render_template("400/bad_request.html"), 400 - if threshold is not None: - if units == "in": - threshold = float(threshold) * 25.4 - else: - threshold = float(threshold) +def convert_operator(operator): + """Convert operator from 'above' or 'below' to '>' or '<'""" + if operator == "above": + operator = ">" else: - return render_template("400/bad_request.html"), 400 - - # precipitation thresholds should be >= 0 - if variable == "pr" and threshold is not None and threshold < 0: - return render_template("400/bad_request.html"), 400 - - return units, threshold - - -def validate_stat(stat): - """Validate that stat is one of 'max', 'min', 'mean', or 'sum'.""" - if stat not in ["max", "min", "mean", "sum"]: - return render_template("400/bad_request.html"), 400 - if stat == "mean": - stat = "avg" # NOTE: rasdaman uses 'avg' instead of 'mean' in WCPS queries - return stat - - -def validate_place_id(place_id): - poly_type = validate_var_id(place_id) - if type(poly_type) is tuple: - return poly_type - try: - polygon = get_poly(place_id) - except: - return render_template("422/invalid_area.html"), 422 - return polygon + operator = "<" + return operator def build_year_and_coverage_lists_for_iteration( @@ -336,10 +303,6 @@ def postprocess_count_days(data, start_year, end_year): } } """ - - start_year = int(start_year) - end_year = int(end_year) - result = {} current_index = 0 if ( @@ -474,8 +437,6 @@ def c_to_f(c): else: # round values regardless of units convert = lambda x: round(x, 2) - start_year = int(start_year) - end_year = int(end_year) result = {} current_index = 0 @@ -536,8 +497,6 @@ def c_to_f(c): def postprocess_annual_rank(data, start_year, end_year, position, direction): """Postprocess annual rank data into structured dictionary output.""" - start_year = int(start_year) - end_year = int(end_year) result = {} current_index = 0 @@ -629,29 +588,42 @@ def count_days_point( - http://127.0.0.1:5000/dynamic_indicators/count_days/above/10/mm/pr/point/64.5/-147.5/2000/2030/ ->>> can recreate the "days above 10mm precip" indicator - http://127.0.0.1:5000/dynamic_indicators/count_days/above/1/mm/pr/point/64.5/-147.5/2000/2030/ ->>> can recreate the "wet days" indicator """ - # Validate request params - try: - operator = validate_operator(operator) - units, threshold = validate_units_threshold_and_variable( - units=units, threshold=threshold, variable=variable - ) - lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon, variable) - validate_year(start_year, end_year) - except: + # validations + year_validation = validate_year(start_year, end_year) + latlon_validation = latlon_is_numeric_and_in_geodetic_range(float(lat), float(lon)) + if latlon_validation == 400 or year_validation == 400: return render_template("400/bad_request.html"), 400 + bound_validation = validate_latlon_in_coverage_bounds( + float(lat), float(lon), variable + ) + if bound_validation is not True: + return bound_validation + + op_validation = validate_operator(operator) + if op_validation is not True: + return op_validation + utv_validation = validate_units_threshold_and_variable(units, threshold, variable) + if utv_validation is not True: + return utv_validation + + # conversions + lon, lat = project_latlon(float(lat), float(lon), dst_crs=3338) + operator = convert_operator(operator) + threshold = convert_threshold(units, threshold) # build lists for iteration year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( int(start_year), int(end_year), variable, time_domains, all_coverages ) + # fetch and postprocess data try: data = asyncio.run( fetch_count_days_data( var_coverages, year_ranges, threshold, operator, lon, lat ) ) - result = postprocess_count_days(data, start_year, end_year) + result = postprocess_count_days(data, int(start_year), int(end_year)) return result except Exception as exc: if hasattr(exc, "status") and exc.status == 404: @@ -671,27 +643,40 @@ def get_annual_stat_point(stat, variable, units, lat, lon, start_year, end_year) - http://127.0.0.1:5000/dynamic_indicators/stat/sum/pr/mm/point/64.5/-147.5/2000/2030/ ->>> total annual precipitation (NOTE: summary section of return will show mean annual precip over the year range) - http://127.0.0.1:5000/dynamic_indicators/stat/mean/pr/mm/point/64.5/-147.5/2000/2030/ ->>> mean daily precipitation (NOTE: this is not a common mean statistic for precip - avg amount of precip per day over the year) """ - # Validate request params - try: - stat = validate_stat(stat) - units, _threshold = validate_units_threshold_and_variable( - units=units, threshold=None, variable=variable - ) - lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon, variable) - validate_year(start_year, end_year) - except: + # validations + year_validation = validate_year(start_year, end_year) + latlon_validation = latlon_is_numeric_and_in_geodetic_range(float(lat), float(lon)) + if latlon_validation == 400 or year_validation == 400: return render_template("400/bad_request.html"), 400 + bound_validation = validate_latlon_in_coverage_bounds( + float(lat), float(lon), variable + ) + if bound_validation is not True: + return bound_validation + + utv_validation = validate_units_threshold_and_variable( + units=units, threshold=None, variable=variable + ) + if utv_validation is not True: + return utv_validation + stat = validate_stat(stat) + if type(stat) is tuple: + return stat + + # conversions + lon, lat = project_latlon(float(lat), float(lon), dst_crs=3338) # build lists for iteration year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( int(start_year), int(end_year), variable, time_domains, all_coverages ) + # fetch and postprocess data try: data = asyncio.run( fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat) ) - result = postprocess_annual_stat(data, start_year, end_year, units) + result = postprocess_annual_stat(data, int(start_year), int(end_year), units) return result except Exception as exc: if hasattr(exc, "status") and exc.status == 404: @@ -710,29 +695,41 @@ def get_annual_rank_point( - http://127.0.0.1:5000/dynamic_indicators/rank/6/highest/tasmax/point/64.5/-147.5/2000/2030/ ->>> can recreate "hot day threshold" indicator - http://127.0.0.1:5000/dynamic_indicators/rank/6/lowest/tasmin/point/64.5/-147.5/2000/2030/ ->>> can recreate "cold day threshold" indicators """ - # Validate request params + # validations if variable not in ["tasmax", "tasmin", "pr"]: return render_template("400/bad_request.html"), 400 - try: - position = validate_rank_position(position) - direction = validate_rank_direction(direction) - lon, lat = validate_latlon_and_reproject_to_epsg_3338(lat, lon, variable) - validate_year(start_year, end_year) - except: + year_validation = validate_year(start_year, end_year) + latlon_validation = latlon_is_numeric_and_in_geodetic_range(float(lat), float(lon)) + if latlon_validation == 400 or year_validation == 400: return render_template("400/bad_request.html"), 400 + bound_validation = validate_latlon_in_coverage_bounds( + float(lat), float(lon), variable + ) + if bound_validation is not True: + return bound_validation + position_validation = validate_rank_position(position) + if position_validation is not True: + return position_validation + direction_validation = validate_rank_direction(direction) + if direction_validation is not True: + return direction_validation + + # conversions + lon, lat = project_latlon(float(lat), float(lon), dst_crs=3338) # build lists for iteration year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( int(start_year), int(end_year), variable, time_domains, all_coverages ) + # fetch and postprocess data stat = "" # NOTE: omitting stat will force a return of all values, which we need for ranking try: data = asyncio.run( fetch_annual_stat_data(var_coverages, year_ranges, stat, lon, lat) ) result = postprocess_annual_rank( - data, start_year, end_year, position, direction + data, int(start_year), int(end_year), int(position), direction ) return result except Exception as exc: @@ -758,32 +755,40 @@ def count_days_area( - http://127.0.0.1:5000/dynamic_indicators/count_days/above/10/mm/pr/area/1908030609/2000/2030/ ->>> can recreate the "days above 10mm precip" indicator - http://127.0.0.1:5000/dynamic_indicators/count_days/above/1/mm/pr/area/1908030609/2000/2030/ ->>> can recreate the "wet days" indicator """ - # Validate request params - try: - operator = validate_operator(operator) - units, threshold = validate_units_threshold_and_variable( - units=units, threshold=threshold, variable=variable - ) - validate_year(start_year, end_year) - polygon = validate_place_id(place_id) - except: + # validations + poly_type = validate_var_id(place_id) + if type(poly_type) is tuple: + return poly_type + else: + polygon = get_poly(place_id) + year_validation = validate_year(start_year, end_year) + if year_validation == 400: return render_template("400/bad_request.html"), 400 + op_validation = validate_operator(operator) + if op_validation is not True: + return op_validation + utv_validation = validate_units_threshold_and_variable(units, threshold, variable) + if utv_validation is not True: + return utv_validation + + # conversions + operator = convert_operator(operator) + threshold = convert_threshold(units, threshold) + # build lists for iteration year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( int(start_year), int(end_year), variable, time_domains, all_coverages ) + # fetch and postprocess data try: data = asyncio.run( fetch_count_days_area_data( variable, var_coverages, year_ranges, threshold, operator, polygon ) ) - - print(data) - - result = postprocess_count_days(data, start_year, end_year) + result = postprocess_count_days(data, int(start_year), int(end_year)) return result except Exception as exc: if hasattr(exc, "status") and exc.status == 404: @@ -803,29 +808,36 @@ def get_annual_stat_area(stat, variable, units, place_id, start_year, end_year): - http://127.0.0.1:5000/dynamic_indicators/stat/sum/pr/mm/area/1908030609/2000/2030/ ->>> total annual precipitation (NOTE: summary section of return will show mean annual precip over the year range) - http://127.0.0.1:5000/dynamic_indicators/stat/mean/pr/mm/area/1908030609/2000/2030/ ->>> mean daily precipitation (NOTE: this is not a common mean statistic for precip - avg amount of precip per day over the year) """ - # Validate request params - try: - stat = validate_stat(stat) - units, _threshold = validate_units_threshold_and_variable( - units=units, threshold=None, variable=variable - ) - validate_year(start_year, end_year) - polygon = validate_place_id(place_id) - except: + # validations + poly_type = validate_var_id(place_id) + if type(poly_type) is tuple: + return poly_type + else: + polygon = get_poly(place_id) + year_validation = validate_year(start_year, end_year) + if year_validation == 400: return render_template("400/bad_request.html"), 400 + utv_validation = validate_units_threshold_and_variable( + units, threshold=None, variable=variable + ) + if utv_validation is not True: + return utv_validation + stat = validate_stat(stat) + # build lists for iteration year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( int(start_year), int(end_year), variable, time_domains, all_coverages ) + # fetch and postprocess data try: data = asyncio.run( fetch_annual_stat_area_data( variable, var_coverages, year_ranges, stat, polygon ) ) - result = postprocess_annual_stat(data, start_year, end_year, units) + result = postprocess_annual_stat(data, int(start_year), int(end_year), units) return result except Exception as exc: if hasattr(exc, "status") and exc.status == 404: @@ -842,24 +854,31 @@ def get_annual_rank_area(position, direction, variable, place_id, start_year, en - http://127.0.0.1:5000/dynamic_indicators/rank/6/highest/tasmax/area/1908030609/2000/2030/ ->>> can recreate "hot day threshold" indicator - http://127.0.0.1:5000/dynamic_indicators/rank/6/lowest/tasmin/area/1908030609/2000/2030/ ->>> can recreate "cold day threshold" indicators """ - # Validate request params + # validations if variable not in ["tasmax", "tasmin", "pr"]: return render_template("400/bad_request.html"), 400 - try: - position = validate_rank_position(position) - direction = validate_rank_direction(direction) - validate_year(start_year, end_year) - polygon = validate_place_id(place_id) - except: + poly_type = validate_var_id(place_id) + if type(poly_type) is tuple: + return poly_type + else: + polygon = get_poly(place_id) + year_validation = validate_year(start_year, end_year) + if year_validation == 400: return render_template("400/bad_request.html"), 400 + position_validation = validate_rank_position(position) + if position_validation is not True: + return position_validation + direction_validation = validate_rank_direction(direction) + if direction_validation is not True: + return direction_validation # build lists for iteration year_ranges, var_coverages = build_year_and_coverage_lists_for_iteration( int(start_year), int(end_year), variable, time_domains, all_coverages ) - # stat is empty to force return of all values for ranking - stat = "" + # fetch and postprocess data + stat = "" # NOTE: omitting stat will force a return of all values, which we need for ranking try: data = asyncio.run( fetch_annual_stat_area_data( @@ -867,7 +886,7 @@ def get_annual_rank_area(position, direction, variable, place_id, start_year, en ) ) result = postprocess_annual_rank( - data, start_year, end_year, position, direction + data, int(start_year), int(end_year), int(position), direction ) return result except Exception as exc: diff --git a/validate_request.py b/validate_request.py index d861eb86..3b84a74f 100644 --- a/validate_request.py +++ b/validate_request.py @@ -571,11 +571,7 @@ def validate_operator(operator): """Validate operator is 'above' or 'below' and convert to '>' or '<'""" if operator not in ["above", "below"]: return render_template("400/bad_request.html"), 400 - if operator == "above": - operator = ">" - else: - operator = "<" - return operator + return True def validate_rank_position(position): @@ -586,11 +582,43 @@ def validate_rank_position(position): raise ValueError except ValueError: return render_template("400/bad_request.html"), 400 - return position + return True def validate_rank_direction(direction): """Validate rank direction. Direction must be 'highest' or 'lowest'.""" if direction not in ["highest", "lowest"]: return render_template("400/bad_request.html"), 400 - return direction + return True + + +def validate_stat(stat): + """Validate that stat is one of 'max', 'min', 'mean', or 'sum'.""" + if stat not in ["max", "min", "mean", "sum"]: + return render_template("400/bad_request.html"), 400 + if stat == "mean": + stat = "avg" # NOTE: rasdaman uses 'avg' instead of 'mean' in WCPS queries + return stat + + +def validate_units_threshold_and_variable(units, threshold, variable): + """Validate units and threshold based on variable type.""" + if units not in ["C", "F", "mm", "in"]: + return render_template("400/bad_request.html"), 400 + if threshold is not None: + try: + threshold = float(threshold) + except ValueError: + return render_template("400/bad_request.html"), 400 + if variable not in ["tasmax", "tasmin", "pr"]: + return render_template("400/bad_request.html"), 400 + if variable in ["tasmax", "tasmin"]: + if units not in ["C", "F"]: + return render_template("400/bad_request.html"), 400 + if variable == "pr": + if units not in ["mm", "in"]: + return render_template("400/bad_request.html"), 400 + # precipitation thresholds should be >= 0 + if variable == "pr" and threshold is not None and threshold < 0: + return render_template("400/bad_request.html"), 400 + return True