From 5ce7ae83435d7e124d40920c3a0021fa325a06c1 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 6 Feb 2025 13:56:18 -0700 Subject: [PATCH 01/18] start thinking about snotel rewrite --- metloom/pointdata/snotel.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/metloom/pointdata/snotel.py b/metloom/pointdata/snotel.py index f978d92..d85b071 100644 --- a/metloom/pointdata/snotel.py +++ b/metloom/pointdata/snotel.py @@ -4,6 +4,7 @@ import geopandas as gpd import pandas as pd from functools import reduce +import requests from .base import PointData from ..variables import SnotelVariables, SensorDescription @@ -30,6 +31,7 @@ class SnotelPointData(PointData): ALLOWED_VARIABLES = SnotelVariables DATASOURCE = "NRCS" + API_URL = "https://wcc.sc.egov.usda.gov/awdbRestApi/" def __init__(self, station_id, name, metadata=None): """ @@ -115,7 +117,19 @@ def _fetch_data_for_variables(self, client: SeriesSnotelClient, ): result_map = {} for variable in variables: +<<<<<<< Updated upstream params = variable.extra or {} +======= + # need to add extra_params for ground temp call, this may not be the + # best logic + if 'GROUND' in variable.name or 'SOIL' in variable.name: + params = extra_params[variable.name] + else: + params = {} + requests.get( + self.API_URL + "" + ) +>>>>>>> Stashed changes data = client.get_data(element_cd=variable.code, **params) if len(data) > 0: result_map[variable] = data From b245160ff67f93c00039a33f6270a498f9123f73 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 3 Jul 2025 16:48:05 -0600 Subject: [PATCH 02/18] Start work on switching APIs --- metloom/pointdata/__init__.py | 2 +- metloom/pointdata/base.py | 2 +- metloom/pointdata/snotel/__init__.py | 2 + metloom/pointdata/{ => snotel}/snotel.py | 124 +++++++------ .../pointdata/{ => snotel}/snotel_client.py | 101 ---------- metloom/pointdata/snotel/variables.py | 108 +++++++++++ metloom/sensors.py | 68 +++++++ metloom/variables.py | 174 +----------------- 8 files changed, 251 insertions(+), 330 deletions(-) create mode 100644 metloom/pointdata/snotel/__init__.py rename metloom/pointdata/{ => snotel}/snotel.py (76%) rename metloom/pointdata/{ => snotel}/snotel_client.py (60%) create mode 100644 metloom/pointdata/snotel/variables.py create mode 100644 metloom/sensors.py diff --git a/metloom/pointdata/__init__.py b/metloom/pointdata/__init__.py index 7ad0419..d152c2e 100644 --- a/metloom/pointdata/__init__.py +++ b/metloom/pointdata/__init__.py @@ -1,6 +1,6 @@ from .base import PointData, PointDataCollection from .cdec import CDECPointData -from .snotel import SnotelPointData +from metloom.pointdata.snotel.snotel import SnotelPointData from .mesowest import MesowestPointData from .usgs import USGSPointData from .geosphere_austria import GeoSphereHistPointData, GeoSphereCurrentPointData diff --git a/metloom/pointdata/base.py b/metloom/pointdata/base.py index 47f0591..4d4cd5d 100644 --- a/metloom/pointdata/base.py +++ b/metloom/pointdata/base.py @@ -7,7 +7,7 @@ import geopandas as gpd -from ..variables import SensorDescription, VariableBase +from ..sensors import SensorDescription, VariableBase LOG = logging.getLogger("metloom.pointdata.base") diff --git a/metloom/pointdata/snotel/__init__.py b/metloom/pointdata/snotel/__init__.py new file mode 100644 index 0000000..0014c11 --- /dev/null +++ b/metloom/pointdata/snotel/__init__.py @@ -0,0 +1,2 @@ +from .snotel import SnotelPointData +from .variables import SnotelVariables diff --git a/metloom/pointdata/snotel.py b/metloom/pointdata/snotel/snotel.py similarity index 76% rename from metloom/pointdata/snotel.py rename to metloom/pointdata/snotel/snotel.py index d85b071..43f2dd9 100644 --- a/metloom/pointdata/snotel.py +++ b/metloom/pointdata/snotel/snotel.py @@ -6,9 +6,9 @@ from functools import reduce import requests -from .base import PointData -from ..variables import SnotelVariables, SensorDescription -from ..dataframe_utils import append_df, merge_df +from metloom.pointdata.base import PointData +from .variables import SnotelVariables, SensorDescription +from metloom.dataframe_utils import append_df, merge_df from .snotel_client import ( DailySnotelDataClient, MetaDataSnotelClient, HourlySnotelDataClient, @@ -44,8 +44,10 @@ def __init__(self, station_id, name, metadata=None): self._raw_elements = None self._tzinfo = None - def _snotel_response_to_df(self, result_map: Dict[SensorDescription, List[dict]], - duration: str, include_measurement_date=False): + def _snotel_response_to_df( + self, result_map: Dict[SensorDescription, Dict[str, List[dict]]], + duration: str, include_measurement_date=False + ): """ Convert the response from climata.snotel classes into Args: @@ -63,14 +65,19 @@ def _snotel_response_to_df(self, result_map: Dict[SensorDescription, List[dict]] if include_measurement_date: final_columns += ["measurementDate"] - for variable, data in result_map.items(): + date_key = "datetime" if duration == "HOURLY" else "date" + for variable, info in result_map.items(): + data = info["values"] + element = info["stationElement"] + unit_name = element["storedUnitCode"] transformed = [] for row in data: + row_obj = { - "datetime": row["datetime"], + "datetime": row[date_key], "site": self.id, variable.name: row["value"], - f"{variable.name}_units": self._get_units(variable, duration), + f"{variable.name}_units": unit_name, } if include_measurement_date: row_obj["measurementDate"] = row["datetime"] @@ -110,33 +117,60 @@ def _snotel_response_to_df(self, result_map: Dict[SensorDescription, List[dict]] self.validate_sensor_df(df) return df - def _fetch_data_for_variables(self, client: SeriesSnotelClient, - variables: List[SensorDescription], - duration: str, - include_measurement_date=False, - ): + def _fetch_data_for_variables( + self, start_date: datetime, end_date: datetime, + variables: List[SensorDescription], + duration: str, + include_measurement_date=False, + ): + """ + Fetch data for the given variables using the Snotel API. + Args: + start_date: start date for the data + end_date: end date for the data + variables: list of SensorDescription objects for the variables + duration: string representation of the duration tag for the + API (i.e. HOURLY) + include_measurement_date: boolean for including the + 'measurementDate' column in the resulting dataframe. + + """ + endpoint_url = self.API_URL + "services/v1/data" result_map = {} + params = dict( + beginDate=start_date.strftime("%Y-%m-%d %H:%M"), + endDate=end_date.strftime("%Y-%m-%d %H:%M"), + stationTriplets=self.id, + duration=duration, + ) for variable in variables: -<<<<<<< Updated upstream - params = variable.extra or {} -======= - # need to add extra_params for ground temp call, this may not be the - # best logic - if 'GROUND' in variable.name or 'SOIL' in variable.name: - params = extra_params[variable.name] + extra = variable.extra or {} + height_depth = extra.get("height_depth", {}) + # Add the height depth for sensors with a heigh component + if height_depth: + code = variable.code + f":{height_depth['value']}" else: - params = {} - requests.get( - self.API_URL + "" + code = variable.code + + # TODO: we could request multiple variables at once + result = requests.get( + endpoint_url, params={**params, "elements": code} ) ->>>>>>> Stashed changes - data = client.get_data(element_cd=variable.code, **params) - if len(data) > 0: - result_map[variable] = data + result.raise_for_status() + data = result.json() + # Get the first station return, since we only requested one station + data = data[0]["data"] + # TODO: this is where we could iterate through multiple variables + # if we wanted to. We would need to be careful of the meas height + if len(data) == 1: + result_map[variable] = data[0] + elif len(data["data"]) > 1: + raise RuntimeError("We received too many results") else: LOG.warning(f"No {variable.name} found for {self.name}") return self._snotel_response_to_df( - result_map, duration, include_measurement_date=include_measurement_date + result_map, duration, + include_measurement_date=include_measurement_date ) def get_daily_data( @@ -148,13 +182,9 @@ def get_daily_data( """ See docstring for PointData.get_daily_data """ - client = DailySnotelDataClient( - station_triplet=self.id, - begin_date=start_date, - end_date=end_date, + return self._fetch_data_for_variables( + start_date, end_date, variables, "DAILY" ) - return self._fetch_data_for_variables(client, variables, - client.DURATION) def get_hourly_data( self, @@ -165,13 +195,9 @@ def get_hourly_data( """ See docstring for PointData.get_hourly_data """ - - client = HourlySnotelDataClient( - station_triplet=self.id, - begin_date=start_date, - end_date=end_date, + return self._fetch_data_for_variables( + start_date, end_date, variables, "HOURLY" ) - return self._fetch_data_for_variables(client, variables, "HOURLY") def get_snow_course_data( self, @@ -182,13 +208,9 @@ def get_snow_course_data( """ See docstring for PointData.get_snow_course_data """ - client = SemiMonthlySnotelClient( - station_triplet=self.id, - begin_date=start_date, - end_date=end_date, - ) + return self._fetch_data_for_variables( - client, variables, client.DURATION, include_measurement_date=True + start_date, end_date, variables, "SEMIMONTHLY", include_measurement_date=True ) def _get_all_metadata(self): @@ -209,16 +231,6 @@ def _get_all_elements(self): self._raw_elements = client.get_data() return self._raw_elements - def _get_units(self, variable: SensorDescription, duration: str): - units = None - for meta in self._get_all_elements(): - if meta["elementCd"] == variable.code and meta["duration"] == duration: - units = meta["storedUnitCd"] - break - if units is None: - raise ValueError(f"Could not find units for {variable}") - return units - def _get_metadata(self): """ See docstring for PointData._get_metadata diff --git a/metloom/pointdata/snotel_client.py b/metloom/pointdata/snotel/snotel_client.py similarity index 60% rename from metloom/pointdata/snotel_client.py rename to metloom/pointdata/snotel/snotel_client.py index ff4451c..26cfdd2 100644 --- a/metloom/pointdata/snotel_client.py +++ b/metloom/pointdata/snotel/snotel_client.py @@ -141,104 +141,3 @@ def __init__(self, max_latitude: float, min_latitude: float, network_cds=network_cds, element_cds=element_cds, **kwargs ) - - -class SeriesSnotelClient(BaseSnotelClient): - """ - Base extension for services that return timseries data. - """ - SERVICE_NAME = "getData" - DURATION = "DAILY" - DEFAULT_PARAMS = { - "ordinal": 1, - "getFlags": "true", - "alwaysReturnDailyFeb29": "false", - } - - def __init__(self, begin_date: datetime, end_date: datetime, - station_triplet: str, **kwargs): - super(SeriesSnotelClient, self).__init__( - begin_date=begin_date, end_date=end_date, - station_triplets=[station_triplet], **kwargs) - - @staticmethod - def _parse_data(raw_data): - """ - Parse the return data to return a consistent format for timeseries - data - """ - data = raw_data[0] - mapped_data = [] - collection_dates = getattr(data, "collectionDates", None) - if len(data["values"]) == 0: - return mapped_data - if collection_dates: - date_list = [pd.to_datetime(d) for d in collection_dates] - else: - date_list = pd.date_range(data["beginDate"], data["endDate"]) - - for date_obj, flag, value in zip(date_list, data["flags"], data["values"]): - if date_obj is not None: - mapped_data.append({ - "datetime": date_obj, - "flag": flag, - "value": float(value) if value is not None else None - }) - return mapped_data - - def get_data(self, element_cd: str, **extra_params): - """ - get the timeseires data - - Args: - element_cd: the variable code from the allowed variable codes - in the API - extra_params: kwargs for any extra parameters. These override - the default parameters - """ - extra_params.update(element_cd=element_cd) - mapped_params = self._get_params(**extra_params) - params = {**self.params, **mapped_params} - data = self._make_request(**params) - return self._parse_data(data) - - -class DailySnotelDataClient(SeriesSnotelClient): - """ - Class for getting daily data - """ - pass - - -class SemiMonthlySnotelClient(SeriesSnotelClient): - """ - Class for getting semi monthly (snow course) data - """ - DURATION = "SEMIMONTHLY" - - -class HourlySnotelDataClient(SeriesSnotelClient): - """ - Class for getting hourly data - """ - DURATION = None - SERVICE_NAME = "getHourlyData" - DEFAULT_PARAMS = { - "ordinal": 1, - } - - @staticmethod - def _parse_data(raw_data): - """ - Clean the hourly data to be consistent with timeseries results - """ - data = raw_data[0] - mapped_data = [] - for row in data["values"]: - value = row["value"] - mapped_data.append({ - "datetime": pd.to_datetime(row["dateTime"]), - "flag": row["flag"], - "value": float(value) if value is not None else None - }) - return mapped_data diff --git a/metloom/pointdata/snotel/variables.py b/metloom/pointdata/snotel/variables.py new file mode 100644 index 0000000..c86f5e2 --- /dev/null +++ b/metloom/pointdata/snotel/variables.py @@ -0,0 +1,108 @@ +from dataclasses import field, make_dataclass + +from metloom.sensors import SensorDescription, VariableBase + +# Available sensors from Snotel +SnotelVariables = make_dataclass( + "SnotelVariables", + [ + ( + "SNOWDEPTH", + SensorDescription, + field(default=SensorDescription("SNWD", "SNOWDEPTH")), + ), + ("SWE", SensorDescription, field(default=SensorDescription("WTEQ", "SWE"))), + ( + "TEMP", + SensorDescription, + field(default=SensorDescription("TOBS", "AIR TEMP")), + ), + ( + "TEMPAVG", + SensorDescription, + field(default=SensorDescription("TAVG", "AVG AIR TEMP", "AIR TEMPERATURE AVERAGE")), + ), + ( + "TEMPMIN", + SensorDescription, + field(default=SensorDescription("TMIN", "MIN AIR TEMP", "AIR TEMPERATURE MINIMUM")), + ), + ( + "TEMPMAX", + SensorDescription, + field(default=SensorDescription("TMAX", "MAX AIR TEMP", "AIR TEMPERATURE MAXIMUM")), + ), + ( + "PRECIPITATION", + SensorDescription, + field(default=SensorDescription("PRCPSA", "PRECIPITATION", "PRECIPITATION INCREMENT SNOW-ADJUSTED")), + ), + ( + "PRECIPITATIONACCUM", + SensorDescription, + field(default=SensorDescription("PREC", "ACCUMULATED PRECIPITATION", "PRECIPITATION ACCUMULATION")), + ), + # TODO for the SCAN network this appears to be "RHUM", we may need a new class + ( + "RH", + SensorDescription, + field(default=SensorDescription("RHUMV", "Relative Humidity", "RELATIVE HUMIDITY")), + ), + ( + "STREAMVOLUMEOBS", + SensorDescription, + field(default=SensorDescription("SRVO", "STREAM VOLUME OBS", "STREAM VOLUME OBS")), + ), + ( + "STREAMVOLUMEADJ", + SensorDescription, + field(default=SensorDescription("SRVOX", "STREAM VOLUME ADJ", "STREAM VOLUME ADJ")), + ), + ] + + [ + ( + f"TEMPGROUND{abs(d)}IN", + SensorDescription, + field( + default=SensorDescription( + "STO", + f"GROUND TEMPERATURE -{d}IN", + f"GROUND TEMPERATURE OBS -{d}IN", + extra={"height_depth": {"value": -d, "unitCd": "in"}}, + ) + ), + ) + for d in [2, 4, 8, 20] + ] + + [ + ( + f"SOILMOISTURE{abs(d)}IN", + SensorDescription, + field( + default=SensorDescription( + "SMS", + f"SOIL MOISTURE -{d}IN", + f"SOIL MOISTURE PERCENT -{d}IN", + extra={"height_depth": {"value": -d, "unitCd": "in"}}, + ) + ), + ) + for d in [2, 4, 8, 20] + ] + + [ + ( + f"TEMPPROFILENEG{abs(d)}IN" if d < 0 else f"TEMPPROFILE{d}IN", + SensorDescription, + field( + default=SensorDescription( + "PTEMP", + f"PROFILE TEMPERATURE {d}IN", + f"PROFILE TEMPERATURE OBS{d}IN", + extra={"height_depth": {"value": d, "unitCd": "in"}}, + ) + ), + ) + for d in [-8, 0, 8, 16, 24, 31, 39, 47, 55, 63, 71, 79, 87, 94, 102, 110, 118, 126] # noqa: E501 + ], + bases=(VariableBase,), +) diff --git a/metloom/sensors.py b/metloom/sensors.py new file mode 100644 index 0000000..a3eb59d --- /dev/null +++ b/metloom/sensors.py @@ -0,0 +1,68 @@ +from dataclasses import field, dataclass +import typing + + +@dataclass(eq=True, frozen=True) +class SensorDescription: + """ + data class for describing a snow sensor + """ + + code: str = "-1" # code used within the applicable API + name: str = "basename" # desired name for the sensor + description: str = None # description of the sensor + accumulated: bool = False # whether the data is accumulated + units: str = None # Optional units kwarg + extra: typing.Any = field(default=None, hash=False) # Optional extra data for sub-class specific information + + +@dataclass(eq=True, frozen=True) +class InstrumentDescription(SensorDescription): + """ + Extend the Sensor Description to include instrument + """ + + # description of the specific instrument for the variable + instrument: str = None + + +class VariableBase: + """ + Base class to store all variables for a specific datasource. Each + datasource should implement the class. The goal is that the variables + are synonymous across implementations.(i.e. PRECIPITATION should have the + same meaning in each implementation). + Additionally, variables with the same meaning should have the same + `name` attribute of the SensorDescription. This way, if multiple datsources + are used to sample the same variable, they can be written to the same + column in a csv. + + Variables in this base class should ideally be implemented by all classes + and cannot be directly used from the base class. + """ + + PRECIPITATION = SensorDescription() + SWE = SensorDescription() + SNOWDEPTH = SensorDescription() + + @staticmethod + def _validate_sensor(sensor: SensorDescription): + """ + Validate that a sensor is not using the default values since they + are meaningless + """ + default = SensorDescription() + if sensor.name == default.name and sensor.code == default.code: + raise ValueError(f"{sensor.name} is the default implementation") + + @classmethod + def from_code(cls, code): + """ + Get the correct sensor description from the code + """ + for k, v in cls.__dict__.items(): + if isinstance(v, SensorDescription) and v.code == str(code): + cls._validate_sensor(v) + return v + raise ValueError(f"Could not find sensor for code {code}") + diff --git a/metloom/variables.py b/metloom/variables.py index ba4bc97..dc2e813 100644 --- a/metloom/variables.py +++ b/metloom/variables.py @@ -1,70 +1,7 @@ -from dataclasses import dataclass, field, make_dataclass -import typing +from metloom.sensors import VariableBase, SensorDescription, InstrumentDescription - -@dataclass(eq=True, frozen=True) -class SensorDescription: - """ - data class for describing a snow sensor - """ - - code: str = "-1" # code used within the applicable API - name: str = "basename" # desired name for the sensor - description: str = None # description of the sensor - accumulated: bool = False # whether the data is accumulated - units: str = None # Optional units kwarg - extra: typing.Any = field(default=None, hash=False) # Optional extra data for sub-class specific information - - -@dataclass(eq=True, frozen=True) -class InstrumentDescription(SensorDescription): - """ - Extend the Sensor Description to include instrument - """ - - # description of the specific instrument for the variable - instrument: str = None - - -class VariableBase: - """ - Base class to store all variables for a specific datasource. Each - datasource should implement the class. The goal is that the variables - are synonymous across implementations.(i.e. PRECIPITATION should have the - same meaning in each implementation). - Additionally, variables with the same meaning should have the same - `name` attribute of the SensorDescription. This way, if multiple datsources - are used to sample the same variable, they can be written to the same - column in a csv. - - Variables in this base class should ideally be implemented by all classes - and cannot be directly used from the base class. - """ - - PRECIPITATION = SensorDescription() - SWE = SensorDescription() - SNOWDEPTH = SensorDescription() - - @staticmethod - def _validate_sensor(sensor: SensorDescription): - """ - Validate that a sensor is not using the default values since they - are meaningless - """ - default = SensorDescription() - if sensor.name == default.name and sensor.code == default.code: - raise ValueError(f"{sensor.name} is the default implementation") - - @classmethod - def from_code(cls, code): - """ - Get the correct sensor description from the code - """ - for k, v in cls.__dict__.items(): - if isinstance(v, SensorDescription) and v.code == str(code): - cls._validate_sensor(v) - return v - raise ValueError(f"Could not find sensor for code {code}") +# Maintain the import path for compatibility +from metloom.pointdata.snotel import SnotelVariables # noqa: F401 class CdecStationVariables(VariableBase): @@ -93,111 +30,6 @@ class CdecStationVariables(VariableBase): WINDDIR = SensorDescription("10", "WIND DIRECTION", "WIND DIRECTION") -# Available sensors from Snotel -SnotelVariables = make_dataclass( - "SnotelVariables", - [ - ( - "SNOWDEPTH", - SensorDescription, - field(default=SensorDescription("SNWD", "SNOWDEPTH")), - ), - ("SWE", SensorDescription, field(default=SensorDescription("WTEQ", "SWE"))), - ( - "TEMP", - SensorDescription, - field(default=SensorDescription("TOBS", "AIR TEMP")), - ), - ( - "TEMPAVG", - SensorDescription, - field(default=SensorDescription("TAVG", "AVG AIR TEMP", "AIR TEMPERATURE AVERAGE")), - ), - ( - "TEMPMIN", - SensorDescription, - field(default=SensorDescription("TMIN", "MIN AIR TEMP", "AIR TEMPERATURE MINIMUM")), - ), - ( - "TEMPMAX", - SensorDescription, - field(default=SensorDescription("TMAX", "MAX AIR TEMP", "AIR TEMPERATURE MAXIMUM")), - ), - ( - "PRECIPITATION", - SensorDescription, - field(default=SensorDescription("PRCPSA", "PRECIPITATION", "PRECIPITATION INCREMENT SNOW-ADJUSTED")), - ), - ( - "PRECIPITATIONACCUM", - SensorDescription, - field(default=SensorDescription("PREC", "ACCUMULATED PRECIPITATION", "PRECIPITATION ACCUMULATION")), - ), - # TODO for the SCAN network this appears to be "RHUM", we may need a new class - ( - "RH", - SensorDescription, - field(default=SensorDescription("RHUMV", "Relative Humidity", "RELATIVE HUMIDITY")), - ), - ( - "STREAMVOLUMEOBS", - SensorDescription, - field(default=SensorDescription("SRVO", "STREAM VOLUME OBS", "STREAM VOLUME OBS")), - ), - ( - "STREAMVOLUMEADJ", - SensorDescription, - field(default=SensorDescription("SRVOX", "STREAM VOLUME ADJ", "STREAM VOLUME ADJ")), - ), - ] - + [ - ( - f"TEMPGROUND{abs(d)}IN", - SensorDescription, - field( - default=SensorDescription( - "STO", - f"GROUND TEMPERATURE -{d}IN", - f"GROUND TEMPERATURE OBS -{d}IN", - extra={"height_depth": {"value": -d, "unitCd": "in"}}, - ) - ), - ) - for d in [2, 4, 8, 20] - ] - + [ - ( - f"SOILMOISTURE{abs(d)}IN", - SensorDescription, - field( - default=SensorDescription( - "SMS", - f"SOIL MOISTURE -{d}IN", - f"SOIL MOISTURE PERCENT -{d}IN", - extra={"height_depth": {"value": -d, "unitCd": "in"}}, - ) - ), - ) - for d in [2, 4, 8, 20] - ] - + [ - ( - f"TEMPPROFILENEG{abs(d)}IN" if d < 0 else f"TEMPPROFILE{d}IN", - SensorDescription, - field( - default=SensorDescription( - "PTEMP", - f"PROFILE TEMPERATURE {d}IN", - f"PROFILE TEMPERATURE OBS{d}IN", - extra={"height_depth": {"value": d, "unitCd": "in"}}, - ) - ), - ) - for d in [-8, 0, 8, 16, 24, 31, 39, 47, 55, 63, 71, 79, 87, 94, 102, 110, 118, 126] # noqa: E501 - ], - bases=(VariableBase,), -) - class MesowestVariables(VariableBase): """ From 0e0029721a0d4e16942540a77819da6628167ebc Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 3 Jul 2025 16:52:10 -0600 Subject: [PATCH 03/18] excited to drop zeep --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 4e4c955..763cf50 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,6 @@ 'lxml>=5.4.0,<6.0.0', 'requests>2.0.0,<3.0.0', 'beautifulsoup4>4,<5', - 'zeep>4.0.0', 'pydash>=8.0.0,<9.0.0', ] From 9c6d47a8c2fa16440960deb652701b19dd02a347 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Mon, 7 Jul 2025 12:31:13 -0600 Subject: [PATCH 04/18] snotel metadata --- metloom/pointdata/snotel/snotel.py | 34 ++++++++--------- metloom/pointdata/snotel/snotel_client.py | 36 ------------------ tests/test_snotel.py | 45 +++++++---------------- 3 files changed, 29 insertions(+), 86 deletions(-) diff --git a/metloom/pointdata/snotel/snotel.py b/metloom/pointdata/snotel/snotel.py index 43f2dd9..c2a92d3 100644 --- a/metloom/pointdata/snotel/snotel.py +++ b/metloom/pointdata/snotel/snotel.py @@ -3,7 +3,7 @@ import logging import geopandas as gpd import pandas as pd -from functools import reduce +from functools import reduce, cached_property import requests from metloom.pointdata.base import PointData @@ -11,9 +11,7 @@ from metloom.dataframe_utils import append_df, merge_df from .snotel_client import ( - DailySnotelDataClient, MetaDataSnotelClient, HourlySnotelDataClient, - SemiMonthlySnotelClient, PointSearchSnotelClient, SeriesSnotelClient, - ElementSnotelClient + PointSearchSnotelClient ) LOG = logging.getLogger("metloom.pointdata.snotel") @@ -213,29 +211,27 @@ def get_snow_course_data( start_date, end_date, variables, "SEMIMONTHLY", include_measurement_date=True ) - def _get_all_metadata(self): + @cached_property + def _all_metadata(self): """ Set _raw_metadata once using Snotel API """ - if self._raw_metadata is None: - client = MetaDataSnotelClient(station_triplet=self.id) - self._raw_metadata = client.get_data() - return self._raw_metadata - - def _get_all_elements(self): - """ - Set _raw_metadata once using Snotel API - """ - if self._raw_elements is None: - client = ElementSnotelClient(station_triplet=self.id) - self._raw_elements = client.get_data() - return self._raw_elements + endpoint_url = self.API_URL + "services/v1/stations" + params = dict( + stationTriplets=self.id, + ) + result = requests.get( + endpoint_url, params=params + ) + result.raise_for_status() + data = result.json() + return data def _get_metadata(self): """ See docstring for PointData._get_metadata """ - all_metadata = self._get_all_metadata() + all_metadata = self._all_metadata if isinstance(all_metadata, list): data = all_metadata[0] else: diff --git a/metloom/pointdata/snotel/snotel_client.py b/metloom/pointdata/snotel/snotel_client.py index 26cfdd2..b052ced 100644 --- a/metloom/pointdata/snotel/snotel_client.py +++ b/metloom/pointdata/snotel/snotel_client.py @@ -85,42 +85,6 @@ def get_data(self): return data -class MetaDataSnotelClient(BaseSnotelClient): - """ - Read metadata from the metadata service for a particular station triplet - """ - SERVICE_NAME = "getStationMetadata" - - def __init__(self, station_triplet: str, **kwargs): - super(MetaDataSnotelClient, self).__init__( - station_triplet=station_triplet, **kwargs - ) - - def get_data(self): - """ - Returns a dictionary of metadata values - """ - data = self._make_request(**self.params) - # change ordered dict of values to regular dict - return dict(data.__values__) - - -class ElementSnotelClient(BaseSnotelClient): - """ - Get all station elements for a station triplet. Station triplets - are descriptions of each sensor on the station - - get_data returns a list of zeep objects. Zeep objects are indexible - or attributes can be accessed with getattr or ``.`` - """ - SERVICE_NAME = "getStationElements" - - def __init__(self, station_triplet: str, **kwargs): - super(ElementSnotelClient, self).__init__( - station_triplet=station_triplet, **kwargs - ) - - class PointSearchSnotelClient(BaseSnotelClient): """ Search for stations based on criteria. This search is default logical diff --git a/tests/test_snotel.py b/tests/test_snotel.py index 2de41a9..c07b832 100644 --- a/tests/test_snotel.py +++ b/tests/test_snotel.py @@ -44,42 +44,24 @@ def snotel_meta_sideeffect(*args, **kwargs): code = kwargs["stationTriplet"] available_stations = { "538:CO:SNTL": { - "actonId": "07M27S", - "beginDate": "1979-10-01 00:00:00", - "countyName": "Ouray", - "elevation": 9800.0, - "endDate": "2100-01-01 00:00:00", - "fipsCountryCd": "US", - "fipsCountyCd": "091", - "fipsStateNumber": "08", - "huc": "140200060201", - "hud": "14020006", - "latitude": 37.9339, - "longitude": -107.67552, - "name": "Idarado", - "shefId": "IDRC2", "stationTriplet": "538:CO:SNTL", - "stationDataTimeZone": -8.0 - }, - "538:CO:SNOW": { - "actonId": "07M27S", - "beginDate": "1979-10-01 00:00:00", + "stationId": "538", + "stateCode": "CO", + "networkCode": "SNTL", + "name": "Idarado", + "dcoCode": "CO", "countyName": "Ouray", - "elevation": 9800.0, - "endDate": "2100-01-01 00:00:00", - "fipsCountryCd": "US", - "fipsCountyCd": "091", - "fipsStateNumber": "08", "huc": "140200060201", - "hud": "14020006", - "latitude": 37.9339, - "longitude": -107.67552, - "name": "Idarado", + "elevation": 9780, + "latitude": 37.93389, + "longitude": -107.6762, + "dataTimeZone": -8, "shefId": "IDRC2", - "stationTriplet": "538:CO:SNOW", + "operator": "NRCS", + "beginDate": "1979-10-01 00:00", + "endDate": "2100-01-01 00:00" }, "FFF:CA:SNOW": { - "actonId": None, "beginDate": "1930-02-01 00:00:00", "countyName": "Tuolumne", "elevation": 6500.0, @@ -92,10 +74,10 @@ def snotel_meta_sideeffect(*args, **kwargs): "longitude": -119.78, "name": "Fake1", "shefId": None, + "dataTimeZone": -8, "stationTriplet": "FFF:CA:SNOW", }, "BBB:CA:SNOW": { - "actonId": None, "beginDate": "1948-02-01 00:00:00", "countyName": "Tuolumne", "elevation": 9300.0, @@ -109,6 +91,7 @@ def snotel_meta_sideeffect(*args, **kwargs): "longitude": -119.61667, "name": "Fake2", "shefId": None, + "dataTimeZone": -8, "stationTriplet": "BBB:CA:SNOW", }, } From 380ac62874dd9d49579eded3308f3a459ca0d6e0 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Mon, 7 Jul 2025 12:32:44 -0600 Subject: [PATCH 05/18] timezone --- metloom/pointdata/snotel/snotel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metloom/pointdata/snotel/snotel.py b/metloom/pointdata/snotel/snotel.py index c2a92d3..c609812 100644 --- a/metloom/pointdata/snotel/snotel.py +++ b/metloom/pointdata/snotel/snotel.py @@ -244,9 +244,9 @@ def _get_tzinfo(self): """ Return timezone info that pandas can use from the raw_metadata """ - metadata = self._get_all_metadata() + metadata = self._all_metadata # Snow courses might not have a timezone attached - tz_hours = metadata.get("stationDataTimeZone") + tz_hours = metadata.get("dataTimeZone") if tz_hours is None: LOG.error(f"Could not find timezone info for {self.id} ({self.name})") tz_hours = 0 From 2a00ad9b9fe92674a1f4f24915ba768364934d29 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Mon, 7 Jul 2025 12:45:36 -0600 Subject: [PATCH 06/18] Use of metadata in station search --- metloom/pointdata/snotel/snotel.py | 35 +++++++++++++++++++----------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/metloom/pointdata/snotel/snotel.py b/metloom/pointdata/snotel/snotel.py index c609812..b137b24 100644 --- a/metloom/pointdata/snotel/snotel.py +++ b/metloom/pointdata/snotel/snotel.py @@ -211,14 +211,18 @@ def get_snow_course_data( start_date, end_date, variables, "SEMIMONTHLY", include_measurement_date=True ) - @cached_property - def _all_metadata(self): + @classmethod + def _metadata_call(cls, point_ids): """ - Set _raw_metadata once using Snotel API + Call the Snotel API to get metadata for a point_id + Args: + point_ids: string of comma separated station triplets + Returns: + dict: metadata for the point """ - endpoint_url = self.API_URL + "services/v1/stations" + endpoint_url = cls.API_URL + "services/v1/stations" params = dict( - stationTriplets=self.id, + stationTriplets=point_ids, ) result = requests.get( endpoint_url, params=params @@ -227,6 +231,13 @@ def _all_metadata(self): data = result.json() return data + @cached_property + def _all_metadata(self): + """ + Set _raw_metadata once using Snotel API + """ + return self._metadata_call(self.id) + def _get_metadata(self): """ See docstring for PointData._get_metadata @@ -316,15 +327,13 @@ def points_from_geometry( # no duplicate codes point_codes = list(set(point_codes)) - dfs = [ - pd.DataFrame.from_records( - [MetaDataSnotelClient(station_triplet=code).get_data()] - ).set_index("stationTriplet") for code in point_codes - ] + codes_string = ",".join(point_codes) + df = pd.DataFrame.from_records( + cls._metadata_call(codes_string) + ).set_index("stationTriplet") - if len(dfs) > 0: - df = reduce(lambda a, b: append_df(a, b), dfs) - else: + if len(df) == 0: + # Short circuit, return empty class return cls.ITERATOR_CLASS([]) df.reset_index(inplace=True) From e9c4bc200994a3f86358a3e876e4f2f7a3a4ed5d Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Mon, 7 Jul 2025 17:43:17 -0600 Subject: [PATCH 07/18] test finding stations updated --- tests/test_snotel.py | 102 ++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 65 deletions(-) diff --git a/tests/test_snotel.py b/tests/test_snotel.py index c07b832..4507b3b 100644 --- a/tests/test_snotel.py +++ b/tests/test_snotel.py @@ -36,12 +36,26 @@ class TestSnotelPointData(BasePointDataTest): def points(self): return gpd.points_from_xy([-107.67552], [37.9339], z=[9800.0])[0] + + @classmethod + def side_effect(cls, *args, **kwargs): + url = args[0] + if "services/v1/stations" in url: + result = cls.snotel_meta_sideeffect(*args, **kwargs) + elif "services/v1/data" in url: + result = cls.snotel_data_sideeffect(*args, **kwargs) + else: + raise ValueError("Unknown URL in mock: " + url) + obj = MagicMock() + obj.json.return_value = result + return obj + @staticmethod def snotel_meta_sideeffect(*args, **kwargs): """ Mock out the metadata response """ - code = kwargs["stationTriplet"] + codes = kwargs["params"]["stationTriplets"].split(",") available_stations = { "538:CO:SNTL": { "stationTriplet": "538:CO:SNTL", @@ -95,7 +109,7 @@ def snotel_meta_sideeffect(*args, **kwargs): "stationTriplet": "BBB:CA:SNOW", }, } - return MockZeepObject(available_stations[code]) + return [available_stations[code] for code in codes] @staticmethod def snotel_data_sideeffect(*args, **kwargs): @@ -183,71 +197,29 @@ def snotel_hourly_sideeffect(*args, **kwargs): else: raise ValueError(f"{element_cd} not configured in this mock") - @pytest.fixture(scope="class") - def mock_elements(self): - return [ - MockZeepObject( - { - 'beginDate': '1980-07-23 00:00:00', 'dataPrecision': 1, - 'duration': 'DAILY', 'elementCd': 'WTEQ', - 'endDate': '2100-01-01 00:00:00', 'heightDepth': None, - 'ordinal': 1, - 'originalUnitCd': 'in', 'stationTriplet': '538:CO:SNTL', - 'storedUnitCd': 'in'}), - MockZeepObject( - {'beginDate': '1980-07-23 00:00:00', 'dataPrecision': 1, - 'duration': 'SEMIMONTHLY', 'elementCd': 'WTEQ', - 'endDate': '2100-01-01 00:00:00', 'heightDepth': None, - 'ordinal': 1, - 'originalUnitCd': 'in', 'stationTriplet': '538:CO:SNTL', - 'storedUnitCd': 'in'}), - MockZeepObject( - {'beginDate': '1979-10-01 00:00:00', 'dataPrecision': 1, - 'duration': 'HOURLY', 'elementCd': 'WTEQ', - 'endDate': '2100-01-01 00:00:00', 'heightDepth': None, - 'ordinal': 1, - 'originalUnitCd': 'in', 'stationTriplet': '538:CO:SNTL', - 'storedUnitCd': 'in'}), - MockZeepObject( - {'beginDate': '1980-07-23 00:00:00', 'dataPrecision': 1, - 'duration': 'MONTHLY', 'elementCd': 'WTEQ', - 'endDate': '2100-01-01 00:00:00', 'heightDepth': None, - 'ordinal': 1, - 'originalUnitCd': 'in', 'stationTriplet': '538:CO:SNTL', - 'storedUnitCd': 'in'}), - MockZeepObject( - {'beginDate': '1979-10-01 00:00:00', 'dataPrecision': 1, - 'duration': 'HOURLY', 'elementCd': 'PRCPSA', - 'endDate': '2100-01-01 00:00:00', 'heightDepth': None, - 'ordinal': 1, - 'originalUnitCd': 'in', 'stationTriplet': '538:CO:SNTL', - 'storedUnitCd': 'in'}), - MockZeepObject( - {'beginDate': '1979-10-01 00:00:00', 'dataPrecision': 1, - 'duration': 'HOURLY', 'elementCd': 'STO', - 'endDate': '2100-01-01 00:00:00', - 'heightDepth': {'unitCd': 'in', 'value': '-2'}, - 'ordinal': 1, - 'originalUnitCd': 'degF', 'stationTriplet': '538:CO:SNTL', - 'storedUnitCd': 'degF'}) - ] - @pytest.fixture - def mock_zeep_client(self, mock_elements): - with patch("metloom.pointdata.snotel_client.zeep.Client") as mock_client: + def mock_requests(self): + with patch("requests.get") as mock_get: + # Mock our gets + mock_get.side_effect = self.side_effect + # mock our zeep request + # TODO: mock request return + yield mock_get + + @pytest.fixture(scope="class") + def mock_zeep_find(self): + with patch( + "metloom.pointdata.snotel.snotel_client.zeep.Client" + ) as mock_client: mock_service = MagicMock() # setup the individual services - mock_service.getStationMetadata.side_effect = self.snotel_meta_sideeffect - mock_service.getStationElements.return_value = mock_elements mock_service.getStations.return_value = ["FFF:CA:SNOW", "BBB:CA:SNOW"] - mock_service.getData.side_effect = self.snotel_data_sideeffect - mock_service.getHourlyData.side_effect = self.snotel_hourly_sideeffect # assign service to client mock_client.return_value.service = mock_service yield mock_client - def test_metadata(self, mock_zeep_client): + def test_metadata(self, mock_requests): obj = SnotelPointData("538:CO:SNTL", "eh") assert ( obj.metadata == gpd.points_from_xy( @@ -311,7 +283,7 @@ def test_metadata(self, mock_zeep_client): ) def test_get_data_methods( self, station_id, dts, expected_dts, vals, d1, - d2, fn_name, points, mock_zeep_client): + d2, fn_name, points, mock_requests): station = SnotelPointData(station_id, "TestSite") if 'GROUND TEMPERATURE -2IN' in list(vals.keys()): vrs = [SnotelVariables.TEMPGROUND2IN] @@ -327,7 +299,7 @@ def test_get_data_methods( result.sort_index(axis=1), expected ) - def test_get_hourly_data_multi_sensor(self, points, mock_zeep_client): + def test_get_hourly_data_multi_sensor(self, points, mock_requests): expected_dts = [ "2020-03-20 08:00", "2020-03-20 09:00", "2020-03-20 10:00", "2020-03-20 11:00" @@ -351,7 +323,7 @@ def test_get_hourly_data_multi_sensor(self, points, mock_zeep_client): result.sort_index(axis=1), expected ) - def test_points_from_geometry(self, shape_obj, mock_zeep_client): + def test_points_from_geometry(self, shape_obj, mock_requests, mock_zeep_find): result = SnotelPointData.points_from_geometry( shape_obj, [SnotelVariables.SWE], snow_courses=True ) @@ -361,11 +333,11 @@ def test_points_from_geometry(self, shape_obj, mock_zeep_client): assert set(ids) == {"FFF:CA:SNOW", "BBB:CA:SNOW"} assert set(names) == {"Fake1", "Fake2"} - def test_points_from_geomtery_buffer(self, shape_obj, mock_zeep_client): + def test_points_from_geomtery_buffer(self, shape_obj, mock_requests): SnotelPointData.points_from_geometry( shape_obj, [SnotelVariables.SWE], snow_courses=False, buffer=0.1 ) - search_kwargs = mock_zeep_client().method_calls[0][2] + search_kwargs = mock_requests().method_calls[0][2] expected = { 'maxLatitude': 38.3, 'minLatitude': 37.6, 'maxLongitude': -119.1, 'minLongitude': -119.9 @@ -373,8 +345,8 @@ def test_points_from_geomtery_buffer(self, shape_obj, mock_zeep_client): for k, v in expected.items(): assert v == pytest.approx(search_kwargs[k]) - def test_points_from_geometry_fail(self, shape_obj, mock_zeep_client): - mock_zeep_client.return_value.service.getStations.return_value = [] + def test_points_from_geometry_fail(self, shape_obj, mock_requests): + mock_requests.return_value.service.getStations.return_value = [] result = SnotelPointData.points_from_geometry( shape_obj, [SnotelVariables.SWE], snow_courses=True ) From 8836f4b90c525a4b9c0581ec20eb3a7699bc75bb Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Mon, 7 Jul 2025 18:54:28 -0600 Subject: [PATCH 08/18] working on tests --- metloom/pointdata/snotel/snotel.py | 7 +- tests/data/snotel_mocks/daily.json | 34 +++++ tests/data/snotel_mocks/hourly_precip.json | 34 +++++ tests/data/snotel_mocks/hourly_soil.json | 34 +++++ tests/data/snotel_mocks/hourly_swe.json | 34 +++++ tests/data/snotel_mocks/semimonthly_swe.json | 50 ++++++ tests/test_snotel.py | 153 ++++++------------- tests/utils.py | 7 + 8 files changed, 244 insertions(+), 109 deletions(-) create mode 100644 tests/data/snotel_mocks/daily.json create mode 100644 tests/data/snotel_mocks/hourly_precip.json create mode 100644 tests/data/snotel_mocks/hourly_soil.json create mode 100644 tests/data/snotel_mocks/hourly_swe.json create mode 100644 tests/data/snotel_mocks/semimonthly_swe.json create mode 100644 tests/utils.py diff --git a/metloom/pointdata/snotel/snotel.py b/metloom/pointdata/snotel/snotel.py index b137b24..cd21677 100644 --- a/metloom/pointdata/snotel/snotel.py +++ b/metloom/pointdata/snotel/snotel.py @@ -63,14 +63,13 @@ def _snotel_response_to_df( if include_measurement_date: final_columns += ["measurementDate"] - date_key = "datetime" if duration == "HOURLY" else "date" + date_key = "collectionDate" if duration == "SEMIMONTHLY" else "date" for variable, info in result_map.items(): data = info["values"] element = info["stationElement"] unit_name = element["storedUnitCode"] transformed = [] for row in data: - row_obj = { "datetime": row[date_key], "site": self.id, @@ -78,7 +77,7 @@ def _snotel_response_to_df( f"{variable.name}_units": unit_name, } if include_measurement_date: - row_obj["measurementDate"] = row["datetime"] + row_obj["measurementDate"] = row[date_key] transformed.append(row_obj) final_columns += [variable.name, f"{variable.name}_units"] @@ -257,7 +256,7 @@ def _get_tzinfo(self): """ metadata = self._all_metadata # Snow courses might not have a timezone attached - tz_hours = metadata.get("dataTimeZone") + tz_hours = metadata[0].get("dataTimeZone") if tz_hours is None: LOG.error(f"Could not find timezone info for {self.id} ({self.name})") tz_hours = 0 diff --git a/tests/data/snotel_mocks/daily.json b/tests/data/snotel_mocks/daily.json new file mode 100644 index 0000000..419e798 --- /dev/null +++ b/tests/data/snotel_mocks/daily.json @@ -0,0 +1,34 @@ +[ + { + "stationTriplet": "538:CO:SNTL", + "data": [ + { + "stationElement": { + "elementCode": "WTEQ", + "ordinal": 1, + "durationName": "DAILY", + "dataPrecision": 1, + "storedUnitCode": "in", + "originalUnitCode": "in", + "beginDate": "1980-07-23 00:00", + "endDate": "2100-01-01 00:00", + "derivedData": false + }, + "values": [ + { + "date": "2020-03-20", + "value": 11.6 + }, + { + "date": "2020-03-21", + "value": 11.6 + }, + { + "date": "2020-03-22", + "value": 11.8 + } + ] + } + ] +} +] diff --git a/tests/data/snotel_mocks/hourly_precip.json b/tests/data/snotel_mocks/hourly_precip.json new file mode 100644 index 0000000..d77ce4b --- /dev/null +++ b/tests/data/snotel_mocks/hourly_precip.json @@ -0,0 +1,34 @@ +[ + { + "stationTriplet": "538:CO:SNTL", + "data": [ + { + "stationElement": { + "elementCode": "PREC", + "ordinal": 1, + "durationName": "HOURLY", + "dataPrecision": 1, + "storedUnitCode": "in", + "originalUnitCode": "in", + "beginDate": "1985-10-01 00:00", + "endDate": "2100-01-01 00:00", + "derivedData": false + }, + "values": [ + { + "date": "2020-01-02 00:00", + "value": 6 + }, + { + "date": "2020-01-02 01:00", + "value": 6 + }, + { + "date": "2020-01-02 02:00", + "value": 6.1 + } + ] + } + ] + } +] diff --git a/tests/data/snotel_mocks/hourly_soil.json b/tests/data/snotel_mocks/hourly_soil.json new file mode 100644 index 0000000..2a767be --- /dev/null +++ b/tests/data/snotel_mocks/hourly_soil.json @@ -0,0 +1,34 @@ +[ + { + "stationTriplet": "538:CO:SNTL", + "data": [ + { + "stationElement": { + "elementCode": "STO", + "ordinal": 1, + "durationName": "HOURLY", + "dataPrecision": 1, + "storedUnitCode": "degF", + "originalUnitCode": "degF", + "beginDate": "1985-10-01 00:00", + "endDate": "2100-01-01 00:00", + "derivedData": false + }, + "values": [ + { + "date": "2020-01-02 00:00", + "value": -1.0 + }, + { + "date": "2020-01-02 01:00", + "value": -1.2 + }, + { + "date": "2020-01-02 02:00", + "value": -2.0 + } + ] + } + ] + } +] diff --git a/tests/data/snotel_mocks/hourly_swe.json b/tests/data/snotel_mocks/hourly_swe.json new file mode 100644 index 0000000..0a251bf --- /dev/null +++ b/tests/data/snotel_mocks/hourly_swe.json @@ -0,0 +1,34 @@ +[ + { + "stationTriplet": "538:CO:SNTL", + "data": [ + { + "stationElement": { + "elementCode": "WTEQ", + "ordinal": 1, + "durationName": "HOURLY", + "dataPrecision": 1, + "storedUnitCode": "in", + "originalUnitCode": "in", + "beginDate": "1979-10-01 00:00", + "endDate": "2100-01-01 00:00", + "derivedData": false + }, + "values": [ + { + "date": "2020-01-02 00:00", + "value": 6.9 + }, + { + "date": "2020-01-02 01:00", + "value": 6.9 + }, + { + "date": "2020-01-02 02:00", + "value": 6.8 + } + ] + } + ] + } +] diff --git a/tests/data/snotel_mocks/semimonthly_swe.json b/tests/data/snotel_mocks/semimonthly_swe.json new file mode 100644 index 0000000..dd76c6e --- /dev/null +++ b/tests/data/snotel_mocks/semimonthly_swe.json @@ -0,0 +1,50 @@ +[ + { + "stationTriplet": "658:CO:SNTL", + "data": [ + { + "stationElement": { + "elementCode": "WTEQ", + "ordinal": 1, + "durationName": "SEMIMONTHLY", + "dataPrecision": 1, + "storedUnitCode": "in", + "originalUnitCode": "in", + "beginDate": "1985-10-01 00:00", + "endDate": "2100-01-01 00:00", + "derivedData": false + }, + "values": [ + { + "month": 1, + "monthPart": "1", + "year": 2020, + "collectionDate": "2020-01-16 00:00", + "value": 6.4 + }, + { + "month": 1, + "monthPart": "2", + "year": 2020, + "collectionDate": "2020-02-01 00:00", + "value": 7.3 + }, + { + "month": 2, + "monthPart": "1", + "year": 2020, + "collectionDate": "2020-02-16 00:00", + "value": 8.8 + }, + { + "month": 2, + "monthPart": "2", + "year": 2020, + "collectionDate": "2020-03-01 00:00", + "value": 9.9 + } + ] + } + ] + } +] diff --git a/tests/test_snotel.py b/tests/test_snotel.py index 4507b3b..9d024aa 100644 --- a/tests/test_snotel.py +++ b/tests/test_snotel.py @@ -1,4 +1,5 @@ from datetime import timezone, timedelta, datetime +from pathlib import Path from unittest.mock import patch, MagicMock from collections import OrderedDict @@ -10,6 +11,7 @@ from metloom.pointdata import SnotelPointData from metloom.variables import SnotelVariables from tests.test_point_data import BasePointDataTest +from tests.utils import read_json class MockZeepObject: @@ -31,14 +33,17 @@ def __getitem__(self, item): class TestSnotelPointData(BasePointDataTest): + MOCKS_DIR = Path(__file__).parent.joinpath("data/snotel_mocks/").absolute() @pytest.fixture(scope="class") def points(self): - return gpd.points_from_xy([-107.67552], [37.9339], z=[9800.0])[0] - + return gpd.points_from_xy([-107.6762], [37.93389], z=[9800.0])[0] @classmethod def side_effect(cls, *args, **kwargs): + """ + All request side effects + """ url = args[0] if "services/v1/stations" in url: result = cls.snotel_meta_sideeffect(*args, **kwargs) @@ -66,7 +71,7 @@ def snotel_meta_sideeffect(*args, **kwargs): "dcoCode": "CO", "countyName": "Ouray", "huc": "140200060201", - "elevation": 9780, + "elevation": 9800, "latitude": 37.93389, "longitude": -107.6762, "dataTimeZone": -8, @@ -111,103 +116,41 @@ def snotel_meta_sideeffect(*args, **kwargs): } return [available_stations[code] for code in codes] - @staticmethod - def snotel_data_sideeffect(*args, **kwargs): - duration = kwargs["duration"] + @classmethod + def snotel_data_sideeffect(cls, *args, **kwargs): + duration = kwargs["params"]["duration"] + fname = None if duration == "SEMIMONTHLY": - return [ - MockZeepObject({ - 'beginDate': '2020-01-20 00:00:00', - 'collectionDates': ['2020-01-28', '2020-02-27'], - 'duration': 'SEMIMONTHLY', - 'endDate': '2020-03-14 00:00:00', - 'flags': ['V', 'V'], 'stationTriplet': '538:CO:SNTL', - 'values': [13.19, 13.17]}) - ] - if duration == "DAILY": - return [MockZeepObject({ - 'beginDate': '2020-03-20 00:00:00', - 'collectionDates': [], 'duration': 'DAILY', - 'endDate': '2020-03-22 00:00:00', 'flags': ['V', 'V', 'V'], - 'stationTriplet': '538:CO:SNTL', - 'values': [13.19, 13.17, 13.14]})] + fname = cls.MOCKS_DIR.joinpath("semimonthly_swe.json") + elif duration == "DAILY": + fname = cls.MOCKS_DIR.joinpath("daily.json") + elif duration == "HOURLY": + element_cd = kwargs["params"]["elements"] + if element_cd == "WTEQ": + fname = cls.MOCKS_DIR.joinpath("hourly_swe.json") + elif element_cd == "PRCPSA": + fname = cls.MOCKS_DIR.joinpath("hourly_precip.json") + elif element_cd == "STO:-2": + fname = cls.MOCKS_DIR.joinpath("hourly_soil.json") - @staticmethod - def snotel_hourly_sideeffect(*args, **kwargs): - element_cd = kwargs["elementCd"] - if element_cd == "WTEQ": - return [ - { - 'beginDate': '2020-01-02 00:00', 'endDate': '2020-01-20 00:00', - 'stationTriplet': '538:CO:SNTL', - 'values': [ - { - 'dateTime': '2020-03-20 00:00', - 'flag': 'V', - 'value': 13.19 - }, { - 'dateTime': '2020-03-20 01:00', - 'flag': 'V', - 'value': 13.17 - }, { - 'dateTime': '2020-03-20 02:00', - 'flag': 'V', - 'value': 13.14 - }]}] - elif element_cd == "PRCPSA": - return [ - { - 'beginDate': '2020-01-02 00:00', - 'endDate': '2020-01-20 00:00', - 'stationTriplet': '538:CO:SNTL', - 'values': [ - { - 'dateTime': '2020-03-20 00:00', - 'flag': 'V', - 'value': 4.1 - }, { - 'dateTime': '2020-03-20 02:00', - 'flag': 'V', - 'value': 4.3 - }, { - 'dateTime': '2020-03-20 03:00', - 'flag': 'V', - 'value': 4.4 - }]}] - elif element_cd == "STO": - return [ - { - 'beginDate': '2020-01-02 00:00', - 'endDate': '2020-01-20 00:00', - 'stationTriplet': '538:CO:SNTL', - 'values': [ - { - 'dateTime': '2020-03-20 00:00', - 'flag': 'V', - 'value': -0.3, - }, { - 'dateTime': '2020-03-20 01:00', - 'flag': 'V', - 'value': -0.4, - }, { - 'dateTime': '2020-03-20 02:00', - 'flag': 'V', - 'value': -0.5, - }]}] - else: - raise ValueError(f"{element_cd} not configured in this mock") + if fname is None: + raise ValueError("No mock file found for duration: " + duration) + + return read_json(fname) @pytest.fixture def mock_requests(self): with patch("requests.get") as mock_get: # Mock our gets mock_get.side_effect = self.side_effect - # mock our zeep request - # TODO: mock request return yield mock_get @pytest.fixture(scope="class") def mock_zeep_find(self): + """ + Mock the zeep client to return a mock service with + getStations method returning a list of station triplets. + """ with patch( "metloom.pointdata.snotel.snotel_client.zeep.Client" ) as mock_client: @@ -223,7 +166,7 @@ def test_metadata(self, mock_requests): obj = SnotelPointData("538:CO:SNTL", "eh") assert ( obj.metadata == gpd.points_from_xy( - [-107.67552], [37.9339], z=[9800.0])[0] + [-107.6762], [37.93389], z=[9800.0])[0] ) assert obj.tzinfo == timezone(timedelta(hours=-8.0)) @@ -232,27 +175,27 @@ def test_metadata(self, mock_requests): [ ( "538:CO:SNTL", - ["2020-03-20 00:00", "2020-03-20 01:00", "2020-03-20 02:00"], - ["2020-03-20 08:00", "2020-03-20 09:00", "2020-03-20 10:00"], + ["2020-01-02 00:00", "2020-01-02= 01:00", "2020-01-02 02:00"], + ["2020-01-02 08:00", "2020-01-02 09:00", "2020-01-02 10:00"], { - SnotelVariables.SWE.name: [13.19, 13.17, 13.14], + SnotelVariables.SWE.name: [6.9, 6.9, 6.8], f"{SnotelVariables.SWE.name}_units": ["in", "in", "in"] }, - datetime(2020, 3, 20, 0), - datetime(2020, 3, 20, 2), + datetime(2020, 1, 2, 0), + datetime(2020, 1, 2, 2), "get_hourly_data", ), ( "538:CO:SNTL", - ["2020-03-20 00:00", "2020-03-20 01:00", "2020-03-20 02:00"], - ["2020-03-20 08:00", "2020-03-20 09:00", "2020-03-20 10:00"], + ["2020-01-02 00:00", "2020-01-02 01:00", "2020-01-02 02:00"], + ["2020-01-02 08:00", "2020-01-02 09:00", "2020-01-02 10:00"], { - SnotelVariables.TEMPGROUND2IN.name: [-0.3, -0.4, -0.5], + SnotelVariables.TEMPGROUND2IN.name: [-1.0, -1.2, -2.0], f"{SnotelVariables.TEMPGROUND2IN.name}_units": ["degF", "degF", "degF"] }, - datetime(2020, 3, 20, 0), - datetime(2020, 3, 20, 2), + datetime(2020, 1, 2, 0), + datetime(2020, 1, 2, 2), "get_hourly_data", ), ( @@ -260,7 +203,7 @@ def test_metadata(self, mock_requests): ["2020-03-20", "2020-03-21", "2020-03-22"], ["2020-03-20 08:00", "2020-03-21 08:00", "2020-03-22 08:00"], { - SnotelVariables.SWE.name: [13.19, 13.17, 13.14], + SnotelVariables.SWE.name: [11.6, 11.6, 11.8], f"{SnotelVariables.SWE.name}_units": ["in", "in", "in"] }, datetime(2020, 3, 20), @@ -268,12 +211,12 @@ def test_metadata(self, mock_requests): "get_daily_data", ), ( - "538:CO:SNOW", - ["2020-01-28", "2020-02-27"], - ["2020-01-28 00:00", "2020-02-27 00:00"], + "538:CO:SNTL", + ["2020-01-16", "2020-02-01", "2020-02-16", "2020-02-27"], + ["2020-01-16 08:00", "2020-02-01 08:00", "2020-02-16 08:00", "2020-03-01 08:00"], { - SnotelVariables.SWE.name: [13.19, 13.17], - f"{SnotelVariables.SWE.name}_units": ["in", "in"] + SnotelVariables.SWE.name: [6.4, 7.3, 8.8, 9.9], + f"{SnotelVariables.SWE.name}_units": ["in", "in", "in", "in"] }, datetime(2020, 1, 20), datetime(2020, 3, 15), diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..12267a7 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,7 @@ +from pathlib import Path +import json + + +def read_json(file: Path): + with open(file, "r") as f: + return json.load(f) From 972e7e0f672baefa158187694653d8436d973779 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Mon, 7 Jul 2025 19:02:15 -0600 Subject: [PATCH 09/18] Closer to passing tests --- metloom/pointdata/snotel/snotel.py | 4 +++- tests/data/snotel_mocks/hourly_precip.json | 4 ++++ tests/test_snotel.py | 22 +++++++++++----------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/metloom/pointdata/snotel/snotel.py b/metloom/pointdata/snotel/snotel.py index cd21677..51e062e 100644 --- a/metloom/pointdata/snotel/snotel.py +++ b/metloom/pointdata/snotel/snotel.py @@ -329,12 +329,14 @@ def points_from_geometry( codes_string = ",".join(point_codes) df = pd.DataFrame.from_records( cls._metadata_call(codes_string) - ).set_index("stationTriplet") + ) if len(df) == 0: # Short circuit, return empty class return cls.ITERATOR_CLASS([]) + df = df.set_index("stationTriplet") + df.reset_index(inplace=True) gdf = gpd.GeoDataFrame( df, diff --git a/tests/data/snotel_mocks/hourly_precip.json b/tests/data/snotel_mocks/hourly_precip.json index d77ce4b..9f13479 100644 --- a/tests/data/snotel_mocks/hourly_precip.json +++ b/tests/data/snotel_mocks/hourly_precip.json @@ -26,6 +26,10 @@ { "date": "2020-01-02 02:00", "value": 6.1 + }, + { + "date": "2020-01-02 03:00", + "value": 6.5 } ] } diff --git a/tests/test_snotel.py b/tests/test_snotel.py index 9d024aa..b82cb7a 100644 --- a/tests/test_snotel.py +++ b/tests/test_snotel.py @@ -114,7 +114,7 @@ def snotel_meta_sideeffect(*args, **kwargs): "stationTriplet": "BBB:CA:SNOW", }, } - return [available_stations[code] for code in codes] + return [available_stations[code] for code in codes if code in available_stations] @classmethod def snotel_data_sideeffect(cls, *args, **kwargs): @@ -244,20 +244,20 @@ def test_get_data_methods( def test_get_hourly_data_multi_sensor(self, points, mock_requests): expected_dts = [ - "2020-03-20 08:00", "2020-03-20 09:00", "2020-03-20 10:00", - "2020-03-20 11:00" + "2020-01-02 08:00", "2020-01-02 09:00", "2020-01-02 10:00", + "2020-01-02 11:00" ] expected_vals_obj = { - SnotelVariables.SWE.name: [13.19, 13.17, 13.14, np.nan], + SnotelVariables.SWE.name: [6.9, 6.9, 6.8, np.nan], f"{SnotelVariables.SWE.name}_units": ["in", "in", "in", np.nan], - SnotelVariables.PRECIPITATION.name: [4.1, np.nan, 4.3, 4.4], + SnotelVariables.PRECIPITATION.name: [6, 6, 6.1, 6.5], f"{SnotelVariables.PRECIPITATION.name}_units": [ - "in", np.nan, "in", "in"], + "in", "in", "in", "in"], } station = SnotelPointData("538:CO:SNTL", "TestSite") vrs = [SnotelVariables.PRECIPITATION, SnotelVariables.SWE] result = station.get_hourly_data( - datetime(2020, 3, 20, 0), datetime(2020, 3, 20, 4), vrs + datetime(2020, 1, 2, 0), datetime(2020, 1, 2, 4), vrs ) expected = self.expected_response( expected_dts, expected_vals_obj, station, points @@ -276,11 +276,11 @@ def test_points_from_geometry(self, shape_obj, mock_requests, mock_zeep_find): assert set(ids) == {"FFF:CA:SNOW", "BBB:CA:SNOW"} assert set(names) == {"Fake1", "Fake2"} - def test_points_from_geomtery_buffer(self, shape_obj, mock_requests): + def test_points_from_geomtery_buffer(self, shape_obj, mock_requests, mock_zeep_find): SnotelPointData.points_from_geometry( shape_obj, [SnotelVariables.SWE], snow_courses=False, buffer=0.1 ) - search_kwargs = mock_requests().method_calls[0][2] + search_kwargs = mock_zeep_find().method_calls[0][2] expected = { 'maxLatitude': 38.3, 'minLatitude': 37.6, 'maxLongitude': -119.1, 'minLongitude': -119.9 @@ -288,8 +288,8 @@ def test_points_from_geomtery_buffer(self, shape_obj, mock_requests): for k, v in expected.items(): assert v == pytest.approx(search_kwargs[k]) - def test_points_from_geometry_fail(self, shape_obj, mock_requests): - mock_requests.return_value.service.getStations.return_value = [] + def test_points_from_geometry_fail(self, shape_obj, mock_requests, mock_zeep_find): + mock_zeep_find.return_value.service.getStations.return_value = [] result = SnotelPointData.points_from_geometry( shape_obj, [SnotelVariables.SWE], snow_courses=True ) From a4aa1287a5efb506d9d0f05651ba60a6767c367a Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 8 Jul 2025 13:35:02 -0600 Subject: [PATCH 10/18] Fix tests --- tests/test_snotel.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/test_snotel.py b/tests/test_snotel.py index b82cb7a..8e4affe 100644 --- a/tests/test_snotel.py +++ b/tests/test_snotel.py @@ -126,12 +126,12 @@ def snotel_data_sideeffect(cls, *args, **kwargs): fname = cls.MOCKS_DIR.joinpath("daily.json") elif duration == "HOURLY": element_cd = kwargs["params"]["elements"] - if element_cd == "WTEQ": - fname = cls.MOCKS_DIR.joinpath("hourly_swe.json") - elif element_cd == "PRCPSA": - fname = cls.MOCKS_DIR.joinpath("hourly_precip.json") - elif element_cd == "STO:-2": - fname = cls.MOCKS_DIR.joinpath("hourly_soil.json") + cd_file_map = { + "WTEQ": "hourly_swe.json", + "PRCPSA": "hourly_precip.json", + "STO:-2": "hourly_soil.json", + } + fname = cls.MOCKS_DIR.joinpath(cd_file_map.get(element_cd)) if fname is None: raise ValueError("No mock file found for duration: " + duration) @@ -145,7 +145,7 @@ def mock_requests(self): mock_get.side_effect = self.side_effect yield mock_get - @pytest.fixture(scope="class") + @pytest.fixture def mock_zeep_find(self): """ Mock the zeep client to return a mock service with @@ -276,7 +276,9 @@ def test_points_from_geometry(self, shape_obj, mock_requests, mock_zeep_find): assert set(ids) == {"FFF:CA:SNOW", "BBB:CA:SNOW"} assert set(names) == {"Fake1", "Fake2"} - def test_points_from_geomtery_buffer(self, shape_obj, mock_requests, mock_zeep_find): + def test_points_from_geomtery_buffer( + self, shape_obj, mock_requests, mock_zeep_find + ): SnotelPointData.points_from_geometry( shape_obj, [SnotelVariables.SWE], snow_courses=False, buffer=0.1 ) From f8b65710c2ad663437d590f772b14d666d0aa54a Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 8 Jul 2025 13:38:04 -0600 Subject: [PATCH 11/18] got ahead of myself on zeep --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 763cf50..25114b3 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,8 @@ 'geopandas>=1.0.0,<2.0.0', 'pandas>=1.0.0,<3.0.0', 'lxml>=5.4.0,<6.0.0', - 'requests>2.0.0,<3.0.0', + 'requests>=2.32.4,<3.0.0', + 'zeep>4.0.0', 'beautifulsoup4>4,<5', 'pydash>=8.0.0,<9.0.0', ] From f9bb5c41fda72d3a3e03d88f8677792328e463d4 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 8 Jul 2025 13:41:51 -0600 Subject: [PATCH 12/18] Flake8 --- metloom/pointdata/snotel/__init__.py | 2 ++ metloom/pointdata/snotel/snotel.py | 4 ++-- metloom/pointdata/snotel/snotel_client.py | 1 - metloom/sensors.py | 1 - metloom/variables.py | 1 - tests/test_snotel.py | 4 ++-- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/metloom/pointdata/snotel/__init__.py b/metloom/pointdata/snotel/__init__.py index 0014c11..f807784 100644 --- a/metloom/pointdata/snotel/__init__.py +++ b/metloom/pointdata/snotel/__init__.py @@ -1,2 +1,4 @@ from .snotel import SnotelPointData from .variables import SnotelVariables + +__all__ = ["SnotelPointData", "SnotelVariables"] diff --git a/metloom/pointdata/snotel/snotel.py b/metloom/pointdata/snotel/snotel.py index 51e062e..80740e9 100644 --- a/metloom/pointdata/snotel/snotel.py +++ b/metloom/pointdata/snotel/snotel.py @@ -3,12 +3,12 @@ import logging import geopandas as gpd import pandas as pd -from functools import reduce, cached_property +from functools import cached_property import requests from metloom.pointdata.base import PointData from .variables import SnotelVariables, SensorDescription -from metloom.dataframe_utils import append_df, merge_df +from metloom.dataframe_utils import merge_df from .snotel_client import ( PointSearchSnotelClient diff --git a/metloom/pointdata/snotel/snotel_client.py b/metloom/pointdata/snotel/snotel_client.py index b052ced..b2131c8 100644 --- a/metloom/pointdata/snotel/snotel_client.py +++ b/metloom/pointdata/snotel/snotel_client.py @@ -1,5 +1,4 @@ from datetime import datetime -import pandas as pd import zeep from metloom.request_utils import no_ssl_verification diff --git a/metloom/sensors.py b/metloom/sensors.py index a3eb59d..fb0fe5d 100644 --- a/metloom/sensors.py +++ b/metloom/sensors.py @@ -65,4 +65,3 @@ def from_code(cls, code): cls._validate_sensor(v) return v raise ValueError(f"Could not find sensor for code {code}") - diff --git a/metloom/variables.py b/metloom/variables.py index dc2e813..d7600f1 100644 --- a/metloom/variables.py +++ b/metloom/variables.py @@ -30,7 +30,6 @@ class CdecStationVariables(VariableBase): WINDDIR = SensorDescription("10", "WIND DIRECTION", "WIND DIRECTION") - class MesowestVariables(VariableBase): """ Available sensors from Mesowest diff --git a/tests/test_snotel.py b/tests/test_snotel.py index 8e4affe..069e577 100644 --- a/tests/test_snotel.py +++ b/tests/test_snotel.py @@ -178,7 +178,7 @@ def test_metadata(self, mock_requests): ["2020-01-02 00:00", "2020-01-02= 01:00", "2020-01-02 02:00"], ["2020-01-02 08:00", "2020-01-02 09:00", "2020-01-02 10:00"], { - SnotelVariables.SWE.name: [6.9, 6.9, 6.8], + SnotelVariables.SWE.name: [6.9, 6.9, 6.8], f"{SnotelVariables.SWE.name}_units": ["in", "in", "in"] }, datetime(2020, 1, 2, 0), @@ -248,7 +248,7 @@ def test_get_hourly_data_multi_sensor(self, points, mock_requests): "2020-01-02 11:00" ] expected_vals_obj = { - SnotelVariables.SWE.name: [6.9, 6.9, 6.8, np.nan], + SnotelVariables.SWE.name: [6.9, 6.9, 6.8, np.nan], f"{SnotelVariables.SWE.name}_units": ["in", "in", "in", np.nan], SnotelVariables.PRECIPITATION.name: [6, 6, 6.1, 6.5], f"{SnotelVariables.PRECIPITATION.name}_units": [ From ec65ab712bbf2839c1f96dc59c61d882119d8183 Mon Sep 17 00:00:00 2001 From: Micah Sandusky <32111103+micah-prime@users.noreply.github.com> Date: Tue, 8 Jul 2025 13:43:49 -0600 Subject: [PATCH 13/18] Update tests/test_snotel.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/test_snotel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_snotel.py b/tests/test_snotel.py index 069e577..b163a1f 100644 --- a/tests/test_snotel.py +++ b/tests/test_snotel.py @@ -175,7 +175,7 @@ def test_metadata(self, mock_requests): [ ( "538:CO:SNTL", - ["2020-01-02 00:00", "2020-01-02= 01:00", "2020-01-02 02:00"], + ["2020-01-02 00:00", "2020-01-02 01:00", "2020-01-02 02:00"], ["2020-01-02 08:00", "2020-01-02 09:00", "2020-01-02 10:00"], { SnotelVariables.SWE.name: [6.9, 6.9, 6.8], From aadfd3cc81dfa61913d28484f77d54f6cf3e0603 Mon Sep 17 00:00:00 2001 From: Micah Sandusky <32111103+micah-prime@users.noreply.github.com> Date: Tue, 8 Jul 2025 13:44:07 -0600 Subject: [PATCH 14/18] Update tests/test_snotel.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/test_snotel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_snotel.py b/tests/test_snotel.py index b163a1f..3f1da98 100644 --- a/tests/test_snotel.py +++ b/tests/test_snotel.py @@ -276,7 +276,7 @@ def test_points_from_geometry(self, shape_obj, mock_requests, mock_zeep_find): assert set(ids) == {"FFF:CA:SNOW", "BBB:CA:SNOW"} assert set(names) == {"Fake1", "Fake2"} - def test_points_from_geomtery_buffer( + def test_points_from_geometry_buffer( self, shape_obj, mock_requests, mock_zeep_find ): SnotelPointData.points_from_geometry( From 543537f08cdba07cb475b51516d782ef6af8fcc5 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Tue, 8 Jul 2025 13:46:58 -0600 Subject: [PATCH 15/18] chatgpt catching bugs --- metloom/pointdata/snotel/snotel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metloom/pointdata/snotel/snotel.py b/metloom/pointdata/snotel/snotel.py index 80740e9..2be828d 100644 --- a/metloom/pointdata/snotel/snotel.py +++ b/metloom/pointdata/snotel/snotel.py @@ -161,7 +161,7 @@ def _fetch_data_for_variables( # if we wanted to. We would need to be careful of the meas height if len(data) == 1: result_map[variable] = data[0] - elif len(data["data"]) > 1: + elif len(data) > 1: raise RuntimeError("We received too many results") else: LOG.warning(f"No {variable.name} found for {self.name}") From 2a07c718813ca8231ede9aeade3c1103830acd7c Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Wed, 9 Jul 2025 09:08:47 -0600 Subject: [PATCH 16/18] No returns was causing issues for nrcs reader --- metloom/pointdata/snotel/snotel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metloom/pointdata/snotel/snotel.py b/metloom/pointdata/snotel/snotel.py index 2be828d..a319fbc 100644 --- a/metloom/pointdata/snotel/snotel.py +++ b/metloom/pointdata/snotel/snotel.py @@ -156,6 +156,8 @@ def _fetch_data_for_variables( result.raise_for_status() data = result.json() # Get the first station return, since we only requested one station + if len(data) == 0: + return None data = data[0]["data"] # TODO: this is where we could iterate through multiple variables # if we wanted to. We would need to be careful of the meas height From b71642b56d876d88ecba1abec51cb6c68ed43cc2 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 10 Jul 2025 15:43:16 -0600 Subject: [PATCH 17/18] Sometimes no value in return --- metloom/pointdata/snotel/snotel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metloom/pointdata/snotel/snotel.py b/metloom/pointdata/snotel/snotel.py index a319fbc..f5002a7 100644 --- a/metloom/pointdata/snotel/snotel.py +++ b/metloom/pointdata/snotel/snotel.py @@ -73,7 +73,7 @@ def _snotel_response_to_df( row_obj = { "datetime": row[date_key], "site": self.id, - variable.name: row["value"], + variable.name: row.get("value", np.nan), f"{variable.name}_units": unit_name, } if include_measurement_date: From 7f10a26eed992409c184a86ea2872aec502c08d2 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 10 Jul 2025 16:07:52 -0600 Subject: [PATCH 18/18] Need numpy import --- metloom/pointdata/snotel/snotel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metloom/pointdata/snotel/snotel.py b/metloom/pointdata/snotel/snotel.py index f5002a7..c3b8985 100644 --- a/metloom/pointdata/snotel/snotel.py +++ b/metloom/pointdata/snotel/snotel.py @@ -5,6 +5,7 @@ import pandas as pd from functools import cached_property import requests +import numpy as np from metloom.pointdata.base import PointData from .variables import SnotelVariables, SensorDescription