From 7aff3709e06976f9a5aa192f97798d23d57030a6 Mon Sep 17 00:00:00 2001 From: Binita Date: Wed, 2 Oct 2024 11:25:11 -0500 Subject: [PATCH 1/6] add Z at the end of datetime formats --- pyQuARC/code/constants.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyQuARC/code/constants.py b/pyQuARC/code/constants.py index e77c74fc..03f1c25d 100644 --- a/pyQuARC/code/constants.py +++ b/pyQuARC/code/constants.py @@ -79,10 +79,10 @@ CMR_URL = "https://cmr.earthdata.nasa.gov" DATE_FORMATS = [ - "%Y-%m-%dT%H:%M:%S.%f", # Year to microsecond - "%Y-%m-%dT%H:%M:%S", # Year to second - "%Y-%m-%dT%H:%M", # Year to minute - "%Y-%m-%dT%H", # Year to hour + "%Y-%m-%dT%H:%M:%S.%fZ", # Year to microsecond + "%Y-%m-%dT%H:%M:%SZ", # Year to second + "%Y-%m-%dT%H:%MZ", # Year to minute + "%Y-%m-%dT%HZ", # Year to hour "%Y-%m-%d", # Year to day "%Y-%m", # Year to month "%Y", # Year From d888f7166b9c7d1c21569de1e0ba8c52c0ca74f4 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Fri, 8 Aug 2025 14:37:31 -0500 Subject: [PATCH 2/6] Update error message --- pyQuARC/schemas/check_messages.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyQuARC/schemas/check_messages.json b/pyQuARC/schemas/check_messages.json index 0b8b38c8..a786e043 100644 --- a/pyQuARC/schemas/check_messages.json +++ b/pyQuARC/schemas/check_messages.json @@ -768,7 +768,7 @@ "remediation": "Recommend providing a unique name for each characteristic." }, "validate_beginning_datetime_against_granules": { - "failure": "The collection beginning date time `{}` is not consistent with the first granule's beginning date time `{}`.", + "failure": "The collection beginning date time `{}` is not consistent with the beginning date time in the metadata for the first granule `{}`.", "help": { "message": "", "url": "https://wiki.earthdata.nasa.gov/display/CMR/Temporal+Extent" @@ -776,7 +776,7 @@ "remediation": "Recommend updating the beginning date time to match the granule extent." }, "validate_ending_datetime_against_granules": { - "failure": "The collection ending date time `{}` is not consistent with the last granule's ending date time `{}`.", + "failure": "The collection ending date time `{}` is not consistent with the ending date time in the metadata for the last granule `{}`.", "help": { "message": "", "url": "https://wiki.earthdata.nasa.gov/display/CMR/Temporal+Extent" From 91ee31c9dd9bbf68fcee4207409ed10558ba6bed Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Fri, 15 Aug 2025 15:21:05 -0500 Subject: [PATCH 3/6] Update severity based on time difference and tests --- pyQuARC/code/checker.py | 2 +- pyQuARC/code/datetime_validator.py | 17 ++++++- tests/test_datetime_validator.py | 74 ++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/pyQuARC/code/checker.py b/pyQuARC/code/checker.py index 4bb401c7..a222bb8e 100644 --- a/pyQuARC/code/checker.py +++ b/pyQuARC/code/checker.py @@ -117,9 +117,9 @@ def build_message(self, result, rule_id): rule_mapping = self.rules_override.get(rule_id) or self.rule_mapping.get( rule_id ) - severity = rule_mapping.get("severity", "error") messages = [] if not (result["valid"]) and result.get("value"): + severity = result.get("severity") or rule_mapping.get("severity", "error") for value in result["value"]: formatted_message = failure_message value = value if isinstance(value, tuple) else (value,) diff --git a/pyQuARC/code/datetime_validator.py b/pyQuARC/code/datetime_validator.py index fd67e4ef..22b11dca 100644 --- a/pyQuARC/code/datetime_validator.py +++ b/pyQuARC/code/datetime_validator.py @@ -140,7 +140,6 @@ def validate_datetime_against_granules( "granules", ) granules = cmr_request(cmr_prms) - validity = True last_granule_datetime = None date_time = None @@ -152,8 +151,22 @@ def validate_datetime_against_granules( date_time = get_date_time(datetime_string) last_granule_datetime = get_date_time(last_granule_datetime) validity = date_time == last_granule_datetime + else: + validity = False + + return_value = {} + if ( + (not date_time) + or not last_granule_datetime + or ((abs(date_time - last_granule_datetime).total_seconds() / 3600) > 24) + ): + return_value["severity"] = "error" - return {"valid": validity, "value": (date_time, last_granule_datetime)} + return { + **return_value, + "valid": validity, + "value": (date_time, last_granule_datetime), + } @staticmethod @if_arg diff --git a/tests/test_datetime_validator.py b/tests/test_datetime_validator.py index 9f9c0262..0358f6d0 100644 --- a/tests/test_datetime_validator.py +++ b/tests/test_datetime_validator.py @@ -1,3 +1,7 @@ +import pytest +from unittest.mock import patch +from datetime import datetime + from pyQuARC.code.datetime_validator import DatetimeValidator from tests.fixtures.validator import INPUT_OUTPUT @@ -18,3 +22,73 @@ def test_datetime_iso_format_check(self): def test_datetime_compare(self): pass + + @patch("pyQuARC.code.datetime_validator.set_cmr_prms") + @patch("pyQuARC.code.datetime_validator.cmr_request") + @patch("pyQuARC.code.datetime_validator.get_date_time") + @pytest.mark.parametrize( + "datetime_string, granule_datetime, expected_valid, expected_severity", + [ + # Exact match → valid, no severity + ("2025-08-01T00:00:00Z", "2025-08-01T00:00:00Z", True, None), + + # Different date but within 24 hours → invalid, no severity + ("2025-08-02T00:00:00Z", "2025-08-01T12:00:00Z", False, None), + + # More than 24 hours difference → invalid, severity error + ("2025-08-03T00:00:00Z", "2025-08-01T00:00:00Z", False, "error"), + + # No granules returned → valid=False, severity error + ("2025-08-01T00:00:00Z", None, False, "error"), + ], + ) + def test_validate_datetime_against_granules( + self, + mock_get_date_time, + mock_cmr_request, + mock_set_cmr_prms, + datetime_string, + granule_datetime, + expected_valid, + expected_severity, + ): + # Arrange: cmr_request mock + if granule_datetime is None: + mock_cmr_request.return_value = {"feed": {"entry": []}} + else: + mock_cmr_request.return_value = { + "feed": { + "entry": [ + { + "time_start": granule_datetime, + "time_end": granule_datetime, + } + ] + } + } + + mock_set_cmr_prms.return_value = {"mock": "params"} + + # Mock get_date_time to return datetime objects or None + def fake_get_date_time(val): + if val is None: + return None + return datetime.strptime(val, "%Y-%m-%dT%H:%M:%SZ") + + mock_get_date_time.side_effect = fake_get_date_time + + # Act + result = DatetimeValidator.validate_datetime_against_granules( + datetime_string, + collection_shortname="TEST", + version="1", + sort_key="start_date", + time_key="time_start", + ) + + # Assert + assert result["valid"] == expected_valid + if expected_severity: + assert result["severity"] == expected_severity + else: + assert "severity" not in result From d85925c7ee88b89702cd1c0eef6a0b7322e01979 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Wed, 8 Oct 2025 10:09:51 -0500 Subject: [PATCH 4/6] Add severity to result --- pyQuARC/code/custom_checker.py | 3 +++ pyQuARC/code/datetime_validator.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pyQuARC/code/custom_checker.py b/pyQuARC/code/custom_checker.py index f38cedda..55f514d0 100644 --- a/pyQuARC/code/custom_checker.py +++ b/pyQuARC/code/custom_checker.py @@ -184,6 +184,7 @@ def run( for future in as_completed(future_results): try: func_return = future.result() + severity = func_return.get("severity") valid = func_return["valid"] # can be True, False or None if valid is not None: if valid: @@ -196,4 +197,6 @@ def run( raise e result["valid"] = validity result["value"] = invalid_values + if severity: + result["severity"] = severity return result diff --git a/pyQuARC/code/datetime_validator.py b/pyQuARC/code/datetime_validator.py index 22b11dca..b312d3fa 100644 --- a/pyQuARC/code/datetime_validator.py +++ b/pyQuARC/code/datetime_validator.py @@ -158,7 +158,7 @@ def validate_datetime_against_granules( if ( (not date_time) or not last_granule_datetime - or ((abs(date_time - last_granule_datetime).total_seconds() / 3600) > 24) + or abs((date_time - last_granule_datetime).total_seconds() / 3600) > 24 ): return_value["severity"] = "error" From ce05f46b833ec52aa3298dfc5c20034a8ea29c04 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Wed, 8 Oct 2025 13:35:15 -0500 Subject: [PATCH 5/6] Return datetime string instead of datetime instance --- pyQuARC/code/datetime_validator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyQuARC/code/datetime_validator.py b/pyQuARC/code/datetime_validator.py index b312d3fa..dfe907dc 100644 --- a/pyQuARC/code/datetime_validator.py +++ b/pyQuARC/code/datetime_validator.py @@ -147,9 +147,9 @@ def validate_datetime_against_granules( # Compare the precision of the two datetime strings if len(granules["feed"]["entry"]) > 0: last_granule = granules["feed"]["entry"][0] - last_granule_datetime = last_granule.get(time_key) + last_granule_datetime_string = last_granule.get(time_key) date_time = get_date_time(datetime_string) - last_granule_datetime = get_date_time(last_granule_datetime) + last_granule_datetime = get_date_time(last_granule_datetime_string) validity = date_time == last_granule_datetime else: validity = False @@ -165,7 +165,7 @@ def validate_datetime_against_granules( return { **return_value, "valid": validity, - "value": (date_time, last_granule_datetime), + "value": (datetime_string, last_granule_datetime_string), } @staticmethod From ffacfce4170aa171cc64f3000e6aa9a1605b1833 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Mon, 13 Oct 2025 13:55:49 -0500 Subject: [PATCH 6/6] Fix calling before assignment --- pyQuARC/code/datetime_validator.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyQuARC/code/datetime_validator.py b/pyQuARC/code/datetime_validator.py index dfe907dc..03da3faf 100644 --- a/pyQuARC/code/datetime_validator.py +++ b/pyQuARC/code/datetime_validator.py @@ -142,6 +142,7 @@ def validate_datetime_against_granules( granules = cmr_request(cmr_prms) validity = True last_granule_datetime = None + last_granule_datetime_string = None date_time = None # Compare the precision of the two datetime strings @@ -151,6 +152,9 @@ def validate_datetime_against_granules( date_time = get_date_time(datetime_string) last_granule_datetime = get_date_time(last_granule_datetime_string) validity = date_time == last_granule_datetime + diff_bigger_than_a_day = abs( + (date_time - last_granule_datetime).total_seconds() / 3600 + ) > 24 else: validity = False @@ -158,7 +162,7 @@ def validate_datetime_against_granules( if ( (not date_time) or not last_granule_datetime - or abs((date_time - last_granule_datetime).total_seconds() / 3600) > 24 + or diff_bigger_than_a_day ): return_value["severity"] = "error"