diff --git a/morpc/utils.py b/morpc/utils.py index 49225ac..fc20613 100644 --- a/morpc/utils.py +++ b/morpc/utils.py @@ -24,26 +24,81 @@ ] def datetime_from_string(date, errors: Literal['coerce', 'error']='coerce') -> datetime.datetime: + """ + Best-effort conversion of a wide range of date/datetime representations to a + timezone-naive ``datetime.datetime``. + + Handled inputs, in order of preference: + + - ``pandas.NaT`` / ``None`` / empty-ish strings (``''``, ``'nan'``, ``'None'``, + ``'NaT'``) -> ``pandas.NaT`` + - existing ``datetime.datetime`` / ``datetime.date`` objects + - numeric epochs, dispatched by digit count: 19 -> ns, 13 -> ms, 10 -> s + (works for ``int`` *and* ``float``) + - compact numeric dates: 8 digits -> ``YYYYMMDD``, 6 digits -> ``YYYYMM`` + (``int``, ``float``, or ``str``) + - ISO 8601 strings (validated/parsed by pandas) + - general date strings parsed month-first / US order (via ``dateutil``) + - natural-language, relative, and localized strings (via ``dateparser``, if + installed; otherwise this final fallback is skipped) + + Ambiguous numeric dates such as ``'10/2/2020'`` are interpreted month-first + (US): October 2, 2020. + + Timezone-aware inputs (e.g. ISO strings with an offset, or aware ``datetime`` + objects) are returned tz-naive, preserving the written wall-clock time, so + results never mix aware and naive values. + + Parameters + ---------- + date : Any + The value to convert. + errors : {'coerce', 'error'}, default 'coerce' + ``'coerce'`` returns ``pandas.NaT`` on failure; ``'error'`` re-raises. + + Returns + ------- + datetime.datetime or pandas.NaT + """ import datetime import math + import re import pandas as pd import dateutil.parser - import re + + def _from_number(value): + """Interpret a number as an epoch (by digit count) or compact YYYYMMDD/YYYYMM date.""" + n = int(value) + digits = len(str(abs(n))) + if digits == 19: + return pd.to_datetime(n, unit='ns').to_pydatetime() + if digits == 13: + return pd.to_datetime(n, unit='ms').to_pydatetime() + if digits == 10: + return pd.to_datetime(n, unit='s').to_pydatetime() + if digits == 8: + return datetime.datetime.strptime(str(n), '%Y%m%d') + if digits == 6: + return datetime.datetime.strptime(str(n), '%Y%m') + raise ValueError( + f"Numeric value {value!r} does not match a recognized epoch length " + f"(10, 13, or 19 digits) or compact date (8=YYYYMMDD, 6=YYYYMM)." + ) + + def _make_naive(value): + """Drop tzinfo (preserving wall-clock) so outputs are consistently tz-naive.""" + if isinstance(value, datetime.datetime) and value.tzinfo is not None: + return value.replace(tzinfo=None) + return value dt = pd.NaT try: - if isinstance(date, pd.api.typing.NaTType): - dt = pd.NaT - - elif date is None: + if isinstance(date, pd.api.typing.NaTType) or date is None: dt = pd.NaT elif isinstance(date, float): - if math.isnan(date): - dt = pd.NaT - else: - dt = pd.to_datetime(int(date), unit='s').to_pydatetime() + dt = pd.NaT if math.isnan(date) else _from_number(date) # datetime.datetime must be checked before datetime.date (it's a subclass) elif isinstance(date, datetime.datetime): @@ -53,14 +108,7 @@ def datetime_from_string(date, errors: Literal['coerce', 'error']='coerce') -> d dt = datetime.datetime.combine(date, datetime.time.min) elif isinstance(date, int): - if re.fullmatch(r'\d{19}', str(date)): - dt = pd.to_datetime(date, unit='ns').to_pydatetime() - elif re.fullmatch(r'\d{13}', str(date)): - dt = pd.to_datetime(date, unit='ms').to_pydatetime() - elif re.fullmatch(r'\d{10}', str(date)): - dt = pd.to_datetime(date, unit='s').to_pydatetime() - else: - raise ValueError(f"Integer {date} does not match a recognized epoch length (10, 13, or 19 digits).") + dt = _from_number(date) else: if not isinstance(date, str): @@ -70,48 +118,47 @@ def datetime_from_string(date, errors: Literal['coerce', 'error']='coerce') -> d logger.error(f"Failed to convert {date!r} to string.") raise e + date = date.strip() + if date in ('nan', 'None', 'NaT', ''): dt = pd.NaT - elif re.match( - r'^\d{4}-(?:0[1-9]|1[0-2])-(?:[0-2][1-9]|[1-3]0|3[01])' - r'T(?:[0-1][0-9]|2[0-3])(?::[0-6]\d)(?::[0-6]\d)?' - r'(?:\.\d{3})?(?:[+-][0-2]\d:[0-5]\d|Z)?$', - date - ): - try: - dt = pd.to_datetime(date, format='ISO8601').to_pydatetime() - except Exception as e: - logger.error(f"Maybe found ISO format in {date!r} but couldn't convert.") - raise e - - elif re.fullmatch(r'\d+[/\-\.]\d+[/\-\.]\d+', date): - try: - dt = dateutil.parser.parse(date) - except Exception as e: - logger.error(f"Maybe found date with separators in {date!r} but couldn't convert. {e}") - raise e - - elif re.fullmatch(r'\d{8}', date): - try: - dt = datetime.datetime.strptime(date, '%Y%m%d') - except Exception as e: - logger.error(f"Maybe found YYYYMMDD in {date!r} but couldn't convert. {e}") - raise e - - elif re.fullmatch(r'\d{6}', date): - try: - dt = datetime.datetime.strptime(date, '%Y%m') - except Exception as e: - logger.error(f"Maybe found YYYYMM in {date!r} but couldn't convert. {e}") - raise e + # Pure-digit strings share the numeric epoch / compact-date logic so that, + # e.g., '20210310' and 20210310 resolve identically. + elif re.fullmatch(r'\d{6}|\d{8}|\d{10}|\d{13}|\d{19}', date): + dt = _from_number(int(date)) else: + dt = None + + # 1) ISO 8601 (pandas validates and raises if the string isn't ISO). try: - dt = dateutil.parser.parse(date) - except Exception as e: - logger.error(f"Last ditch effort to convert {date!r} failed. {e}") - raise e + dt = pd.to_datetime(date, format='ISO8601').to_pydatetime() + except Exception: + dt = None + + # 2) General date strings, US month-first ordering. + if dt is None: + try: + dt = dateutil.parser.parse(date, dayfirst=False) + except Exception: + dt = None + + # 3) Natural-language / relative / localized strings. dateparser is an + # optional catch-all; if it isn't installed we simply skip it. + if dt is None: + try: + import dateparser + dt = dateparser.parse(date, settings={'DATE_ORDER': 'MDY'}) + except ImportError: + dt = None + except Exception: + dt = None + + if dt is None: + raise ValueError(f"Could not parse {date!r} as a date/datetime.") + + dt = _make_naive(dt) except Exception as e: if errors == 'error': diff --git a/pyproject.toml b/pyproject.toml index 870596a..062c820 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,8 @@ dependencies = [ "shapely", "IPython", "xlsxwriter", - "plotnine" + "plotnine", + "dateparser" ] [build-system] diff --git a/reference/dev_notes.md b/reference/dev_notes.md new file mode 100644 index 0000000..5d52d20 --- /dev/null +++ b/reference/dev_notes.md @@ -0,0 +1,27 @@ +# Dev Notes + +High-level, append-only notes on notable changes. Newest entries at the bottom. + +## 2026-07-07 — Improve `datetime_from_string` robustness and coverage + +Branch: `improve-datetime-parsing` + +Reworked `morpc.utils.datetime_from_string` to catch a wider range of date/datetime +string and numeric representations and to fix correctness bugs: + +- Numeric inputs (int **and** float) now dispatch by digit count + (19=ns, 13=ms, 10=s, 8=YYYYMMDD, 6=YYYYMM). Fixes the float bug where any float + was treated as Unix seconds (e.g. `20210310.0` -> 1970). `int`, `float`, and + all-digit `str` now resolve identically. +- Removed the hand-rolled ISO regex; pandas validates/parses ISO 8601 directly. +- Ambiguous dates (e.g. `10/2/2020`) are parsed month-first (US) via `dateutil`. +- Added `dateparser` as an optional final fallback for natural-language, relative, + and localized strings; skipped gracefully if not installed. +- Output normalized to tz-naive (wall-clock preserved) so results never mix + aware and naive datetimes. **Behavior change:** ISO-with-offset inputs previously + returned tz-aware; they now return naive. +- Added `dateparser` to `pyproject.toml` dependencies. + +Not execution-tested in the dev container (pandas/dateparser not installed there); +pure-Python digit/regex logic verified, file byte-compiles. Run `pytest` in a full +env to confirm.