From 5e992f41bc2e61ebd3c593890fde6888e973e0ea Mon Sep 17 00:00:00 2001 From: Steve Larson <9larsons@gmail.com> Date: Wed, 1 Jul 2026 10:07:07 -0500 Subject: [PATCH 1/2] Normalized absolute date-time values in filters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relative dates (e.g. `now-30d`) are already normalized to the stored "YYYY-MM-DD HH:mm:ss" UTC format at parse time, but absolute ISO values from a filter (e.g. `published_at:>'2025-02-27T19:03:00.000-05:00'`) were passed through untouched. On SQLite datetimes are stored as text and compared lexically, so the "T" sorts after the stored space separator and the comparison returns the wrong rows; on MySQL the timezone offset was dropped rather than applied. Normalize absolute date-time values to the same stored format in the `VALUE` grammar rule, next to the existing relative-date handling, so date comparisons behave identically across SQLite and MySQL. Only full ISO-8601 date-times (a date with a time component) are rewritten — bare dates and other plain strings are left untouched, since nql-lang has no column-type information and must not corrupt a legitimate non-date value. Fixes https://github.com/TryGhost/Ghost/issues/23441 --- packages/nql-lang/dist/parser.js | 4 +-- packages/nql-lang/lib/scope.js | 36 +++++++++++++++++++ packages/nql-lang/src/nql.y | 4 +-- packages/nql-lang/test/parser.test.js | 36 +++++++++++++++++++ packages/nql-lang/test/scope.test.js | 51 +++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 4 deletions(-) diff --git a/packages/nql-lang/dist/parser.js b/packages/nql-lang/dist/parser.js index c20cabc7..c5a4ae48 100644 --- a/packages/nql-lang/dist/parser.js +++ b/packages/nql-lang/dist/parser.js @@ -169,10 +169,10 @@ case 24: this.$ = yy.relDateToAbsolute($$[$0-2], $$[$0-1], $$[$0]) break; case 25: - this.$ = yy.unescape($$[$0]); + this.$ = yy.normalizeAbsoluteDate(yy.unescape($$[$0])); break; case 26: - $$[$0] = $$[$0].replace(/^'|'$/g, ''); this.$ = yy.unescape($$[$0]); + $$[$0] = $$[$0].replace(/^'|'$/g, ''); this.$ = yy.normalizeAbsoluteDate(yy.unescape($$[$0])); break; case 27: this.$ = "add"; diff --git a/packages/nql-lang/lib/scope.js b/packages/nql-lang/lib/scope.js index 01f8f35c..51440f7a 100644 --- a/packages/nql-lang/lib/scope.js +++ b/packages/nql-lang/lib/scope.js @@ -34,6 +34,15 @@ const formatDateForSQL = (date) => { return isoDate.replace('T', ' ').replace(/\.[0-9]{3}Z/, ''); }; +// A full ISO-8601 date-time: a date WITH a time component, optionally with +// fractional seconds and a timezone (Z or ±HH:MM). We deliberately require a +// time component so bare dates ("2025-02-27") and any other plain string are +// never rewritten — nql-lang has no column-type information, so this is the +// only shape we can safely normalize without risking a legitimate non-date +// value (e.g. a date-like slug). +// Groups: 1=date, 2=time (HH:mm[:ss]), 3=fraction, 4=zone. +const ISO_DATE_TIME = /^(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}(?::\d{2})?)(\.\d+)?(Z|[+-]\d{2}:?\d{2})?$/; + module.exports = { ungroup(value) { return value.yg ? value.yg : value; @@ -71,6 +80,33 @@ module.exports = { return formatDateForSQL(relDate); }, + // Normalizes an absolute date-time value to the format dates are stored in + // ("YYYY-MM-DD HH:mm:ss", UTC) — the same format `relDateToAbsolute` + // produces for relative dates. This makes date comparisons behave + // identically on SQLite (where datetimes are text compared lexically, so a + // raw ISO "T" sorts after the stored space separator) and MySQL (which + // otherwise drops the timezone offset instead of applying it). A value that + // isn't a full ISO-8601 date-time, or that fails to parse, is returned + // untouched. + normalizeAbsoluteDate(value) { + if (typeof value !== 'string') { + return value; + } + + const match = ISO_DATE_TIME.exec(value); + if (!match) { + return value; + } + + const [, date, time, fraction = '', zone] = match; + // A zone-less value is interpreted as UTC (dates are stored in UTC), so + // we append "Z" rather than letting `new Date` treat it as local time. + const isoString = `${date}T${time}${fraction}${zone || 'Z'}`; + const parsed = new Date(isoString); + + return Number.isNaN(parsed.getTime()) ? value : formatDateForSQL(parsed); + }, + debug() { if (!process.env.DEBUG || !/nql/.test(process.env.DEBUG)) { return; diff --git a/packages/nql-lang/src/nql.y b/packages/nql-lang/src/nql.y index c8bbe644..47d2b811 100644 --- a/packages/nql-lang/src/nql.y +++ b/packages/nql-lang/src/nql.y @@ -82,8 +82,8 @@ VALUE | FALSE { $$ = false } | NUMBER { $$ = parseInt(yytext); } | NOW DATEOP AMOUNT INTERVAL { $$ = yy.relDateToAbsolute($2, $3, $4) } - | LITERAL { $$ = yy.unescape($1); } - | STRING { $1 = $1.replace(/^'|'$/g, ''); $$ = yy.unescape($1); } + | LITERAL { $$ = yy.normalizeAbsoluteDate(yy.unescape($1)); } + | STRING { $1 = $1.replace(/^'|'$/g, ''); $$ = yy.normalizeAbsoluteDate(yy.unescape($1)); } ; DATEOP diff --git a/packages/nql-lang/test/parser.test.js b/packages/nql-lang/test/parser.test.js index 6fc4f089..381c439c 100644 --- a/packages/nql-lang/test/parser.test.js +++ b/packages/nql-lang/test/parser.test.js @@ -745,4 +745,40 @@ describe('Parser', function () { }); }); }); + + describe('Absolute dates', function () { + it('normalizes an ISO date-time with an offset to UTC db format', function () { + parse('published_at:>\'2025-02-27T19:03:00.000-05:00\'') + .should.eql({published_at: {$gt: '2025-02-28 00:03:00'}}); + }); + + it('normalizes a Zulu ISO date-time', function () { + parse('published_at:<\'2025-02-27T19:03:00Z\'') + .should.eql({published_at: {$lt: '2025-02-27 19:03:00'}}); + }); + + it('normalizes an equality date-time value', function () { + parse('created_at:\'2025-02-27T19:03:00.000Z\'') + .should.eql({created_at: '2025-02-27 19:03:00'}); + }); + + it('normalizes date-times inside an $in list', function () { + parse('published_at:[\'2025-02-27T19:03:00Z\',\'2025-03-01T00:00:00Z\']') + .should.eql({published_at: {$in: ['2025-02-27 19:03:00', '2025-03-01 00:00:00']}}); + }); + + it('normalizes date-times within a logical group', function () { + parse('featured:true+published_at:>\'2025-02-27T19:03:00-05:00\'') + .should.eql({$and: [{featured: true}, {published_at: {$gt: '2025-02-28 00:03:00'}}]}); + }); + + it('leaves a bare date untouched', function () { + parse('published_at:>\'2025-02-27\'') + .should.eql({published_at: {$gt: '2025-02-27'}}); + }); + + it('does not rewrite a non-date string value', function () { + parse('slug:\'2025-02-27\'').should.eql({slug: '2025-02-27'}); + }); + }); }); diff --git a/packages/nql-lang/test/scope.test.js b/packages/nql-lang/test/scope.test.js index a48a359f..1c93f4af 100644 --- a/packages/nql-lang/test/scope.test.js +++ b/packages/nql-lang/test/scope.test.js @@ -52,4 +52,55 @@ describe('Scope date helpers', function () { }); }); }); + + describe('normalizeAbsoluteDate', function () { + it('converts an ISO date-time with a timezone offset to UTC db format', function () { + scope.normalizeAbsoluteDate('2025-02-27T19:03:00.000-05:00') + .should.equal('2025-02-28 00:03:00'); + }); + + it('converts a Zulu ISO date-time to db format', function () { + scope.normalizeAbsoluteDate('2025-02-27T19:03:00Z') + .should.equal('2025-02-27 19:03:00'); + }); + + it('supports a "T" date-time without seconds', function () { + scope.normalizeAbsoluteDate('2025-02-27T19:03Z') + .should.equal('2025-02-27 19:03:00'); + }); + + it('truncates fractional seconds', function () { + scope.normalizeAbsoluteDate('2025-02-27T19:03:00.567Z') + .should.equal('2025-02-27 19:03:00'); + }); + + it('interprets a zone-less date-time as UTC', function () { + scope.normalizeAbsoluteDate('2025-02-27T19:03:00') + .should.equal('2025-02-27 19:03:00'); + }); + + it('is idempotent for a value already in db format', function () { + scope.normalizeAbsoluteDate('2025-02-27 19:03:00') + .should.equal('2025-02-27 19:03:00'); + }); + + it('leaves a bare date untouched (no time component)', function () { + scope.normalizeAbsoluteDate('2025-02-27').should.equal('2025-02-27'); + }); + + it('leaves a non-date string untouched', function () { + scope.normalizeAbsoluteDate('not-a-date').should.equal('not-a-date'); + }); + + it('leaves an unparseable date-time shaped string untouched', function () { + scope.normalizeAbsoluteDate('2025-13-45T99:99:99Z') + .should.equal('2025-13-45T99:99:99Z'); + }); + + it('returns non-string values unchanged', function () { + (scope.normalizeAbsoluteDate(null) === null).should.be.true(); + scope.normalizeAbsoluteDate(5).should.equal(5); + scope.normalizeAbsoluteDate(true).should.equal(true); + }); + }); }); From 8ae6de1a7d2521a1e957977f3bbd8fc34d132291 Mon Sep 17 00:00:00 2001 From: Steve Larson <9larsons@gmail.com> Date: Fri, 3 Jul 2026 09:01:30 -0500 Subject: [PATCH 2/2] Harden absolute date-time normalization - Reject calendar-invalid dates (e.g. Feb 30) instead of letting new Date roll them over to a different day - Enforce hour/minute/second ranges in the ISO regex so out-of-range times (T24:00, 19:60) pass through instead of rolling over - Require the T separator: space-separated values are either already in the stored format or arbitrary text, and only the ISO T form exhibits the comparison bug being fixed - Skip normalization when preserveRelativeDates is set, so the lossless parse mode also preserves absolute date-times --- packages/nql-lang/lib/scope.js | 33 ++++++++++++++++++------ packages/nql-lang/test/parser.test.js | 16 ++++++++++++ packages/nql-lang/test/scope.test.js | 36 +++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/packages/nql-lang/lib/scope.js b/packages/nql-lang/lib/scope.js index 51440f7a..cc1a3915 100644 --- a/packages/nql-lang/lib/scope.js +++ b/packages/nql-lang/lib/scope.js @@ -34,14 +34,19 @@ const formatDateForSQL = (date) => { return isoDate.replace('T', ' ').replace(/\.[0-9]{3}Z/, ''); }; -// A full ISO-8601 date-time: a date WITH a time component, optionally with -// fractional seconds and a timezone (Z or ±HH:MM). We deliberately require a -// time component so bare dates ("2025-02-27") and any other plain string are -// never rewritten — nql-lang has no column-type information, so this is the -// only shape we can safely normalize without risking a legitimate non-date -// value (e.g. a date-like slug). +// A full ISO-8601 date-time: a date WITH a "T"-separated time component, +// optionally with fractional seconds and a timezone (Z or ±HH:MM). We +// deliberately require a time component so bare dates ("2025-02-27") and any +// other plain string are never rewritten — nql-lang has no column-type +// information, so this is the only shape we can safely normalize without +// risking a legitimate non-date value (e.g. a date-like slug). The "T" +// separator is required for the same reason: space-separated values are +// already in the stored format (or arbitrary text), and only the ISO "T" form +// exhibits the comparison bug being fixed. Hour/minute/second ranges are +// enforced so out-of-range times ("T24:00") can't slip through to `new Date`, +// which would roll them over to a different day instead of rejecting them. // Groups: 1=date, 2=time (HH:mm[:ss]), 3=fraction, 4=zone. -const ISO_DATE_TIME = /^(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}(?::\d{2})?)(\.\d+)?(Z|[+-]\d{2}:?\d{2})?$/; +const ISO_DATE_TIME = /^(\d{4}-\d{2}-\d{2})T((?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d)?)(\.\d+)?(Z|[+-]\d{2}:?\d{2})?$/; module.exports = { ungroup(value) { @@ -89,7 +94,10 @@ module.exports = { // isn't a full ISO-8601 date-time, or that fails to parse, is returned // untouched. normalizeAbsoluteDate(value) { - if (typeof value !== 'string') { + // `preserveRelativeDates` opts in to a lossless parse (values survive + // for rendering/round-tripping), so absolute dates must survive + // untouched there too. + if (this.preserveRelativeDates || typeof value !== 'string') { return value; } @@ -99,6 +107,15 @@ module.exports = { } const [, date, time, fraction = '', zone] = match; + + // `new Date` rejects most out-of-range fields but silently rolls over + // days that are ≤31 yet invalid for their month (Feb 30 → Mar 1), which + // would make the filter query a different day than the user wrote. + const dayCheck = new Date(`${date}T00:00:00Z`); + if (Number.isNaN(dayCheck.getTime()) || dayCheck.toISOString().slice(0, 10) !== date) { + return value; + } + // A zone-less value is interpreted as UTC (dates are stored in UTC), so // we append "Z" rather than letting `new Date` treat it as local time. const isoString = `${date}T${time}${fraction}${zone || 'Z'}`; diff --git a/packages/nql-lang/test/parser.test.js b/packages/nql-lang/test/parser.test.js index 381c439c..577ca698 100644 --- a/packages/nql-lang/test/parser.test.js +++ b/packages/nql-lang/test/parser.test.js @@ -709,6 +709,12 @@ describe('Parser', function () { }); }); + it('leaves absolute date-times untouched when enabled', function () { + parse('created_at:>=\'2025-02-27T19:03:00.000-05:00\'', {preserveRelativeDates: true}).should.eql({ + created_at: {$gte: '2025-02-27T19:03:00.000-05:00'} + }); + }); + it('preserves the same units the lexer recognises', function () { const intervals = [ ['d', 'days'], @@ -780,5 +786,15 @@ describe('Parser', function () { it('does not rewrite a non-date string value', function () { parse('slug:\'2025-02-27\'').should.eql({slug: '2025-02-27'}); }); + + it('does not rewrite a space-separated date-time (already in stored format)', function () { + parse('published_at:>\'2025-02-27 19:03:00\'') + .should.eql({published_at: {$gt: '2025-02-27 19:03:00'}}); + }); + + it('does not rewrite a calendar-invalid date-time', function () { + parse('published_at:>\'2025-02-30T00:00:00Z\'') + .should.eql({published_at: {$gt: '2025-02-30T00:00:00Z'}}); + }); }); }); diff --git a/packages/nql-lang/test/scope.test.js b/packages/nql-lang/test/scope.test.js index 1c93f4af..bd02a10c 100644 --- a/packages/nql-lang/test/scope.test.js +++ b/packages/nql-lang/test/scope.test.js @@ -84,6 +84,29 @@ describe('Scope date helpers', function () { .should.equal('2025-02-27 19:03:00'); }); + it('leaves space-separated date-times untouched (only the ISO "T" form is rewritten)', function () { + scope.normalizeAbsoluteDate('2025-02-27 19:03') + .should.equal('2025-02-27 19:03'); + scope.normalizeAbsoluteDate('2025-02-27 19:03:00Z') + .should.equal('2025-02-27 19:03:00Z'); + }); + + it('leaves calendar-invalid dates untouched instead of rolling them over', function () { + // `new Date` would roll these to Mar 1 / May 1 rather than rejecting them + scope.normalizeAbsoluteDate('2025-02-30T00:00:00Z') + .should.equal('2025-02-30T00:00:00Z'); + scope.normalizeAbsoluteDate('2025-04-31T10:00:00Z') + .should.equal('2025-04-31T10:00:00Z'); + }); + + it('leaves out-of-range times untouched instead of rolling them over', function () { + // 24:00 is spec-legal for `new Date` and rolls to next-day midnight + scope.normalizeAbsoluteDate('2025-01-01T24:00:00Z') + .should.equal('2025-01-01T24:00:00Z'); + scope.normalizeAbsoluteDate('2025-01-01T19:60:00Z') + .should.equal('2025-01-01T19:60:00Z'); + }); + it('leaves a bare date untouched (no time component)', function () { scope.normalizeAbsoluteDate('2025-02-27').should.equal('2025-02-27'); }); @@ -102,5 +125,18 @@ describe('Scope date helpers', function () { scope.normalizeAbsoluteDate(5).should.equal(5); scope.normalizeAbsoluteDate(true).should.equal(true); }); + + describe('with preserveRelativeDates flag set', function () { + afterEach(function () { + scope.preserveRelativeDates = false; + }); + + it('leaves absolute date-times untouched (lossless parse)', function () { + scope.preserveRelativeDates = true; + + scope.normalizeAbsoluteDate('2025-02-27T19:03:00.000-05:00') + .should.equal('2025-02-27T19:03:00.000-05:00'); + }); + }); }); });