From 6c9f4338fa5565ed9c6ba207c5da18c0a286cbd0 Mon Sep 17 00:00:00 2001 From: Franck Nijhof Date: Sat, 27 Jun 2026 20:37:55 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20incorrect=20date=20filte?= =?UTF-8?q?ring=20on=20SQLite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fixes https://github.com/TryGhost/Ghost/issues/23441 - date columns are stored as "YYYY-MM-DD HH:MM:SS", but absolute date values in filters were passed through untouched while only relative dates (now-30d) were normalized - on SQLite, datetimes are stored as text and compared lexically, so the "T" in an ISO value sorts after the stored space separator and greater/less-than filters returned the wrong rows - normalizes date-column filter values to the stored UTC format before the query is built, leaving non-date columns and unparseable values untouched --- .../core/core/server/models/base/bookshelf.js | 5 + .../server/models/base/plugins/date-filter.js | 136 ++++++++++++++++++ .../model/post-date-filter.test.js | 57 ++++++++ .../server/models/base/date-filter.test.js | 110 ++++++++++++++ 4 files changed, 308 insertions(+) create mode 100644 ghost/core/core/server/models/base/plugins/date-filter.js create mode 100644 ghost/core/test/integration/model/post-date-filter.test.js create mode 100644 ghost/core/test/unit/server/models/base/date-filter.test.js diff --git a/ghost/core/core/server/models/base/bookshelf.js b/ghost/core/core/server/models/base/bookshelf.js index 7c6518b4d59..28166e95ed5 100644 --- a/ghost/core/core/server/models/base/bookshelf.js +++ b/ghost/core/core/server/models/base/bookshelf.js @@ -20,6 +20,11 @@ ghostBookshelf.plugin(plugins.customQuery); // Load the Ghost filter plugin, which handles applying a 'filter' to findPage requests ghostBookshelf.plugin(plugins.filter); +// Normalize absolute date values in filters to the database date format, so date +// comparisons behave consistently across SQLite and MySQL. Must come after the +// filter plugin, which it wraps. +ghostBookshelf.plugin(require('./plugins/date-filter')); + // Load the Ghost filter plugin, which handles applying a 'order' to findPage requests ghostBookshelf.plugin(plugins.order); diff --git a/ghost/core/core/server/models/base/plugins/date-filter.js b/ghost/core/core/server/models/base/plugins/date-filter.js new file mode 100644 index 00000000000..a84174e7bbf --- /dev/null +++ b/ghost/core/core/server/models/base/plugins/date-filter.js @@ -0,0 +1,136 @@ +const moment = require('moment'); +const {chainTransformers} = require('@tryghost/mongo-utils'); +const schemaTables = require('../../../data/schema/schema'); + +// Date columns are stored as "YYYY-MM-DD HH:MM:SS" (UTC). NQL normalizes relative +// dates (e.g. `now-30d`) to that format, but absolute values from a filter +// (e.g. `published_at:>'2025-02-27T19:03:00.000-05:00'`) are passed through as-is. +// On SQLite, datetimes are stored as text and compared lexically, so the "T" +// sorts after the space separator and the comparison returns the wrong rows. +// We normalize those values to the stored format before the query is built. +// See https://github.com/TryGhost/Ghost/issues/23441 +const ACCEPTED_DATE_FORMATS = [moment.ISO_8601, 'YYYY-MM-DD HH:mm:ss']; +const DB_DATE_FORMAT = 'YYYY-MM-DD HH:mm:ss'; + +// Columns to treat as dates, keyed by column name. A name only qualifies when +// it is a `dateTime` in every table that has it, so we never normalize a value +// for a same-named column of another type. Filters resolve by column name (the +// trailing segment), which keeps relation-qualified fields like `tags.created_at` +// working without needing the model's table. +let dateColumns = null; +const getDateColumns = () => { + if (!dateColumns) { + const otherColumns = new Set(); + dateColumns = new Set(); + + for (const columns of Object.values(schemaTables)) { + for (const [name, spec] of Object.entries(columns)) { + if (spec && spec.type === 'dateTime') { + dateColumns.add(name); + } else if (spec && spec.type) { + otherColumns.add(name); + } + } + } + + otherColumns.forEach(name => dateColumns.delete(name)); + } + return dateColumns; +}; + +const isDateColumn = (key) => { + const column = key.includes('.') ? key.slice(key.lastIndexOf('.') + 1) : key; + return getDateColumns().has(column); +}; + +// Reformat a single value to the database date format. Non-strings and values we +// can't parse as a date are returned untouched, so unexpected input is never +// corrupted. +const normalizeValue = (value) => { + if (typeof value !== 'string') { + return value; + } + + const parsed = moment.utc(value, ACCEPTED_DATE_FORMATS, true); + return parsed.isValid() ? parsed.format(DB_DATE_FORMAT) : value; +}; + +// An operator map like `{$gt: ...}`: a plain object whose keys are all operators. +// Anything else that happens to be an object (e.g. a `Date`) is not one and must +// be left untouched rather than reduced to `{}`. +const isOperatorMap = (value) => { + if (!value || Object.prototype.toString.call(value) !== '[object Object]') { + return false; + } + + const keys = Object.keys(value); + return keys.length > 0 && keys.every(key => key.charAt(0) === '$'); +}; + +// A field value is a plain value (equality), an array (e.g. `$in`), or an operator +// map (e.g. `{$gt: ...}`). +const normalizeFieldValue = (value) => { + if (Array.isArray(value)) { + return value.map(normalizeValue); + } + + if (isOperatorMap(value)) { + const result = {}; + for (const [operator, operatorValue] of Object.entries(value)) { + result[operator] = Array.isArray(operatorValue) + ? operatorValue.map(normalizeValue) + : normalizeValue(operatorValue); + } + return result; + } + + return normalizeValue(value); +}; + +// Walk the parsed mongo-JSON filter, normalizing date column values and recursing +// into `$and`/`$or` groups. +const normalizeDateFilters = (node) => { + if (Array.isArray(node)) { + return node.map(normalizeDateFilters); + } + + if (!node || typeof node !== 'object') { + return node; + } + + const result = {}; + for (const [key, value] of Object.entries(node)) { + if (key.charAt(0) === '$') { + result[key] = normalizeDateFilters(value); + } else if (isDateColumn(key)) { + result[key] = normalizeFieldValue(value); + } else { + result[key] = value; + } + } + return result; +}; + +/** + * Normalizes absolute date values in NQL filters to the database date format, so + * date comparisons behave the same on SQLite and MySQL. Wraps + * `applyDefaultAndCustomFilters` and chains the date transformer after any + * transformer the caller supplied. + * + * @param {import('bookshelf')} Bookshelf + */ +module.exports = function (Bookshelf) { + const parentApply = Bookshelf.Model.prototype.applyDefaultAndCustomFilters; + + Bookshelf.Model = Bookshelf.Model.extend({ + applyDefaultAndCustomFilters: function applyDefaultAndCustomFilters(options = {}) { + const mongoTransformer = options.mongoTransformer + ? chainTransformers(options.mongoTransformer, normalizeDateFilters) + : normalizeDateFilters; + + return parentApply.call(this, Object.assign({}, options, {mongoTransformer})); + } + }); +}; + +module.exports.normalizeDateFilters = normalizeDateFilters; diff --git a/ghost/core/test/integration/model/post-date-filter.test.js b/ghost/core/test/integration/model/post-date-filter.test.js new file mode 100644 index 00000000000..aae9523bf66 --- /dev/null +++ b/ghost/core/test/integration/model/post-date-filter.test.js @@ -0,0 +1,57 @@ +const assert = require('node:assert/strict'); +const testUtils = require('../../utils'); +const models = require('../../../core/server/models'); + +const context = testUtils.context.owner; +const markdownToMobiledoc = testUtils.DataGenerator.markdownToMobiledoc; + +// Regression test for https://github.com/TryGhost/Ghost/issues/23441 +// On SQLite an absolute ISO date in a filter used to sort after the stored +// "YYYY-MM-DD HH:MM:SS" format (because "T" > " "), returning the wrong rows. +describe('Integration: Post date filtering', function () { + const early = new Date(Date.UTC(2025, 5, 15, 9, 0, 0)); // same day, before the boundary + const late = new Date(Date.UTC(2025, 5, 15, 12, 0, 0)); // same day, after the boundary + const later = new Date(Date.UTC(2025, 11, 31, 23, 0, 0)); // a later day + + const addPost = (title, publishedAt) => models.Post.add({ + status: 'published', + title, + published_at: publishedAt, + mobiledoc: markdownToMobiledoc('content') + }, context); + + beforeAll(testUtils.teardownDb); + beforeAll(testUtils.setup('users:roles')); + beforeAll(async function () { + await addPost('early-same-day', early); + await addPost('late-same-day', late); + await addPost('later-day', later); + }); + afterAll(testUtils.teardownDb); + + const titlesFor = async (filter) => { + const result = await models.Post.findPage({filter, status: 'all'}); + return result.data.map(post => post.get('title')).sort(); + }; + + it('includes a same-day post that is after a "greater than" ISO boundary', async function () { + // Boundary is 2025-06-15 10:00:00 UTC. "late-same-day" (12:00) is after it. + const titles = await titlesFor("published_at:>'2025-06-15T10:00:00.000Z'"); + + assert.deepEqual(titles, ['late-same-day', 'later-day']); + }); + + it('excludes a same-day post that is before a "less than" ISO boundary', async function () { + // Boundary is 2025-06-15 10:00:00 UTC. Only "early-same-day" (09:00) is before it. + const titles = await titlesFor("published_at:<'2025-06-15T10:00:00.000Z'"); + + assert.deepEqual(titles, ['early-same-day']); + }); + + it('handles an ISO boundary with a timezone offset', async function () { + // 2025-06-15T05:00:00-05:00 is 2025-06-15 10:00:00 UTC, same boundary as above. + const titles = await titlesFor("published_at:>'2025-06-15T05:00:00.000-05:00'"); + + assert.deepEqual(titles, ['late-same-day', 'later-day']); + }); +}); diff --git a/ghost/core/test/unit/server/models/base/date-filter.test.js b/ghost/core/test/unit/server/models/base/date-filter.test.js new file mode 100644 index 00000000000..2b3af1a453e --- /dev/null +++ b/ghost/core/test/unit/server/models/base/date-filter.test.js @@ -0,0 +1,110 @@ +const assert = require('node:assert/strict'); +const {normalizeDateFilters} = require('../../../../../core/server/models/base/plugins/date-filter'); + +describe('Models: date-filter', function () { + describe('normalizeDateFilters', function () { + it('normalizes an ISO date with a timezone offset on a date column to UTC db format', function () { + const result = normalizeDateFilters({ + published_at: {$gt: '2025-02-27T19:03:00.000-05:00'} + }); + + assert.deepEqual(result, { + published_at: {$gt: '2025-02-28 00:03:00'} + }); + }); + + it('normalizes a Zulu ISO date on a date column', function () { + const result = normalizeDateFilters({ + published_at: {$lt: '2025-02-27T19:03:00Z'} + }); + + assert.deepEqual(result, { + published_at: {$lt: '2025-02-27 19:03:00'} + }); + }); + + it('normalizes an equality value on a date column', function () { + const result = normalizeDateFilters({ + created_at: '2025-02-27T19:03:00.000Z' + }); + + assert.deepEqual(result, { + created_at: '2025-02-27 19:03:00' + }); + }); + + it('leaves values already in db format untouched', function () { + const filter = {published_at: {$gt: '2025-02-27 19:03:00'}}; + + assert.deepEqual(normalizeDateFilters(filter), { + published_at: {$gt: '2025-02-27 19:03:00'} + }); + }); + + it('does not touch non-date columns even when the value looks like a date', function () { + const filter = {slug: '2025-02-27', title: {$ne: '2025-02-27T19:03:00Z'}}; + + assert.deepEqual(normalizeDateFilters(filter), filter); + }); + + it('leaves unparseable values on a date column untouched', function () { + const filter = {published_at: {$gt: 'not-a-date'}}; + + assert.deepEqual(normalizeDateFilters(filter), filter); + }); + + it('leaves a non-plain object value (e.g. a Date) on a date column untouched', function () { + const date = new Date('2025-02-27T19:03:00Z'); + const result = normalizeDateFilters({published_at: date}); + + assert.equal(result.published_at, date); + }); + + it('normalizes arrays of values ($in)', function () { + const result = normalizeDateFilters({ + published_at: {$in: ['2025-02-27T19:03:00Z', '2025-03-01T00:00:00Z']} + }); + + assert.deepEqual(result, { + published_at: {$in: ['2025-02-27 19:03:00', '2025-03-01 00:00:00']} + }); + }); + + it('recurses into $and / $or groups', function () { + const result = normalizeDateFilters({ + $and: [ + {published_at: {$gt: '2025-02-27T19:03:00Z'}}, + {$or: [ + {featured: true}, + {updated_at: {$lt: '2025-03-01T00:00:00Z'}} + ]} + ] + }); + + assert.deepEqual(result, { + $and: [ + {published_at: {$gt: '2025-02-27 19:03:00'}}, + {$or: [ + {featured: true}, + {updated_at: {$lt: '2025-03-01 00:00:00'}} + ]} + ] + }); + }); + + it('resolves relation-qualified date columns by column name', function () { + const result = normalizeDateFilters({ + 'posts.published_at': {$gt: '2025-02-27T19:03:00Z'} + }); + + assert.deepEqual(result, { + 'posts.published_at': {$gt: '2025-02-27 19:03:00'} + }); + }); + + it('returns primitive nodes unchanged', function () { + assert.equal(normalizeDateFilters(null), null); + assert.equal(normalizeDateFilters('string'), 'string'); + }); + }); +});