Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,7 @@ class CommentReportsImporter extends TableImporter {
return null;
}

const commentCreatedAt = new Date(this.model.created_at);
const now = new Date();
const reportTime = faker.date.between(commentCreatedAt, now);
const reportTime = dateToDatabaseString.randomBetween(this.model.created_at, new Date());

const reporter = this.possibleReporters[faker.datatype.number(this.possibleReporters.length - 1)];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ class CommentsImporter extends TableImporter {

setReferencedModel(model) {
this.model = model;
const publishedAt = dateToDatabaseString.parse(model.published_at);

this.commentIds = []; // Store [id, parent_id, timestamp] tuples for reply-to-reply

Expand All @@ -32,11 +33,11 @@ class CommentsImporter extends TableImporter {
trend: 'negative',
// Use commentsPerPost as a baseline with some variance (+/- 20%)
total: Math.round(this.commentsPerPost * faker.datatype.float({min: 0.8, max: 1.2})),
startTime: new Date(model.published_at),
startTime: publishedAt,
endTime: new Date()
}).sort((a, b) => a.getTime() - b.getTime()); // Sort chronologically so replies always come after their targets

this.possibleMembers = this.members.filter(member => new Date(member.created_at) < new Date(model.published_at));
this.possibleMembers = this.members.filter(member => dateToDatabaseString.parse(member.created_at) < publishedAt);
}

generate() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ class EmailBatchesImporter extends TableImporter {
}

generate() {
const emailSentDate = new Date(this.model.created_at);
const latestUpdatedDate = new Date(this.model.created_at);
const emailSentDate = dateToDatabaseString.parse(this.model.created_at);
const latestUpdatedDate = dateToDatabaseString.parse(this.model.created_at);
latestUpdatedDate.setHours(latestUpdatedDate.getHours() + 1);

return {
Expand All @@ -30,7 +30,7 @@ class EmailBatchesImporter extends TableImporter {
provider_id: `${new Date().toISOString().split('.')[0].replace(/[^0-9]/g, '')}.${faker.datatype.hexadecimal({length: 16, prefix: '', case: 'lower'})}@m.example.com`,
status: 'submitted', // TODO: introduce failures
created_at: this.model.created_at,
updated_at: dateToDatabaseString(faker.date.between(emailSentDate, latestUpdatedDate))
updated_at: dateToDatabaseString(dateToDatabaseString.randomBetween(emailSentDate, latestUpdatedDate))
};
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ class EmailRecipientsImporter extends TableImporter {

if (!(memberSubscribeEvent.created_at instanceof Date)) {
// SQLite fix
memberSubscribeEvent.created_at = new Date(memberSubscribeEvent.created_at);
memberSubscribeEvent.created_at = dateToDatabaseString.parse(memberSubscribeEvent.created_at);
}
this.membersSubscribeEventsCreatedAtsByNewsletterId.get(memberSubscribeEvent.newsletter_id).push(memberSubscribeEvent.created_at.getTime());
}
Expand All @@ -125,8 +125,8 @@ class EmailRecipientsImporter extends TableImporter {
this.batchIndex = this.batch.index;

// Shallow clone members list so we can shuffle and modify it
const earliestOpenTime = new Date(this.batch.updated_at);
const latestOpenTime = new Date(this.batch.updated_at);
const earliestOpenTime = dateToDatabaseString.parse(this.batch.updated_at);
const latestOpenTime = dateToDatabaseString.parse(this.batch.updated_at);
latestOpenTime.setDate(latestOpenTime.getDate() + 14);

// Get all members that were subscribed to this newsletter BEFORE the batch was sent
Expand Down Expand Up @@ -164,7 +164,7 @@ class EmailRecipientsImporter extends TableImporter {
}

// The events are generated for a different time, so we need to move them to the batch time
timestamp = new Date(timestamp.getTime() - this.eventStartTimeUsed.getTime() + new Date(this.batch.updated_at).getTime());
timestamp = new Date(timestamp.getTime() - this.eventStartTimeUsed.getTime() + dateToDatabaseString.parse(this.batch.updated_at).getTime());

if (timestamp > new Date()) {
timestamp = new Date();
Expand All @@ -187,9 +187,9 @@ class EmailRecipientsImporter extends TableImporter {

let deliveredTime;
if (status === emailStatus.opened) {
const startDate = this.batch.updated_at;
const startDate = dateToDatabaseString.parse(this.batch.updated_at);
const endDate = timestamp;
deliveredTime = faker.date.between(startDate, endDate);
deliveredTime = dateToDatabaseString.randomBetween(startDate, endDate);
}

return {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,19 +42,20 @@ class EmailsImporter extends TableImporter {
: this.newsletters[1];
}

const publishedAt = dateToDatabaseString.parse(this.model.published_at);
const timestamp = luck(60)
? new Date(this.model.published_at)
? publishedAt
: generateEvents({
shape: 'ease-out',
trend: 'negative',
total: 1,
startTime: new Date(this.model.published_at),
startTime: publishedAt,
endTime: new Date()
})[0];

const recipientCount = this.membersSubscribeEvents
.filter(entry => entry.newsletter_id === newsletter.id)
.filter(entry => new Date(entry.created_at) < timestamp).length;
.filter(entry => dateToDatabaseString.parse(entry.created_at) < timestamp).length;
const deliveredCount = Math.ceil(recipientCount * faker.datatype.float({
max: 1,
min: 0.9,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ class MembersClickEventsImporter extends TableImporter {
}
this.amount -= 1;

const openedAt = new Date(this.model.opened_at);
const openedAt = dateToDatabaseString.parse(this.model.opened_at);
const laterOn = new Date(openedAt.getTime() + 1000 * 60 * 15);
const clickTime = faker.date.between(openedAt.getTime(), laterOn.getTime()); //added getTime here because it threw random errors

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ class MembersCreatedEventsImporter extends TableImporter {
};

if (source === 'member' && luck(30)) {
const post = this.posts.find(p => p.visibility === 'public' && new Date(p.published_at) < new Date(this.model.created_at));
const memberCreatedAt = dateToDatabaseString.parse(this.model.created_at);
const post = this.posts.find(p => p.visibility === 'public' && dateToDatabaseString.parse(p.published_at) < memberCreatedAt);
if (post) {
attribution = {
attribution_id: post.id,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
const TableImporter = require('./table-importer');
const {faker} = require('@faker-js/faker');
const {luck} = require('../utils/random');
const dateToDatabaseString = require('../utils/database-date');

Expand All @@ -25,10 +24,10 @@ class MembersFeedbackImporter extends TableImporter {
return null;
}

const openedAt = new Date(this.model.opened_at);
const laterOn = new Date(this.model.opened_at);
const openedAt = dateToDatabaseString.parse(this.model.opened_at);
const laterOn = dateToDatabaseString.parse(this.model.opened_at);
laterOn.setMinutes(laterOn.getMinutes() + 60);
const feedbackTime = faker.date.between(openedAt, laterOn);
const feedbackTime = dateToDatabaseString.randomBetween(openedAt, laterOn);

const postId = this.emails.find(email => email.id === this.model.email_id).post_id;
return {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,10 @@ class MembersLoginEventsImporter extends TableImporter {

setReferencedModel(model) {
this.model = model;
const memberCreatedAt = dateToDatabaseString.parse(model.created_at);

const endDate = new Date();
const daysBetween = Math.ceil((endDate.valueOf() - new Date(model.created_at).valueOf()) / (1000 * 60 * 60 * 24));
const daysBetween = Math.ceil((endDate.valueOf() - memberCreatedAt.valueOf()) / (1000 * 60 * 60 * 24));

// Assuming most people either subscribe and lose interest, or maintain steady readership
const shape = luck(40) ? 'ease-out' : 'flat';
Expand All @@ -47,7 +48,7 @@ class MembersLoginEventsImporter extends TableImporter {
// Steady readers login more, readers who lose interest read less overall.
// ceil because members will all have logged in at least once
total: Math.min(5, shape === 'flat' ? Math.ceil(daysBetween / 3) : Math.ceil(daysBetween / 7)),
startTime: new Date(model.created_at),
startTime: memberCreatedAt,
endTime: endDate
});
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
const TableImporter = require('./table-importer');
const {faker} = require('@faker-js/faker');
const dateToDatabaseString = require('../utils/database-date');

class MembersStatusEventsImporter extends TableImporter {
Expand Down Expand Up @@ -41,7 +40,7 @@ class MembersStatusEventsImporter extends TableImporter {
member_id: model.id,
from_status: 'free',
to_status: model.status,
created_at: dateToDatabaseString(faker.date.between(new Date(model.created_at), new Date()))
created_at: dateToDatabaseString(dateToDatabaseString.randomBetween(model.created_at, new Date()))
});
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
const {faker} = require('@faker-js/faker');
const TableImporter = require('./table-importer');
const dateToDatabaseString = require('../utils/database-date');

class MembersStripeCustomersImporter extends TableImporter {
static table = 'members_stripe_customers';
Expand Down Expand Up @@ -35,7 +36,7 @@
// Only 30% of free members should have a stripe customer = have had a subscription in the past or tried to subscribe
// The number should increase the older the member is

const daysSinceMemberCreated = Math.floor((new Date() - new Date(this.model.created_at)) / (1000 * 60 * 60 * 24));
const daysSinceMemberCreated = Math.floor((new Date() - dateToDatabaseString.parse(this.model.created_at)) / (1000 * 60 * 60 * 24));

Check warning on line 39 in ghost/core/core/server/data/seeders/importers/members-stripe-customers-importer.js

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Prefer `Date.now()` over `new Date()`.

See more on https://sonarcloud.io/project/issues?id=TryGhost_Ghost&issues=AZ0qZnTA0GVV5QbRYsPi&open=AZ0qZnTA0GVV5QbRYsPi&pullRequest=26985
const shouldHaveStripeCustomer = faker.datatype.number({min: 0, max: 100}) < Math.max(Math.min(daysSinceMemberCreated / 60, 15), 2);

if (!shouldHaveStripeCustomer) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ class MembersStripeCustomersSubscriptionsImporter extends TableImporter {
this.members = await this.transaction.select('id', 'status', 'created_at').from('members').whereIn('id', membersStripeCustomers.map(m => m.member_id));

if (this.members.length === 0) {
offset += limit;
continue;
}

Expand Down Expand Up @@ -95,6 +96,7 @@ class MembersStripeCustomersSubscriptionsImporter extends TableImporter {
(isMonthly ? price.interval === 'month' : price.interval === 'year');
});
const mrr = createValid ? (isMonthly ? stripePrice.amount : Math.floor(stripePrice.amount / 12)) : 0;
const memberCreatedAt = dateToDatabaseString.parse(member.created_at);

const referenceEndDate = this.lastSubscriptionStart ?? new Date();

Expand All @@ -106,15 +108,15 @@ class MembersStripeCustomersSubscriptionsImporter extends TableImporter {
}
}

if (referenceEndDate < member.created_at) {
if (referenceEndDate < memberCreatedAt) {
// Not possible to create an invalid subscription here
return;
}

const [startDate] = generateEvents({
total: 1,
trend: 'negative',
startTime: new Date(member.created_at),
startTime: memberCreatedAt,
endTime: referenceEndDate,
shape: 'ease-out'
});
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
const TableImporter = require('./table-importer');
const {faker} = require('@faker-js/faker');
const {luck} = require('../utils/random');
const dateToDatabaseString = require('../utils/database-date');

Expand Down Expand Up @@ -52,7 +51,7 @@ class MembersSubscribeEventsImporter extends TableImporter {
return null;
}

const createdAt = dateToDatabaseString(faker.date.between(new Date(this.model.created_at), new Date()));
const createdAt = dateToDatabaseString(dateToDatabaseString.randomBetween(this.model.created_at, new Date()));
const newsletterId = this.newsletters[count % this.newsletters.length].id;

return {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
const TableImporter = require('./table-importer');
const {faker} = require('@faker-js/faker');
const {luck} = require('../utils/random');
const dateToDatabaseString = require('../utils/database-date');

class MembersSubscriptionCreatedEventsImporter extends TableImporter {
static table = 'members_subscription_created_events';
Expand Down Expand Up @@ -48,7 +49,8 @@ class MembersSubscriptionCreatedEventsImporter extends TableImporter {
};

if (luck(30)) {
const post = this.posts.find(p => p.visibility === 'public' && new Date(p.published_at) < new Date(this.model.created_at));
const createdAt = dateToDatabaseString.parse(this.model.created_at);
const post = this.posts.find(p => p.visibility === 'public' && dateToDatabaseString.parse(p.published_at) < createdAt);
if (post) {
attribution = {
attribution_id: post.id,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ class OfferRedemptionsImporter extends TableImporter {
this.subscriptionPool.push({
memberId,
subscriptionId: subscription.id,
subscriptionCreatedAt: new Date(subscription.created_at),
subscriptionCreatedAt: dateToDatabaseString.parse(subscription.created_at),
redemptionEndAt: this.getRedemptionEndDate(subscription.current_period_end),
availableOffers: [...matchingOffers],
lastRedeemedAt: null
Expand All @@ -78,15 +78,15 @@ class OfferRedemptionsImporter extends TableImporter {

getRedemptionEndDate(currentPeriodEnd) {
const now = new Date();
const endDate = currentPeriodEnd ? new Date(currentPeriodEnd) : now;
const endDate = currentPeriodEnd ? dateToDatabaseString.parse(currentPeriodEnd) : now;

return endDate > now ? now : endDate;
}

getCreatedAt(subscriptionState, offer) {
const candidateEarliest = new Date(Math.max(
subscriptionState.subscriptionCreatedAt.valueOf(),
new Date(offer.created_at).valueOf(),
dateToDatabaseString.parse(offer.created_at).valueOf(),
subscriptionState.lastRedeemedAt ? subscriptionState.lastRedeemedAt.valueOf() + 1000 : 0
));
const earliest = new Date(Math.min(
Expand Down
27 changes: 26 additions & 1 deletion ghost/core/core/server/data/seeders/utils/database-date.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,32 @@
module.exports = function dateToDatabaseString(date) {
const {faker} = require('@faker-js/faker');

const databaseDatePattern = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)?$/;

function dateToDatabaseString(date) {
if (typeof date === 'string') {
// SQLite fix when reusing other dates from the db
return date;
}
return date.toISOString().replace('Z','').replace('T', ' ');
}

dateToDatabaseString.parse = function parseDatabaseDate(date) {
if (date instanceof Date) {
return new Date(date);
}

if (typeof date === 'string' && databaseDatePattern.test(date)) {
return new Date(date.replace(' ', 'T') + 'Z');
}

return new Date(date);
};

dateToDatabaseString.randomBetween = function randomBetween(start, end) {
const earliest = dateToDatabaseString.parse(start);
const latest = dateToDatabaseString.parse(end);

return latest > earliest ? faker.date.between(earliest, latest) : earliest;
};

module.exports = dateToDatabaseString;
26 changes: 26 additions & 0 deletions ghost/core/test/unit/server/data/seeders/data-generator.test.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
const assert = require('node:assert/strict');
const {spawnSync} = require('node:child_process');

const knex = require('knex').default;

Expand All @@ -9,6 +10,7 @@ const StripeProductsImporter = importers.find(i => i.table === 'stripe_products'
const StripePricesImporter = importers.find(i => i.table === 'stripe_prices');

const generateEvents = require('../../../../../core/server/data/seeders/utils/event-generator');
const dateToDatabaseString = require('../../../../../core/server/data/seeders/utils/database-date');

const DataGenerator = require('../../../../../core/server/data/seeders/data-generator');

Expand Down Expand Up @@ -253,6 +255,30 @@ describe('Importer', function () {
});

describe('Events Generator', function () {
it('Parses database timestamps as UTC in non-UTC timezones', function () {
const script = `
const dateToDatabaseString = require(${JSON.stringify(require.resolve('../../../../../core/server/data/seeders/utils/database-date'))});
process.stdout.write(dateToDatabaseString.parse('2026-03-26 11:50:00.000').toISOString());
`;
const result = spawnSync(process.execPath, ['-e', script], {
env: {
...process.env,
TZ: 'America/New_York'
},
encoding: 'utf8'
});

assert.equal(result.status, 0);
assert.equal(result.stdout, '2026-03-26T11:50:00.000Z');
});

it('Returns the start date when a range is inverted', function () {
const startDate = new Date('2026-03-26T11:50:00.000Z');
const endDate = new Date('2026-03-26T10:00:00.000Z');

assert.equal(dateToDatabaseString.randomBetween(startDate, endDate).toISOString(), startDate.toISOString());
});

it('Generates a set of timestamps which meet the criteria', function () {
const startTime = new Date();
startTime.setDate(startTime.getDate() - 30);
Expand Down
Loading