From 6ce67b42b06b1e99416d3d64fea58b51a7752713 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Oct 2025 14:10:54 +0000 Subject: [PATCH 1/6] Initial plan From edab647b3c1c3d3f052b3159d706fb4d33cbd081 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Oct 2025 14:23:49 +0000 Subject: [PATCH 2/6] Add watch collection schema, types, mongo service integration, and create collection migration Co-authored-by: victor-enogwe <23452630+victor-enogwe@users.noreply.github.com> --- .../src/common/constants/collections.ts | 1 + .../src/common/services/mongo.service.ts | 12 ++ packages/core/src/types/watch.types.test.ts | 124 ++++++++++++ packages/core/src/types/watch.types.ts | 27 +++ ...25.10.13T14.18.20.watch-collection.test.ts | 182 ++++++++++++++++++ .../2025.10.13T14.18.20.watch-collection.ts | 65 +++++++ ...0.13T14.22.21.migrate-events-watch-data.ts | 113 +++++++++++ 7 files changed, 524 insertions(+) create mode 100644 packages/core/src/types/watch.types.test.ts create mode 100644 packages/core/src/types/watch.types.ts create mode 100644 packages/scripts/src/__tests__/integration/2025.10.13T14.18.20.watch-collection.test.ts create mode 100644 packages/scripts/src/migrations/2025.10.13T14.18.20.watch-collection.ts create mode 100644 packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts diff --git a/packages/backend/src/common/constants/collections.ts b/packages/backend/src/common/constants/collections.ts index a4635b1a0..a2b54a30a 100644 --- a/packages/backend/src/common/constants/collections.ts +++ b/packages/backend/src/common/constants/collections.ts @@ -9,4 +9,5 @@ export const Collections = { SYNC: IS_DEV ? "_dev.sync" : "sync", USER: IS_DEV ? "_dev.user" : "user", WAITLIST: IS_DEV ? "_dev.waitlist" : "waitlist", + WATCH: IS_DEV ? "_dev.watch" : "watch", }; diff --git a/packages/backend/src/common/services/mongo.service.ts b/packages/backend/src/common/services/mongo.service.ts index d4ea9cc29..2a0719464 100644 --- a/packages/backend/src/common/services/mongo.service.ts +++ b/packages/backend/src/common/services/mongo.service.ts @@ -18,6 +18,7 @@ import { Schema_Event } from "@core/types/event.types"; import { Schema_Sync } from "@core/types/sync.types"; import { Schema_User } from "@core/types/user.types"; import { Schema_Waitlist } from "@core/types/waitlist/waitlist.types"; +import { Watch } from "@core/types/watch.types"; import { Collections } from "@backend/common/constants/collections"; import { ENV } from "@backend/common/constants/env.constants"; import { waitUntilEvent } from "@backend/common/helpers/common.util"; @@ -33,6 +34,7 @@ interface InternalClient { sync: Collection; user: Collection; waitlist: Collection; + watch: Collection; } class MongoService { @@ -96,6 +98,15 @@ class MongoService { return this.#accessInternalCollectionProps("waitlist"); } + /** + * watch + * + * mongo collection + */ + get watch(): InternalClient["watch"] { + return this.#accessInternalCollectionProps("watch"); + } + private onConnect(client: MongoClient, useDynamicDb = false) { this.#internalClient = this.createInternalClient(client, useDynamicDb); @@ -133,6 +144,7 @@ class MongoService { sync: db.collection(Collections.SYNC), user: db.collection(Collections.USER), waitlist: db.collection(Collections.WAITLIST), + watch: db.collection(Collections.WATCH), }; } diff --git a/packages/core/src/types/watch.types.test.ts b/packages/core/src/types/watch.types.test.ts new file mode 100644 index 000000000..b718553dc --- /dev/null +++ b/packages/core/src/types/watch.types.test.ts @@ -0,0 +1,124 @@ +import { faker } from "@faker-js/faker"; +import { + Watch, + WatchInput, + WatchSchema, + WatchSchemaStrict, +} from "@core/types/watch.types"; + +describe("Watch Types", () => { + const validWatch: Watch = { + _id: faker.string.uuid(), + userId: faker.database.mongodbObjectId(), + resourceId: faker.string.alphanumeric(20), + expiration: faker.date.future(), + createdAt: new Date(), + }; + + describe("WatchSchema", () => { + it("parses valid watch data", () => { + expect(() => WatchSchema.parse(validWatch)).not.toThrow(); + }); + + it("defaults createdAt to current date when not provided", () => { + const watchWithoutCreatedAt = { + ...validWatch, + createdAt: undefined, + }; + + const parsed = WatchSchema.parse(watchWithoutCreatedAt); + expect(parsed.createdAt).toBeInstanceOf(Date); + }); + + it("accepts valid MongoDB ObjectId for userId", () => { + const watchData = { + ...validWatch, + userId: faker.database.mongodbObjectId(), + }; + + expect(() => WatchSchema.parse(watchData)).not.toThrow(); + }); + + it("rejects invalid MongoDB ObjectId for userId", () => { + const watchData = { + ...validWatch, + userId: "invalid-object-id", + }; + + expect(() => WatchSchema.parse(watchData)).toThrow(); + }); + + it("requires all mandatory fields", () => { + const requiredFields = ["_id", "userId", "resourceId", "expiration"]; + + requiredFields.forEach((field) => { + const incompleteWatch = { ...validWatch }; + delete incompleteWatch[field as keyof Watch]; + + expect(() => WatchSchema.parse(incompleteWatch)).toThrow(); + }); + }); + + it("accepts string for _id (channelId)", () => { + const watchData = { + ...validWatch, + _id: "test-channel-id-123", + }; + + expect(() => WatchSchema.parse(watchData)).not.toThrow(); + }); + + it("requires expiration to be a Date", () => { + const watchData = { + ...validWatch, + expiration: "2024-12-31T23:59:59Z", // string instead of Date + }; + + expect(() => WatchSchema.parse(watchData)).toThrow(); + }); + }); + + describe("WatchSchemaStrict", () => { + it("rejects additional properties when using strict schema", () => { + const watchWithExtra = { + ...validWatch, + extraProperty: "should-not-be-allowed", + }; + + expect(() => WatchSchemaStrict.parse(watchWithExtra)).toThrow(); + }); + + it("accepts valid watch data with strict schema", () => { + expect(() => WatchSchemaStrict.parse(validWatch)).not.toThrow(); + }); + }); + + describe("WatchInput type", () => { + it("allows creating watch input without createdAt", () => { + const watchInput: WatchInput = { + _id: faker.string.uuid(), + userId: faker.database.mongodbObjectId(), + resourceId: faker.string.alphanumeric(20), + expiration: faker.date.future(), + }; + + // This should compile without errors + expect(watchInput).toBeDefined(); + expect(watchInput.createdAt).toBeUndefined(); + }); + + it("allows creating watch input with createdAt", () => { + const watchInput: WatchInput = { + _id: faker.string.uuid(), + userId: faker.database.mongodbObjectId(), + resourceId: faker.string.alphanumeric(20), + expiration: faker.date.future(), + createdAt: new Date(), + }; + + // This should compile without errors + expect(watchInput).toBeDefined(); + expect(watchInput.createdAt).toBeInstanceOf(Date); + }); + }); +}); diff --git a/packages/core/src/types/watch.types.ts b/packages/core/src/types/watch.types.ts new file mode 100644 index 000000000..f2a7fbeb4 --- /dev/null +++ b/packages/core/src/types/watch.types.ts @@ -0,0 +1,27 @@ +import { z } from "zod/v4"; +import { IDSchemaV4 } from "@core/types/type.utils"; + +/** + * Watch collection schema for Google Calendar push notification channels + * + * This schema stores channel metadata for Google Calendar push notifications + * to enable reliable lifecycle management of channels (creation, renewal, + * expiration, deletion) separately from sync data. + */ +export const WatchSchema = z.object({ + _id: z.string(), // channel_id - unique identifier for the notification channel + userId: IDSchemaV4, // user who owns this watch channel + resourceId: z.string(), // Google Calendar resource identifier + expiration: z.date(), // when the channel expires + createdAt: z.date().default(() => new Date()), // when this watch was created +}); + +export type Watch = z.infer; + +// Type for creating a new watch (without auto-generated fields) +export type WatchInput = Omit & { + createdAt?: Date; +}; + +// Schema for database storage (with strict validation) +export const WatchSchemaStrict = WatchSchema.strict(); diff --git a/packages/scripts/src/__tests__/integration/2025.10.13T14.18.20.watch-collection.test.ts b/packages/scripts/src/__tests__/integration/2025.10.13T14.18.20.watch-collection.test.ts new file mode 100644 index 000000000..fcda1cbad --- /dev/null +++ b/packages/scripts/src/__tests__/integration/2025.10.13T14.18.20.watch-collection.test.ts @@ -0,0 +1,182 @@ +import { faker } from "@faker-js/faker"; +import { zodToMongoSchema } from "@scripts/common/zod-to-mongo-schema"; +import Migration from "@scripts/migrations/2025.10.13T14.18.20.watch-collection"; +import { Watch, WatchSchemaStrict } from "@core/types/watch.types"; +import { + cleanupCollections, + cleanupTestDb, + setupTestDb, +} from "@backend/__tests__/helpers/mock.db.setup"; +import { Collections } from "@backend/common/constants/collections"; +import mongoService from "@backend/common/services/mongo.service"; + +describe("2025.10.13T14.18.20.watch-collection", () => { + const migration = new Migration(); + const collectionName = Collections.WATCH; + + beforeAll(setupTestDb); + beforeEach(cleanupCollections); + afterEach(() => mongoService.watch.drop()); + afterAll(cleanupTestDb); + + function generateWatch(): Watch { + return { + _id: faker.string.uuid(), + userId: faker.database.mongodbObjectId(), + resourceId: faker.string.alphanumeric(20), + expiration: faker.date.future(), + createdAt: faker.date.recent(), + }; + } + + async function validateUpMigration() { + const indexes = await mongoService.watch.indexes(); + const collectionInfo = await mongoService.watch.options(); + const $jsonSchema = zodToMongoSchema(WatchSchemaStrict); + + expect(collectionInfo["validationLevel"]).toBe("strict"); + expect(collectionInfo["validator"]).toBeDefined(); + expect(collectionInfo["validator"]).toHaveProperty("$jsonSchema"); + expect(collectionInfo["validator"]["$jsonSchema"]).toEqual($jsonSchema); + + expect(indexes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "_id_", key: { _id: 1 } }), + expect.objectContaining({ + name: `${collectionName}_userId_index`, + key: { userId: 1 }, + }), + expect.objectContaining({ + name: `${collectionName}_userId_expiration_index`, + key: { userId: 1, expiration: 1 }, + }), + ]), + ); + } + + it("up() creates collection when it doesn't exist", async () => { + const existsBefore = await mongoService.collectionExists(collectionName); + + expect(existsBefore).toBe(false); + + await migration.up(); + + const existsAfter = await mongoService.collectionExists(collectionName); + + expect(existsAfter).toBe(true); + + await validateUpMigration(); + }); + + it("up() modifies collection when it exists", async () => { + // Create collection first + await mongoService.db.createCollection(collectionName); + + const existsBefore = await mongoService.collectionExists(collectionName); + + expect(existsBefore).toBe(true); + + await migration.up(); + + await validateUpMigration(); + }); + + it("down() removes schema validation and indexes without dropping collection", async () => { + await migration.up(); + + const existsBefore = await mongoService.collectionExists(collectionName); + + expect(existsBefore).toBe(true); + + await validateUpMigration(); + + await migration.down(); + + const existsAfter = await mongoService.collectionExists(collectionName); + + expect(existsAfter).toBe(true); + + const indexes = await mongoService.watch.indexes(); + const collectionInfo = await mongoService.watch.options(); + + expect(indexes).toHaveLength(1); + expect(indexes).toEqual([ + expect.objectContaining({ name: "_id_", key: { _id: 1 } }), + ]); + + expect(collectionInfo["validationLevel"]).toBe("off"); + expect(collectionInfo["validationAction"]).toBe("error"); + expect(collectionInfo).not.toHaveProperty("validator"); + }); + + describe("mongo $jsonSchema validation", () => { + function generateValidWatch() { + return { + _id: faker.string.uuid(), + userId: faker.database.mongodbObjectId(), + resourceId: faker.string.alphanumeric(20), + expiration: new Date(faker.date.future()), + createdAt: new Date(faker.date.recent()), + }; + } + + beforeEach(migration.up.bind(migration)); + + it("allows valid watch documents", async () => { + const watch = generateValidWatch(); + + await expect(mongoService.watch.insertOne(watch)).resolves.toMatchObject({ + acknowledged: true, + insertedId: watch._id, + }); + }); + + it("rejects documents with missing required fields", async () => { + const incompleteWatch = { + _id: faker.string.uuid(), + userId: faker.database.mongodbObjectId(), + // missing resourceId and expiration + createdAt: new Date(), + }; + + await expect( + mongoService.watch.insertOne(incompleteWatch), + ).rejects.toThrow(); + }); + + it("rejects documents with missing userId", async () => { + const watchWithoutUserId = { + ...generateValidWatch(), + }; + // @ts-expect-error testing missing userId field + delete watchWithoutUserId.userId; + + await expect( + mongoService.watch.insertOne(watchWithoutUserId), + ).rejects.toThrow(/Document failed validation/); + }); + + it("rejects documents with additional properties", async () => { + const watchWithExtra = { + ...generateValidWatch(), + extraProperty: "should-not-be-allowed", + }; + + await expect( + mongoService.watch.insertOne(watchWithExtra), + ).rejects.toThrow(); + }); + + it("enforces unique constraint on _id (channelId)", async () => { + const watch1 = generateValidWatch(); + const watch2 = { + ...generateValidWatch(), + _id: watch1._id, // Same channelId + }; + + await mongoService.watch.insertOne(watch1); + + await expect(mongoService.watch.insertOne(watch2)).rejects.toThrow(); + }); + }); +}); diff --git a/packages/scripts/src/migrations/2025.10.13T14.18.20.watch-collection.ts b/packages/scripts/src/migrations/2025.10.13T14.18.20.watch-collection.ts new file mode 100644 index 000000000..e20cea31e --- /dev/null +++ b/packages/scripts/src/migrations/2025.10.13T14.18.20.watch-collection.ts @@ -0,0 +1,65 @@ +import type { RunnableMigration } from "umzug"; +import { MigrationContext } from "@scripts/common/cli.types"; +import { zodToMongoSchema } from "@scripts/common/zod-to-mongo-schema"; +import { WatchSchemaStrict } from "@core/types/watch.types"; +import mongoService from "@backend/common/services/mongo.service"; + +export default class Migration implements RunnableMigration { + readonly name: string = "2025.10.13T14.18.20.watch-collection"; + readonly path: string = "2025.10.13T14.18.20.watch-collection.ts"; + + async up(): Promise { + const { collectionName } = mongoService.watch; + const exists = await mongoService.collectionExists(collectionName); + + const $jsonSchema = zodToMongoSchema(WatchSchemaStrict); + + if (exists) { + // do not run in session + await mongoService.db.command({ + collMod: collectionName, + validationLevel: "strict", + validator: { $jsonSchema }, + }); + } else { + await mongoService.db.createCollection(collectionName, { + validator: { $jsonSchema }, + validationLevel: "strict", + }); + } + + // _id is unique by default in MongoDB, no need to create explicit index + + // Create index on userId for efficient user-based queries + await mongoService.watch.createIndex( + { userId: 1 }, + { name: `${collectionName}_userId_index` }, + ); + + // Create compound index on userId and expiration for cleanup operations + await mongoService.watch.createIndex( + { userId: 1, expiration: 1 }, + { name: `${collectionName}_userId_expiration_index` }, + ); + } + + async down(): Promise { + // do not drop table, just remove indexes and schema validation + const { collectionName } = mongoService.watch; + const exists = await mongoService.collectionExists(collectionName); + + if (!exists) return; + + await mongoService.db.command({ + collMod: collectionName, + validationLevel: "off", + validator: {}, + }); + + // _id index is built-in, no need to drop + await mongoService.watch.dropIndex(`${collectionName}_userId_index`); + await mongoService.watch.dropIndex( + `${collectionName}_userId_expiration_index`, + ); + } +} diff --git a/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts b/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts new file mode 100644 index 000000000..69f8789ee --- /dev/null +++ b/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts @@ -0,0 +1,113 @@ +import type { RunnableMigration } from "umzug"; +import { MigrationContext } from "@scripts/common/cli.types"; +import { Schema_Sync } from "@core/types/sync.types"; +import { Watch } from "@core/types/watch.types"; +import mongoService from "@backend/common/services/mongo.service"; + +export default class Migration implements RunnableMigration { + readonly name: string = "2025.10.13T14.22.21.migrate-events-watch-data"; + readonly path: string = "2025.10.13T14.22.21.migrate-events-watch-data.ts"; + + async up(): Promise { + // This is a non-destructive migration to copy events watch data from sync collection to watch collection + + const cursor = mongoService.sync.find({ + "google.events": { $exists: true, $ne: [] }, + }); + + let migratedCount = 0; + + for await (const syncDoc of cursor) { + if (!syncDoc.google?.events?.length) continue; + + const watchDocuments: Watch[] = []; + + for (const eventSync of syncDoc.google.events) { + if ( + !eventSync.channelId || + !eventSync.resourceId || + !eventSync.expiration + ) { + continue; // Skip incomplete watch data + } + + // Convert expiration string to Date + let expirationDate: Date; + try { + // Google Calendar expiration is typically a timestamp in milliseconds + const expirationMs = parseInt(eventSync.expiration); + expirationDate = new Date(expirationMs); + } catch { + // If parsing fails, skip this watch entry + console.warn( + `Invalid expiration format for channelId ${eventSync.channelId}: ${eventSync.expiration}`, + ); + continue; + } + + const watchDoc: Watch = { + _id: eventSync.channelId, + userId: syncDoc.user, + resourceId: eventSync.resourceId, + expiration: expirationDate, + createdAt: new Date(), // Set current time as creation time for migration + }; + + watchDocuments.push(watchDoc); + } + + if (watchDocuments.length > 0) { + try { + // Use insertMany with ordered: false to continue on duplicates + await mongoService.watch.insertMany(watchDocuments, { + ordered: false, + }); + migratedCount += watchDocuments.length; + } catch (error: any) { + // Log errors but continue migration (some channels might already exist) + if (error?.writeErrors) { + const duplicateErrors = error.writeErrors.filter( + (err: any) => err.code === 11000, + ); + const otherErrors = error.writeErrors.filter( + (err: any) => err.code !== 11000, + ); + + if (duplicateErrors.length > 0) { + console.log( + `Skipped ${duplicateErrors.length} duplicate watch channels for user ${syncDoc.user}`, + ); + } + + if (otherErrors.length > 0) { + console.error( + `Errors inserting watch data for user ${syncDoc.user}:`, + otherErrors, + ); + } + + // Count successful inserts + const successCount = + watchDocuments.length - error.writeErrors.length; + migratedCount += successCount; + } else { + console.error( + `Unexpected error migrating watch data for user ${syncDoc.user}:`, + error, + ); + } + } + } + } + + console.log( + `Migrated ${migratedCount} events watch channels to watch collection`, + ); + } + + async down(): Promise { + // This is a non-destructive migration, we don't remove the data from watch collection + // because it might have been updated or new watches might have been added + console.log("Non-destructive migration: watch collection data left intact"); + } +} From 3c3239f9960f6f048115092f73272216fe596fc5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Oct 2025 14:27:56 +0000 Subject: [PATCH 3/6] Add events watch data migration with comprehensive tests and error handling Co-authored-by: victor-enogwe <23452630+victor-enogwe@users.noreply.github.com> --- ...14.22.21.migrate-events-watch-data.test.ts | 247 ++++++++++++++++++ ...0.13T14.22.21.migrate-events-watch-data.ts | 12 + 2 files changed, 259 insertions(+) create mode 100644 packages/scripts/src/__tests__/integration/2025.10.13T14.22.21.migrate-events-watch-data.test.ts diff --git a/packages/scripts/src/__tests__/integration/2025.10.13T14.22.21.migrate-events-watch-data.test.ts b/packages/scripts/src/__tests__/integration/2025.10.13T14.22.21.migrate-events-watch-data.test.ts new file mode 100644 index 000000000..74a0a5d05 --- /dev/null +++ b/packages/scripts/src/__tests__/integration/2025.10.13T14.22.21.migrate-events-watch-data.test.ts @@ -0,0 +1,247 @@ +import { faker } from "@faker-js/faker"; +import Migration from "@scripts/migrations/2025.10.13T14.22.21.migrate-events-watch-data"; +import { Schema_Sync } from "@core/types/sync.types"; +import { Watch } from "@core/types/watch.types"; +import { + cleanupCollections, + cleanupTestDb, + setupTestDb, +} from "@backend/__tests__/helpers/mock.db.setup"; +import mongoService from "@backend/common/services/mongo.service"; + +describe("2025.10.13T14.22.21.migrate-events-watch-data", () => { + const migration = new Migration(); + + beforeAll(setupTestDb); + beforeEach(async () => { + await cleanupCollections(); + // Ensure watch collection is clean between tests + try { + await mongoService.watch.deleteMany({}); + } catch { + // Collection might not exist yet, which is fine + } + }); + afterAll(cleanupTestDb); + + function createSyncDocWithEventsWatch( + userId: string, + eventsCount = 2, + ): Schema_Sync { + const events = Array.from({ length: eventsCount }, () => ({ + gCalendarId: faker.string.uuid(), + channelId: faker.string.uuid(), + resourceId: faker.string.alphanumeric(20), + expiration: Date.now().toString(), // Google Calendar expiration in ms + nextSyncToken: faker.string.alphanumeric(32), + lastRefreshedAt: faker.date.recent(), + lastSyncedAt: faker.date.recent(), + })); + + return { + user: userId, + google: { + calendarlist: [ + { + gCalendarId: faker.string.uuid(), + nextSyncToken: faker.string.alphanumeric(32), + lastSyncedAt: faker.date.recent(), + }, + ], + events, + }, + }; + } + + function createSyncDocWithoutEvents(userId: string): Schema_Sync { + return { + user: userId, + google: { + calendarlist: [ + { + gCalendarId: faker.string.uuid(), + nextSyncToken: faker.string.alphanumeric(32), + lastSyncedAt: faker.date.recent(), + }, + ], + events: [], + }, + }; + } + + it("migrates events watch data from sync to watch collection", async () => { + const userId = faker.database.mongodbObjectId(); + const syncDoc = createSyncDocWithEventsWatch(userId, 3); + + // Insert sync document + await mongoService.sync.insertOne(syncDoc); + + // Verify no watch data exists initially + const watchCountBefore = await mongoService.watch.countDocuments(); + expect(watchCountBefore).toBe(0); + + // Run migration + await migration.up(); + + // Verify watch data was created + const watchDocs = await mongoService.watch.find({ userId }).toArray(); + expect(watchDocs).toHaveLength(3); + + // Verify each watch document has correct data + for (let i = 0; i < watchDocs.length; i++) { + const watchDoc = watchDocs[i]; + const originalEvent = syncDoc.google.events[i]; + + expect(watchDoc).toEqual( + expect.objectContaining({ + _id: originalEvent.channelId, + userId: syncDoc.user, + resourceId: originalEvent.resourceId, + expiration: new Date(parseInt(originalEvent.expiration)), + createdAt: expect.any(Date), + }), + ); + } + + // Verify original sync data is unchanged + const syncAfter = await mongoService.sync.findOne({ user: userId }); + expect(syncAfter?.google.events).toHaveLength(3); + expect(syncAfter?.google.events[0].channelId).toBe( + syncDoc.google.events[0].channelId, + ); + }); + + it("handles multiple users with events watch data", async () => { + const user1 = faker.database.mongodbObjectId(); + const user2 = faker.database.mongodbObjectId(); + + const syncDoc1 = createSyncDocWithEventsWatch(user1, 2); + const syncDoc2 = createSyncDocWithEventsWatch(user2, 1); + + // Insert sync documents + await mongoService.sync.insertMany([syncDoc1, syncDoc2]); + + // Run migration + await migration.up(); + + // Verify watch data for both users + const user1Watches = await mongoService.watch + .find({ userId: user1 }) + .toArray(); + const user2Watches = await mongoService.watch + .find({ userId: user2 }) + .toArray(); + + expect(user1Watches).toHaveLength(2); + expect(user2Watches).toHaveLength(1); + }); + + it("skips users without events watch data", async () => { + const userId = faker.database.mongodbObjectId(); + const syncDoc = createSyncDocWithoutEvents(userId); + + // Insert sync document without events + await mongoService.sync.insertOne(syncDoc); + + // Run migration + await migration.up(); + + // Verify no watch data was created for this user + const watchCount = await mongoService.watch.countDocuments({ userId }); + expect(watchCount).toBe(0); + }); + + it("handles duplicate channel IDs gracefully", async () => { + const userId = faker.database.mongodbObjectId(); + const channelId = faker.string.uuid(); + + // Create watch document first + const existingWatch: Watch = { + _id: channelId, + userId, + resourceId: faker.string.alphanumeric(20), + expiration: faker.date.future(), + createdAt: faker.date.recent(), + }; + await mongoService.watch.insertOne(existingWatch); + + // Create sync document with same channel ID + const syncDoc = createSyncDocWithEventsWatch(userId, 1); + syncDoc.google.events[0].channelId = channelId; // Use existing channel ID + + await mongoService.sync.insertOne(syncDoc); + + // Run migration + await migration.up(); + + // Verify only one watch document exists (duplicate was skipped) + const watchDocs = await mongoService.watch + .find({ _id: channelId }) + .toArray(); + expect(watchDocs).toHaveLength(1); + + // Original watch document should be unchanged + expect(watchDocs[0]._id).toBe(existingWatch._id); + expect(watchDocs[0].resourceId).toBe(existingWatch.resourceId); + }); + + it("handles invalid expiration dates gracefully", async () => { + const userId = faker.database.mongodbObjectId(); + const syncDoc = createSyncDocWithEventsWatch(userId, 2); + + // Make one expiration invalid + syncDoc.google.events[0].expiration = "invalid-date"; + + await mongoService.sync.insertOne(syncDoc); + + // Run migration + await migration.up(); + + // Verify only valid watch data was migrated + const watchDocs = await mongoService.watch.find({ userId }).toArray(); + expect(watchDocs).toHaveLength(1); // Only the valid one + + // Verify it's the second event (first had invalid expiration) + expect(watchDocs[0]._id).toBe(syncDoc.google.events[1].channelId); + }); + + it("handles incomplete watch data gracefully", async () => { + const userId = faker.database.mongodbObjectId(); + const syncDoc = createSyncDocWithEventsWatch(userId, 2); + + // Make one event incomplete + delete (syncDoc.google.events[0] as any).channelId; + + await mongoService.sync.insertOne(syncDoc); + + // Run migration + await migration.up(); + + // Verify only complete watch data was migrated + const watchDocs = await mongoService.watch.find({ userId }).toArray(); + expect(watchDocs).toHaveLength(1); // Only the complete one + + // Verify it's the second event (first was incomplete) + expect(watchDocs[0]._id).toBe(syncDoc.google.events[1].channelId); + }); + + it("is non-destructive - does not modify watch collection on down", async () => { + // Setup some watch data + const watch: Watch = { + _id: faker.string.uuid(), + userId: faker.database.mongodbObjectId(), + resourceId: faker.string.alphanumeric(20), + expiration: faker.date.future(), + createdAt: faker.date.recent(), + }; + + await mongoService.watch.insertOne(watch); + + // Run down migration + await migration.down(); + + // Verify watch data is unchanged + const watchAfter = await mongoService.watch.findOne({ _id: watch._id }); + expect(watchAfter).toEqual(watch); + }); +}); diff --git a/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts b/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts index 69f8789ee..756ed32a7 100644 --- a/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts +++ b/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts @@ -36,7 +36,19 @@ export default class Migration implements RunnableMigration { try { // Google Calendar expiration is typically a timestamp in milliseconds const expirationMs = parseInt(eventSync.expiration); + if (isNaN(expirationMs)) { + console.warn( + `Invalid expiration format for channelId ${eventSync.channelId}: ${eventSync.expiration}`, + ); + continue; + } expirationDate = new Date(expirationMs); + if (isNaN(expirationDate.getTime())) { + console.warn( + `Invalid expiration date for channelId ${eventSync.channelId}: ${eventSync.expiration}`, + ); + continue; + } } catch { // If parsing fails, skip this watch entry console.warn( From 5e871d942ee60cdcb226a7005d7d5e22bb1dbbdc Mon Sep 17 00:00:00 2001 From: victor-enogwe <23452630+victor-enogwe@users.noreply.github.com> Date: Mon, 13 Oct 2025 16:31:02 +0100 Subject: [PATCH 4/6] :sparkles: feat(sub-calendars): update watch schema and tests --- packages/core/src/types/watch.types.test.ts | 63 +++---------------- packages/core/src/types/watch.types.ts | 15 ++--- ...25.10.13T14.18.20.watch-collection.test.ts | 58 ++++++----------- .../2025.10.13T14.18.20.watch-collection.ts | 23 +++---- ...0.13T14.22.21.migrate-events-watch-data.ts | 33 +++++----- 5 files changed, 59 insertions(+), 133 deletions(-) diff --git a/packages/core/src/types/watch.types.test.ts b/packages/core/src/types/watch.types.test.ts index b718553dc..95b46a25a 100644 --- a/packages/core/src/types/watch.types.test.ts +++ b/packages/core/src/types/watch.types.test.ts @@ -1,15 +1,10 @@ import { faker } from "@faker-js/faker"; -import { - Watch, - WatchInput, - WatchSchema, - WatchSchemaStrict, -} from "@core/types/watch.types"; +import { Watch, WatchSchema } from "@core/types/watch.types"; describe("Watch Types", () => { const validWatch: Watch = { _id: faker.string.uuid(), - userId: faker.database.mongodbObjectId(), + user: faker.database.mongodbObjectId(), resourceId: faker.string.alphanumeric(20), expiration: faker.date.future(), createdAt: new Date(), @@ -30,26 +25,26 @@ describe("Watch Types", () => { expect(parsed.createdAt).toBeInstanceOf(Date); }); - it("accepts valid MongoDB ObjectId for userId", () => { + it("accepts valid MongoDB ObjectId for user", () => { const watchData = { ...validWatch, - userId: faker.database.mongodbObjectId(), + user: faker.database.mongodbObjectId(), }; expect(() => WatchSchema.parse(watchData)).not.toThrow(); }); - it("rejects invalid MongoDB ObjectId for userId", () => { + it("rejects invalid MongoDB ObjectId for user", () => { const watchData = { ...validWatch, - userId: "invalid-object-id", + user: "invalid-object-id", }; expect(() => WatchSchema.parse(watchData)).toThrow(); }); it("requires all mandatory fields", () => { - const requiredFields = ["_id", "userId", "resourceId", "expiration"]; + const requiredFields = ["_id", "user", "resourceId", "expiration"]; requiredFields.forEach((field) => { const incompleteWatch = { ...validWatch }; @@ -77,48 +72,4 @@ describe("Watch Types", () => { expect(() => WatchSchema.parse(watchData)).toThrow(); }); }); - - describe("WatchSchemaStrict", () => { - it("rejects additional properties when using strict schema", () => { - const watchWithExtra = { - ...validWatch, - extraProperty: "should-not-be-allowed", - }; - - expect(() => WatchSchemaStrict.parse(watchWithExtra)).toThrow(); - }); - - it("accepts valid watch data with strict schema", () => { - expect(() => WatchSchemaStrict.parse(validWatch)).not.toThrow(); - }); - }); - - describe("WatchInput type", () => { - it("allows creating watch input without createdAt", () => { - const watchInput: WatchInput = { - _id: faker.string.uuid(), - userId: faker.database.mongodbObjectId(), - resourceId: faker.string.alphanumeric(20), - expiration: faker.date.future(), - }; - - // This should compile without errors - expect(watchInput).toBeDefined(); - expect(watchInput.createdAt).toBeUndefined(); - }); - - it("allows creating watch input with createdAt", () => { - const watchInput: WatchInput = { - _id: faker.string.uuid(), - userId: faker.database.mongodbObjectId(), - resourceId: faker.string.alphanumeric(20), - expiration: faker.date.future(), - createdAt: new Date(), - }; - - // This should compile without errors - expect(watchInput).toBeDefined(); - expect(watchInput.createdAt).toBeInstanceOf(Date); - }); - }); }); diff --git a/packages/core/src/types/watch.types.ts b/packages/core/src/types/watch.types.ts index f2a7fbeb4..3eb70e79f 100644 --- a/packages/core/src/types/watch.types.ts +++ b/packages/core/src/types/watch.types.ts @@ -10,18 +10,13 @@ import { IDSchemaV4 } from "@core/types/type.utils"; */ export const WatchSchema = z.object({ _id: z.string(), // channel_id - unique identifier for the notification channel - userId: IDSchemaV4, // user who owns this watch channel + user: IDSchemaV4, // user who owns this watch channel resourceId: z.string(), // Google Calendar resource identifier expiration: z.date(), // when the channel expires - createdAt: z.date().default(() => new Date()), // when this watch was created + createdAt: z + .date() + .optional() + .default(() => new Date()), // when this watch was created }); export type Watch = z.infer; - -// Type for creating a new watch (without auto-generated fields) -export type WatchInput = Omit & { - createdAt?: Date; -}; - -// Schema for database storage (with strict validation) -export const WatchSchemaStrict = WatchSchema.strict(); diff --git a/packages/scripts/src/__tests__/integration/2025.10.13T14.18.20.watch-collection.test.ts b/packages/scripts/src/__tests__/integration/2025.10.13T14.18.20.watch-collection.test.ts index fcda1cbad..245dddc47 100644 --- a/packages/scripts/src/__tests__/integration/2025.10.13T14.18.20.watch-collection.test.ts +++ b/packages/scripts/src/__tests__/integration/2025.10.13T14.18.20.watch-collection.test.ts @@ -1,7 +1,7 @@ import { faker } from "@faker-js/faker"; import { zodToMongoSchema } from "@scripts/common/zod-to-mongo-schema"; import Migration from "@scripts/migrations/2025.10.13T14.18.20.watch-collection"; -import { Watch, WatchSchemaStrict } from "@core/types/watch.types"; +import { Watch, WatchSchema } from "@core/types/watch.types"; import { cleanupCollections, cleanupTestDb, @@ -22,7 +22,7 @@ describe("2025.10.13T14.18.20.watch-collection", () => { function generateWatch(): Watch { return { _id: faker.string.uuid(), - userId: faker.database.mongodbObjectId(), + user: faker.database.mongodbObjectId(), resourceId: faker.string.alphanumeric(20), expiration: faker.date.future(), createdAt: faker.date.recent(), @@ -32,7 +32,7 @@ describe("2025.10.13T14.18.20.watch-collection", () => { async function validateUpMigration() { const indexes = await mongoService.watch.indexes(); const collectionInfo = await mongoService.watch.options(); - const $jsonSchema = zodToMongoSchema(WatchSchemaStrict); + const $jsonSchema = zodToMongoSchema(WatchSchema); expect(collectionInfo["validationLevel"]).toBe("strict"); expect(collectionInfo["validator"]).toBeDefined(); @@ -43,12 +43,12 @@ describe("2025.10.13T14.18.20.watch-collection", () => { expect.arrayContaining([ expect.objectContaining({ name: "_id_", key: { _id: 1 } }), expect.objectContaining({ - name: `${collectionName}_userId_index`, - key: { userId: 1 }, + name: `${collectionName}_user_index`, + key: { user: 1 }, }), expect.objectContaining({ - name: `${collectionName}_userId_expiration_index`, - key: { userId: 1, expiration: 1 }, + name: `${collectionName}_user_expiration_index`, + key: { user: 1, expiration: 1 }, }), ]), ); @@ -110,20 +110,10 @@ describe("2025.10.13T14.18.20.watch-collection", () => { }); describe("mongo $jsonSchema validation", () => { - function generateValidWatch() { - return { - _id: faker.string.uuid(), - userId: faker.database.mongodbObjectId(), - resourceId: faker.string.alphanumeric(20), - expiration: new Date(faker.date.future()), - createdAt: new Date(faker.date.recent()), - }; - } - beforeEach(migration.up.bind(migration)); it("allows valid watch documents", async () => { - const watch = generateValidWatch(); + const watch = generateWatch(); await expect(mongoService.watch.insertOne(watch)).resolves.toMatchObject({ acknowledged: true, @@ -132,24 +122,20 @@ describe("2025.10.13T14.18.20.watch-collection", () => { }); it("rejects documents with missing required fields", async () => { - const incompleteWatch = { - _id: faker.string.uuid(), - userId: faker.database.mongodbObjectId(), - // missing resourceId and expiration - createdAt: new Date(), - }; + const incompleteWatch = generateWatch(); + + delete (incompleteWatch as Partial).resourceId; + delete (incompleteWatch as Partial).expiration; await expect( mongoService.watch.insertOne(incompleteWatch), ).rejects.toThrow(); }); - it("rejects documents with missing userId", async () => { - const watchWithoutUserId = { - ...generateValidWatch(), - }; - // @ts-expect-error testing missing userId field - delete watchWithoutUserId.userId; + it("rejects documents with missing user", async () => { + const watchWithoutUserId = generateWatch(); + + delete (watchWithoutUserId as Partial).user; await expect( mongoService.watch.insertOne(watchWithoutUserId), @@ -158,7 +144,7 @@ describe("2025.10.13T14.18.20.watch-collection", () => { it("rejects documents with additional properties", async () => { const watchWithExtra = { - ...generateValidWatch(), + ...generateWatch(), extraProperty: "should-not-be-allowed", }; @@ -168,15 +154,11 @@ describe("2025.10.13T14.18.20.watch-collection", () => { }); it("enforces unique constraint on _id (channelId)", async () => { - const watch1 = generateValidWatch(); - const watch2 = { - ...generateValidWatch(), - _id: watch1._id, // Same channelId - }; + const watch = generateWatch(); - await mongoService.watch.insertOne(watch1); + await mongoService.watch.insertOne(watch); - await expect(mongoService.watch.insertOne(watch2)).rejects.toThrow(); + await expect(mongoService.watch.insertOne(watch)).rejects.toThrow(); }); }); }); diff --git a/packages/scripts/src/migrations/2025.10.13T14.18.20.watch-collection.ts b/packages/scripts/src/migrations/2025.10.13T14.18.20.watch-collection.ts index e20cea31e..39e9eab03 100644 --- a/packages/scripts/src/migrations/2025.10.13T14.18.20.watch-collection.ts +++ b/packages/scripts/src/migrations/2025.10.13T14.18.20.watch-collection.ts @@ -1,7 +1,7 @@ import type { RunnableMigration } from "umzug"; import { MigrationContext } from "@scripts/common/cli.types"; import { zodToMongoSchema } from "@scripts/common/zod-to-mongo-schema"; -import { WatchSchemaStrict } from "@core/types/watch.types"; +import { WatchSchema } from "@core/types/watch.types"; import mongoService from "@backend/common/services/mongo.service"; export default class Migration implements RunnableMigration { @@ -11,8 +11,7 @@ export default class Migration implements RunnableMigration { async up(): Promise { const { collectionName } = mongoService.watch; const exists = await mongoService.collectionExists(collectionName); - - const $jsonSchema = zodToMongoSchema(WatchSchemaStrict); + const $jsonSchema = zodToMongoSchema(WatchSchema); if (exists) { // do not run in session @@ -28,18 +27,16 @@ export default class Migration implements RunnableMigration { }); } - // _id is unique by default in MongoDB, no need to create explicit index - - // Create index on userId for efficient user-based queries + // Create index on user for efficient user-based queries await mongoService.watch.createIndex( - { userId: 1 }, - { name: `${collectionName}_userId_index` }, + { user: 1 }, + { name: `${collectionName}_user_index` }, ); - // Create compound index on userId and expiration for cleanup operations + // Create compound index on user and expiration for cleanup operations await mongoService.watch.createIndex( - { userId: 1, expiration: 1 }, - { name: `${collectionName}_userId_expiration_index` }, + { user: 1, expiration: 1 }, + { name: `${collectionName}_user_expiration_index` }, ); } @@ -57,9 +54,9 @@ export default class Migration implements RunnableMigration { }); // _id index is built-in, no need to drop - await mongoService.watch.dropIndex(`${collectionName}_userId_index`); + await mongoService.watch.dropIndex(`${collectionName}_user_index`); await mongoService.watch.dropIndex( - `${collectionName}_userId_expiration_index`, + `${collectionName}_user_expiration_index`, ); } } diff --git a/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts b/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts index 756ed32a7..5d64ad8fd 100644 --- a/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts +++ b/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts @@ -1,19 +1,21 @@ import type { RunnableMigration } from "umzug"; import { MigrationContext } from "@scripts/common/cli.types"; -import { Schema_Sync } from "@core/types/sync.types"; import { Watch } from "@core/types/watch.types"; import mongoService from "@backend/common/services/mongo.service"; +import dayjs from "../../../core/src/util/date/dayjs"; export default class Migration implements RunnableMigration { readonly name: string = "2025.10.13T14.22.21.migrate-events-watch-data"; readonly path: string = "2025.10.13T14.22.21.migrate-events-watch-data.ts"; async up(): Promise { + const session = await mongoService.startSession(); // This is a non-destructive migration to copy events watch data from sync collection to watch collection - const cursor = mongoService.sync.find({ - "google.events": { $exists: true, $ne: [] }, - }); + const cursor = mongoService.sync.find( + { "google.events": { $exists: true, $ne: [] } }, + { batchSize: 100, session }, + ); let migratedCount = 0; @@ -33,22 +35,19 @@ export default class Migration implements RunnableMigration { // Convert expiration string to Date let expirationDate: Date; + try { // Google Calendar expiration is typically a timestamp in milliseconds const expirationMs = parseInt(eventSync.expiration); + if (isNaN(expirationMs)) { console.warn( - `Invalid expiration format for channelId ${eventSync.channelId}: ${eventSync.expiration}`, - ); - continue; - } - expirationDate = new Date(expirationMs); - if (isNaN(expirationDate.getTime())) { - console.warn( - `Invalid expiration date for channelId ${eventSync.channelId}: ${eventSync.expiration}`, + `Invalid expiration ms for channelId ${eventSync.channelId}: ${eventSync.expiration}`, ); continue; } + + expirationDate = dayjs(expirationMs).toDate(); } catch { // If parsing fails, skip this watch entry console.warn( @@ -59,7 +58,7 @@ export default class Migration implements RunnableMigration { const watchDoc: Watch = { _id: eventSync.channelId, - userId: syncDoc.user, + user: syncDoc.user, resourceId: eventSync.resourceId, expiration: expirationDate, createdAt: new Date(), // Set current time as creation time for migration @@ -71,11 +70,13 @@ export default class Migration implements RunnableMigration { if (watchDocuments.length > 0) { try { // Use insertMany with ordered: false to continue on duplicates - await mongoService.watch.insertMany(watchDocuments, { + const result = await mongoService.watch.insertMany(watchDocuments, { ordered: false, + session, }); - migratedCount += watchDocuments.length; - } catch (error: any) { + + migratedCount += result.insertedCount; + } catch (error: unknown) { // Log errors but continue migration (some channels might already exist) if (error?.writeErrors) { const duplicateErrors = error.writeErrors.filter( From 066271b9334246af1446f11b515c2671a96c098e Mon Sep 17 00:00:00 2001 From: victor-enogwe <23452630+victor-enogwe@users.noreply.github.com> Date: Mon, 13 Oct 2025 22:56:08 +0100 Subject: [PATCH 5/6] :sparkles feat(sub-calendars): update watch collection schema and tests --- .../src/__tests__/drivers/util.driver.ts | 39 ++- .../src/common/services/mongo.service.ts | 18 +- .../core/src/types/calendar.types.test.ts | 3 +- packages/core/src/types/calendar.types.ts | 5 +- packages/core/src/types/type.utils.ts | 11 + packages/core/src/types/watch.types.test.ts | 18 +- packages/core/src/types/watch.types.ts | 6 +- ...025.10.03T01.19.59.calendar-schema.test.ts | 3 +- ...25.10.13T14.18.20.watch-collection.test.ts | 15 +- ...14.22.21.migrate-events-watch-data.test.ts | 247 ------------------ ...3T14.22.21.migrate-sync-watch-data.test.ts | 99 +++++++ .../src/common/zod-to-mongo-schema.test.ts | 7 +- .../scripts/src/common/zod-to-mongo-schema.ts | 14 +- ...0.13T14.22.21.migrate-events-watch-data.ts | 126 --------- ....10.13T14.22.21.migrate-sync-watch-data.ts | 84 ++++++ 15 files changed, 268 insertions(+), 427 deletions(-) delete mode 100644 packages/scripts/src/__tests__/integration/2025.10.13T14.22.21.migrate-events-watch-data.test.ts create mode 100644 packages/scripts/src/__tests__/integration/2025.10.13T14.22.21.migrate-sync-watch-data.test.ts delete mode 100644 packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts create mode 100644 packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-sync-watch-data.ts diff --git a/packages/backend/src/__tests__/drivers/util.driver.ts b/packages/backend/src/__tests__/drivers/util.driver.ts index 37f39d109..211967588 100644 --- a/packages/backend/src/__tests__/drivers/util.driver.ts +++ b/packages/backend/src/__tests__/drivers/util.driver.ts @@ -1,8 +1,11 @@ -import { WithId } from "mongodb"; +import { ObjectId, WithId } from "mongodb"; +import { faker } from "@faker-js/faker"; +import { Schema_Sync } from "@core/types/sync.types"; import { Schema_User } from "@core/types/user.types"; import { SyncDriver } from "@backend/__tests__/drivers/sync.driver"; import { UserDriver } from "@backend/__tests__/drivers/user.driver"; import { WaitListDriver } from "@backend/__tests__/drivers/waitlist.driver"; +import mongoService from "@backend/common/services/mongo.service"; export class UtilDriver { static async setupTestUser(): Promise<{ user: WithId }> { @@ -17,4 +20,38 @@ export class UtilDriver { return { user }; } + + static async generateV0SyncData( + numUsers = 3, + ): Promise>>> { + const users = await Promise.all( + Array.from({ length: numUsers }, UserDriver.createUser), + ); + + const data = users.map((user) => ({ + _id: new ObjectId(), + user: user._id.toString(), + google: { + events: [ + { + resourceId: faker.string.ulid(), + gCalendarId: user.email, + lastSyncedAt: faker.date.past(), + nextSyncToken: faker.string.alphanumeric(32), + channelId: faker.string.uuid(), + expiration: faker.date.future().getTime().toString(), + }, + ], + calendarlist: [ + { + nextSyncToken: faker.string.alphanumeric(32), + gCalendarId: user.email, + lastSyncedAt: faker.date.past(), + }, + ], + }, + })); + + return mongoService.sync.insertMany(data).then(() => data); + } } diff --git a/packages/backend/src/common/services/mongo.service.ts b/packages/backend/src/common/services/mongo.service.ts index 2a0719464..11c4d4bdc 100644 --- a/packages/backend/src/common/services/mongo.service.ts +++ b/packages/backend/src/common/services/mongo.service.ts @@ -11,14 +11,14 @@ import { } from "mongodb"; import { Logger } from "@core/logger/winston.logger"; import { - CompassCalendar, - Schema_CalendarList as Schema_Calendar, + Schema_CalendarList as Schema_CalList, + Schema_Calendar, } from "@core/types/calendar.types"; import { Schema_Event } from "@core/types/event.types"; import { Schema_Sync } from "@core/types/sync.types"; import { Schema_User } from "@core/types/user.types"; import { Schema_Waitlist } from "@core/types/waitlist/waitlist.types"; -import { Watch } from "@core/types/watch.types"; +import { Schema_Watch } from "@core/types/watch.types"; import { Collections } from "@backend/common/constants/collections"; import { ENV } from "@backend/common/constants/env.constants"; import { waitUntilEvent } from "@backend/common/helpers/common.util"; @@ -28,13 +28,13 @@ const logger = Logger("app:mongo.service"); interface InternalClient { db: Db; client: MongoClient; - calendar: Collection; - calendarList: Collection; + calendar: Collection; + calendarList: Collection; event: Collection>; sync: Collection; user: Collection; waitlist: Collection; - watch: Collection; + watch: Collection>; } class MongoService { @@ -138,13 +138,13 @@ class MongoService { return { db, client, - calendar: db.collection(Collections.CALENDAR), - calendarList: db.collection(Collections.CALENDARLIST), + calendar: db.collection(Collections.CALENDAR), + calendarList: db.collection(Collections.CALENDARLIST), event: db.collection>(Collections.EVENT), sync: db.collection(Collections.SYNC), user: db.collection(Collections.USER), waitlist: db.collection(Collections.WAITLIST), - watch: db.collection(Collections.WATCH), + watch: db.collection>(Collections.WATCH), }; } diff --git a/packages/core/src/types/calendar.types.test.ts b/packages/core/src/types/calendar.types.test.ts index 9f805c2a6..271c610f4 100644 --- a/packages/core/src/types/calendar.types.test.ts +++ b/packages/core/src/types/calendar.types.test.ts @@ -1,3 +1,4 @@ +import { ObjectId } from "bson"; import { faker } from "@faker-js/faker"; import { CompassCalendarSchema, @@ -65,7 +66,7 @@ describe("Calendar Types", () => { describe("CompassCalendarSchema", () => { const compassCalendar = { - _id: faker.database.mongodbObjectId(), + _id: new ObjectId(), user: faker.database.mongodbObjectId(), backgroundColor: gCalendar.backgroundColor!, color: gCalendar.foregroundColor!, diff --git a/packages/core/src/types/calendar.types.ts b/packages/core/src/types/calendar.types.ts index 57d150ad9..c81a70250 100644 --- a/packages/core/src/types/calendar.types.ts +++ b/packages/core/src/types/calendar.types.ts @@ -5,6 +5,7 @@ import { IDSchemaV4, RGBHexSchema, TimezoneSchema, + zObjectId, } from "@core/types/type.utils"; // @deprecated - will be replaced by Schema_Calendar @@ -55,7 +56,7 @@ export const GoogleCalendarMetadataSchema = z.object({ }); export const CompassCalendarSchema = z.object({ - _id: IDSchemaV4, + _id: zObjectId, user: IDSchemaV4, backgroundColor: RGBHexSchema, color: RGBHexSchema, @@ -67,4 +68,4 @@ export const CompassCalendarSchema = z.object({ metadata: GoogleCalendarMetadataSchema, // use union when other providers present }); -export type CompassCalendar = z.infer; +export type Schema_Calendar = z.infer; diff --git a/packages/core/src/types/type.utils.ts b/packages/core/src/types/type.utils.ts index 2fc7d381d..b7c2a2d81 100644 --- a/packages/core/src/types/type.utils.ts +++ b/packages/core/src/types/type.utils.ts @@ -1,6 +1,7 @@ import { ObjectId } from "bson"; import { z } from "zod"; import { z as zod4 } from "zod/v4"; +import { z as zod4Mini } from "zod/v4-mini"; export type KeyOfType = keyof { [P in keyof T as T[P] extends V ? P : never]: unknown; @@ -19,6 +20,16 @@ export const IDSchemaV4 = zod4.string().refine(ObjectId.isValid, { message: "Invalid id", }); +export const zObjectIdMini = zod4Mini.pipe( + zod4Mini.custom(ObjectId.isValid), + zod4Mini.transform((v) => new ObjectId(v)), +); + +export const zObjectId = zod4.pipe( + zod4.custom((v) => ObjectId.isValid(v as string)), + zod4.transform((v) => new ObjectId(v)), +); + export const TimezoneSchema = zod4.string().refine( (timeZone) => { try { diff --git a/packages/core/src/types/watch.types.test.ts b/packages/core/src/types/watch.types.test.ts index 95b46a25a..7d61e480c 100644 --- a/packages/core/src/types/watch.types.test.ts +++ b/packages/core/src/types/watch.types.test.ts @@ -1,9 +1,10 @@ +import { ObjectId } from "bson"; import { faker } from "@faker-js/faker"; -import { Watch, WatchSchema } from "@core/types/watch.types"; +import { Schema_Watch, WatchSchema } from "@core/types/watch.types"; describe("Watch Types", () => { - const validWatch: Watch = { - _id: faker.string.uuid(), + const validWatch: Schema_Watch = { + _id: new ObjectId(), user: faker.database.mongodbObjectId(), resourceId: faker.string.alphanumeric(20), expiration: faker.date.future(), @@ -48,21 +49,12 @@ describe("Watch Types", () => { requiredFields.forEach((field) => { const incompleteWatch = { ...validWatch }; - delete incompleteWatch[field as keyof Watch]; + delete incompleteWatch[field as keyof Schema_Watch]; expect(() => WatchSchema.parse(incompleteWatch)).toThrow(); }); }); - it("accepts string for _id (channelId)", () => { - const watchData = { - ...validWatch, - _id: "test-channel-id-123", - }; - - expect(() => WatchSchema.parse(watchData)).not.toThrow(); - }); - it("requires expiration to be a Date", () => { const watchData = { ...validWatch, diff --git a/packages/core/src/types/watch.types.ts b/packages/core/src/types/watch.types.ts index 3eb70e79f..0aa663d02 100644 --- a/packages/core/src/types/watch.types.ts +++ b/packages/core/src/types/watch.types.ts @@ -1,5 +1,5 @@ import { z } from "zod/v4"; -import { IDSchemaV4 } from "@core/types/type.utils"; +import { IDSchemaV4, zObjectId } from "@core/types/type.utils"; /** * Watch collection schema for Google Calendar push notification channels @@ -9,7 +9,7 @@ import { IDSchemaV4 } from "@core/types/type.utils"; * expiration, deletion) separately from sync data. */ export const WatchSchema = z.object({ - _id: z.string(), // channel_id - unique identifier for the notification channel + _id: zObjectId, // channel_id - unique identifier for the notification channel user: IDSchemaV4, // user who owns this watch channel resourceId: z.string(), // Google Calendar resource identifier expiration: z.date(), // when the channel expires @@ -19,4 +19,4 @@ export const WatchSchema = z.object({ .default(() => new Date()), // when this watch was created }); -export type Watch = z.infer; +export type Schema_Watch = z.infer; diff --git a/packages/scripts/src/__tests__/integration/2025.10.03T01.19.59.calendar-schema.test.ts b/packages/scripts/src/__tests__/integration/2025.10.03T01.19.59.calendar-schema.test.ts index 57794ed99..bc857e7fa 100644 --- a/packages/scripts/src/__tests__/integration/2025.10.03T01.19.59.calendar-schema.test.ts +++ b/packages/scripts/src/__tests__/integration/2025.10.03T01.19.59.calendar-schema.test.ts @@ -1,3 +1,4 @@ +import { ObjectId } from "bson"; import { faker } from "@faker-js/faker"; import { zodToMongoSchema } from "@scripts/common/zod-to-mongo-schema"; import Migration from "@scripts/migrations/2025.10.03T01.19.59.calendar-schema"; @@ -121,7 +122,7 @@ describe("2025.10.03T01.19.59.calendar-schema", () => { const gCalendar = GoogleCalendarMetadataSchema.parse(gCalendarEntry); return CompassCalendarSchema.parse({ - _id: faker.database.mongodbObjectId(), + _id: new ObjectId(), user: faker.database.mongodbObjectId(), backgroundColor: gCalendarEntry.backgroundColor!, color: gCalendarEntry.foregroundColor!, diff --git a/packages/scripts/src/__tests__/integration/2025.10.13T14.18.20.watch-collection.test.ts b/packages/scripts/src/__tests__/integration/2025.10.13T14.18.20.watch-collection.test.ts index 245dddc47..f660e9e4c 100644 --- a/packages/scripts/src/__tests__/integration/2025.10.13T14.18.20.watch-collection.test.ts +++ b/packages/scripts/src/__tests__/integration/2025.10.13T14.18.20.watch-collection.test.ts @@ -1,7 +1,8 @@ +import { ObjectId, WithId } from "mongodb"; import { faker } from "@faker-js/faker"; import { zodToMongoSchema } from "@scripts/common/zod-to-mongo-schema"; import Migration from "@scripts/migrations/2025.10.13T14.18.20.watch-collection"; -import { Watch, WatchSchema } from "@core/types/watch.types"; +import { Schema_Watch, WatchSchema } from "@core/types/watch.types"; import { cleanupCollections, cleanupTestDb, @@ -10,6 +11,8 @@ import { import { Collections } from "@backend/common/constants/collections"; import mongoService from "@backend/common/services/mongo.service"; +type PartialWatch = Partial>>; + describe("2025.10.13T14.18.20.watch-collection", () => { const migration = new Migration(); const collectionName = Collections.WATCH; @@ -19,9 +22,9 @@ describe("2025.10.13T14.18.20.watch-collection", () => { afterEach(() => mongoService.watch.drop()); afterAll(cleanupTestDb); - function generateWatch(): Watch { + function generateWatch(): WithId> { return { - _id: faker.string.uuid(), + _id: new ObjectId(), user: faker.database.mongodbObjectId(), resourceId: faker.string.alphanumeric(20), expiration: faker.date.future(), @@ -124,8 +127,8 @@ describe("2025.10.13T14.18.20.watch-collection", () => { it("rejects documents with missing required fields", async () => { const incompleteWatch = generateWatch(); - delete (incompleteWatch as Partial).resourceId; - delete (incompleteWatch as Partial).expiration; + delete (incompleteWatch as PartialWatch).resourceId; + delete (incompleteWatch as PartialWatch).expiration; await expect( mongoService.watch.insertOne(incompleteWatch), @@ -135,7 +138,7 @@ describe("2025.10.13T14.18.20.watch-collection", () => { it("rejects documents with missing user", async () => { const watchWithoutUserId = generateWatch(); - delete (watchWithoutUserId as Partial).user; + delete (watchWithoutUserId as PartialWatch).user; await expect( mongoService.watch.insertOne(watchWithoutUserId), diff --git a/packages/scripts/src/__tests__/integration/2025.10.13T14.22.21.migrate-events-watch-data.test.ts b/packages/scripts/src/__tests__/integration/2025.10.13T14.22.21.migrate-events-watch-data.test.ts deleted file mode 100644 index 74a0a5d05..000000000 --- a/packages/scripts/src/__tests__/integration/2025.10.13T14.22.21.migrate-events-watch-data.test.ts +++ /dev/null @@ -1,247 +0,0 @@ -import { faker } from "@faker-js/faker"; -import Migration from "@scripts/migrations/2025.10.13T14.22.21.migrate-events-watch-data"; -import { Schema_Sync } from "@core/types/sync.types"; -import { Watch } from "@core/types/watch.types"; -import { - cleanupCollections, - cleanupTestDb, - setupTestDb, -} from "@backend/__tests__/helpers/mock.db.setup"; -import mongoService from "@backend/common/services/mongo.service"; - -describe("2025.10.13T14.22.21.migrate-events-watch-data", () => { - const migration = new Migration(); - - beforeAll(setupTestDb); - beforeEach(async () => { - await cleanupCollections(); - // Ensure watch collection is clean between tests - try { - await mongoService.watch.deleteMany({}); - } catch { - // Collection might not exist yet, which is fine - } - }); - afterAll(cleanupTestDb); - - function createSyncDocWithEventsWatch( - userId: string, - eventsCount = 2, - ): Schema_Sync { - const events = Array.from({ length: eventsCount }, () => ({ - gCalendarId: faker.string.uuid(), - channelId: faker.string.uuid(), - resourceId: faker.string.alphanumeric(20), - expiration: Date.now().toString(), // Google Calendar expiration in ms - nextSyncToken: faker.string.alphanumeric(32), - lastRefreshedAt: faker.date.recent(), - lastSyncedAt: faker.date.recent(), - })); - - return { - user: userId, - google: { - calendarlist: [ - { - gCalendarId: faker.string.uuid(), - nextSyncToken: faker.string.alphanumeric(32), - lastSyncedAt: faker.date.recent(), - }, - ], - events, - }, - }; - } - - function createSyncDocWithoutEvents(userId: string): Schema_Sync { - return { - user: userId, - google: { - calendarlist: [ - { - gCalendarId: faker.string.uuid(), - nextSyncToken: faker.string.alphanumeric(32), - lastSyncedAt: faker.date.recent(), - }, - ], - events: [], - }, - }; - } - - it("migrates events watch data from sync to watch collection", async () => { - const userId = faker.database.mongodbObjectId(); - const syncDoc = createSyncDocWithEventsWatch(userId, 3); - - // Insert sync document - await mongoService.sync.insertOne(syncDoc); - - // Verify no watch data exists initially - const watchCountBefore = await mongoService.watch.countDocuments(); - expect(watchCountBefore).toBe(0); - - // Run migration - await migration.up(); - - // Verify watch data was created - const watchDocs = await mongoService.watch.find({ userId }).toArray(); - expect(watchDocs).toHaveLength(3); - - // Verify each watch document has correct data - for (let i = 0; i < watchDocs.length; i++) { - const watchDoc = watchDocs[i]; - const originalEvent = syncDoc.google.events[i]; - - expect(watchDoc).toEqual( - expect.objectContaining({ - _id: originalEvent.channelId, - userId: syncDoc.user, - resourceId: originalEvent.resourceId, - expiration: new Date(parseInt(originalEvent.expiration)), - createdAt: expect.any(Date), - }), - ); - } - - // Verify original sync data is unchanged - const syncAfter = await mongoService.sync.findOne({ user: userId }); - expect(syncAfter?.google.events).toHaveLength(3); - expect(syncAfter?.google.events[0].channelId).toBe( - syncDoc.google.events[0].channelId, - ); - }); - - it("handles multiple users with events watch data", async () => { - const user1 = faker.database.mongodbObjectId(); - const user2 = faker.database.mongodbObjectId(); - - const syncDoc1 = createSyncDocWithEventsWatch(user1, 2); - const syncDoc2 = createSyncDocWithEventsWatch(user2, 1); - - // Insert sync documents - await mongoService.sync.insertMany([syncDoc1, syncDoc2]); - - // Run migration - await migration.up(); - - // Verify watch data for both users - const user1Watches = await mongoService.watch - .find({ userId: user1 }) - .toArray(); - const user2Watches = await mongoService.watch - .find({ userId: user2 }) - .toArray(); - - expect(user1Watches).toHaveLength(2); - expect(user2Watches).toHaveLength(1); - }); - - it("skips users without events watch data", async () => { - const userId = faker.database.mongodbObjectId(); - const syncDoc = createSyncDocWithoutEvents(userId); - - // Insert sync document without events - await mongoService.sync.insertOne(syncDoc); - - // Run migration - await migration.up(); - - // Verify no watch data was created for this user - const watchCount = await mongoService.watch.countDocuments({ userId }); - expect(watchCount).toBe(0); - }); - - it("handles duplicate channel IDs gracefully", async () => { - const userId = faker.database.mongodbObjectId(); - const channelId = faker.string.uuid(); - - // Create watch document first - const existingWatch: Watch = { - _id: channelId, - userId, - resourceId: faker.string.alphanumeric(20), - expiration: faker.date.future(), - createdAt: faker.date.recent(), - }; - await mongoService.watch.insertOne(existingWatch); - - // Create sync document with same channel ID - const syncDoc = createSyncDocWithEventsWatch(userId, 1); - syncDoc.google.events[0].channelId = channelId; // Use existing channel ID - - await mongoService.sync.insertOne(syncDoc); - - // Run migration - await migration.up(); - - // Verify only one watch document exists (duplicate was skipped) - const watchDocs = await mongoService.watch - .find({ _id: channelId }) - .toArray(); - expect(watchDocs).toHaveLength(1); - - // Original watch document should be unchanged - expect(watchDocs[0]._id).toBe(existingWatch._id); - expect(watchDocs[0].resourceId).toBe(existingWatch.resourceId); - }); - - it("handles invalid expiration dates gracefully", async () => { - const userId = faker.database.mongodbObjectId(); - const syncDoc = createSyncDocWithEventsWatch(userId, 2); - - // Make one expiration invalid - syncDoc.google.events[0].expiration = "invalid-date"; - - await mongoService.sync.insertOne(syncDoc); - - // Run migration - await migration.up(); - - // Verify only valid watch data was migrated - const watchDocs = await mongoService.watch.find({ userId }).toArray(); - expect(watchDocs).toHaveLength(1); // Only the valid one - - // Verify it's the second event (first had invalid expiration) - expect(watchDocs[0]._id).toBe(syncDoc.google.events[1].channelId); - }); - - it("handles incomplete watch data gracefully", async () => { - const userId = faker.database.mongodbObjectId(); - const syncDoc = createSyncDocWithEventsWatch(userId, 2); - - // Make one event incomplete - delete (syncDoc.google.events[0] as any).channelId; - - await mongoService.sync.insertOne(syncDoc); - - // Run migration - await migration.up(); - - // Verify only complete watch data was migrated - const watchDocs = await mongoService.watch.find({ userId }).toArray(); - expect(watchDocs).toHaveLength(1); // Only the complete one - - // Verify it's the second event (first was incomplete) - expect(watchDocs[0]._id).toBe(syncDoc.google.events[1].channelId); - }); - - it("is non-destructive - does not modify watch collection on down", async () => { - // Setup some watch data - const watch: Watch = { - _id: faker.string.uuid(), - userId: faker.database.mongodbObjectId(), - resourceId: faker.string.alphanumeric(20), - expiration: faker.date.future(), - createdAt: faker.date.recent(), - }; - - await mongoService.watch.insertOne(watch); - - // Run down migration - await migration.down(); - - // Verify watch data is unchanged - const watchAfter = await mongoService.watch.findOne({ _id: watch._id }); - expect(watchAfter).toEqual(watch); - }); -}); diff --git a/packages/scripts/src/__tests__/integration/2025.10.13T14.22.21.migrate-sync-watch-data.test.ts b/packages/scripts/src/__tests__/integration/2025.10.13T14.22.21.migrate-sync-watch-data.test.ts new file mode 100644 index 000000000..a48f4d85a --- /dev/null +++ b/packages/scripts/src/__tests__/integration/2025.10.13T14.22.21.migrate-sync-watch-data.test.ts @@ -0,0 +1,99 @@ +import { ObjectId } from "mongodb"; +import { faker } from "@faker-js/faker"; +import { MigratorType } from "@scripts/common/cli.types"; +import WatchMigration from "@scripts/migrations/2025.10.13T14.18.20.watch-collection"; +import Migration from "@scripts/migrations/2025.10.13T14.22.21.migrate-sync-watch-data"; +import { Logger } from "@core/logger/winston.logger"; +import { UtilDriver } from "@backend/__tests__/drivers/util.driver"; +import { + cleanupCollections, + cleanupTestDb, + setupTestDb, +} from "@backend/__tests__/helpers/mock.db.setup"; +import mongoService from "@backend/common/services/mongo.service"; + +describe("2025.10.13T14.22.21.migrate-sync-watch-data", () => { + const migration = new Migration(); + const syncCount = faker.number.int({ min: 1, max: 5 }); + + const migrationContext = { + name: migration.name, + context: { + logger: Logger(""), + migratorType: MigratorType.MIGRATION, + unsafe: false, + }, + }; + + beforeAll(setupTestDb); + beforeEach(WatchMigration.prototype.up); + beforeEach(UtilDriver.generateV0SyncData.bind(null, syncCount)); + afterEach(cleanupCollections); + afterEach(() => mongoService.watch.drop()); + afterAll(cleanupTestDb); + + it("migrates events watch data from sync to watch collection", async () => { + const syncDocs = await mongoService.sync.find().toArray(); + + // Verify only exact sync data count exists initially + expect(syncDocs).toHaveLength(syncCount); + + // Run migration + await migration.up(migrationContext); + + // Verify watch data was created + const watchDocs = await mongoService.watch.find().toArray(); + + // Verify each watch document has correct data + // calendarlist will be absent since we do not currently store resourceId + expect(watchDocs).toEqual( + expect.arrayContaining( + syncDocs.flatMap(({ user, google }) => + google.events.map(() => + expect.objectContaining({ + _id: expect.any(ObjectId), + user, + resourceId: expect.any(String), + expiration: expect.any(Date), + createdAt: expect.any(Date), + }), + ), + ), + ), + ); + + // Verify original sync data is unchanged + const syncDocsAfter = await mongoService.sync.find().toArray(); + + expect(syncDocsAfter).toHaveLength(syncCount); + }); + + it("is non-destructive - does not modify watch collection on down", async () => { + // Setup some watch data + await migration.up(migrationContext); + + const syncDocs = await mongoService.sync.find().toArray(); + + // Run down migration + await migration.down(migrationContext); + + // Verify watch data is unchanged + const watchDocs = await mongoService.watch.find().toArray(); + + expect(watchDocs).toEqual( + expect.arrayContaining( + syncDocs.flatMap(({ user, google }) => + google.events.map(() => + expect.objectContaining({ + _id: expect.any(ObjectId), + user, + resourceId: expect.any(String), + expiration: expect.any(Date), + createdAt: expect.any(Date), + }), + ), + ), + ), + ); + }); +}); diff --git a/packages/scripts/src/common/zod-to-mongo-schema.test.ts b/packages/scripts/src/common/zod-to-mongo-schema.test.ts index a982d20a4..76d68bbff 100644 --- a/packages/scripts/src/common/zod-to-mongo-schema.test.ts +++ b/packages/scripts/src/common/zod-to-mongo-schema.test.ts @@ -1,11 +1,8 @@ // derived from https://github.com/mission-apprentissage/zod-to-mongodb-schema/blob/main/src/index.test.ts import { z } from "zod/v4"; import type { JSONSchema } from "zod/v4/core"; -import { - zObjectId, - zObjectIdMini, - zodToMongoSchema, -} from "@scripts/common/zod-to-mongo-schema"; +import { zodToMongoSchema } from "@scripts/common/zod-to-mongo-schema"; +import { zObjectId, zObjectIdMini } from "@core/types/type.utils"; describe("zodToMongoSchema", () => { it("should convert zod object properly", () => { diff --git a/packages/scripts/src/common/zod-to-mongo-schema.ts b/packages/scripts/src/common/zod-to-mongo-schema.ts index 1924c9a34..837f49b44 100644 --- a/packages/scripts/src/common/zod-to-mongo-schema.ts +++ b/packages/scripts/src/common/zod-to-mongo-schema.ts @@ -1,9 +1,7 @@ // derived from https://github.com/mission-apprentissage/zod-to-mongodb-schema/tree/main/index.ts -import { ObjectId } from "bson"; -import { z } from "zod/v4"; -import { z as zMini } from "zod/v4-mini"; import type { $ZodType, JSONSchema } from "zod/v4/core"; import { registry, toJSONSchema } from "zod/v4/core"; +import { zObjectId, zObjectIdMini } from "@core/types/type.utils"; type MongoType = "object" | "array" | "number" | "boolean" | "string" | "null"; @@ -124,16 +122,6 @@ function convertTypeToBsonType( } } -export const zObjectIdMini = zMini.pipe( - zMini.custom(ObjectId.isValid), - zMini.transform((v) => new ObjectId(v)), -); - -export const zObjectId = z.pipe( - z.custom((v) => ObjectId.isValid(v as string)), - z.transform((v) => new ObjectId(v)), -); - function resolveRef(root: JSONSchema.Schema, ref: string) { const parts: string[] = ref.split("/").slice(1); const schema = parts.reduce((acc, part) => { diff --git a/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts b/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts deleted file mode 100644 index 5d64ad8fd..000000000 --- a/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-events-watch-data.ts +++ /dev/null @@ -1,126 +0,0 @@ -import type { RunnableMigration } from "umzug"; -import { MigrationContext } from "@scripts/common/cli.types"; -import { Watch } from "@core/types/watch.types"; -import mongoService from "@backend/common/services/mongo.service"; -import dayjs from "../../../core/src/util/date/dayjs"; - -export default class Migration implements RunnableMigration { - readonly name: string = "2025.10.13T14.22.21.migrate-events-watch-data"; - readonly path: string = "2025.10.13T14.22.21.migrate-events-watch-data.ts"; - - async up(): Promise { - const session = await mongoService.startSession(); - // This is a non-destructive migration to copy events watch data from sync collection to watch collection - - const cursor = mongoService.sync.find( - { "google.events": { $exists: true, $ne: [] } }, - { batchSize: 100, session }, - ); - - let migratedCount = 0; - - for await (const syncDoc of cursor) { - if (!syncDoc.google?.events?.length) continue; - - const watchDocuments: Watch[] = []; - - for (const eventSync of syncDoc.google.events) { - if ( - !eventSync.channelId || - !eventSync.resourceId || - !eventSync.expiration - ) { - continue; // Skip incomplete watch data - } - - // Convert expiration string to Date - let expirationDate: Date; - - try { - // Google Calendar expiration is typically a timestamp in milliseconds - const expirationMs = parseInt(eventSync.expiration); - - if (isNaN(expirationMs)) { - console.warn( - `Invalid expiration ms for channelId ${eventSync.channelId}: ${eventSync.expiration}`, - ); - continue; - } - - expirationDate = dayjs(expirationMs).toDate(); - } catch { - // If parsing fails, skip this watch entry - console.warn( - `Invalid expiration format for channelId ${eventSync.channelId}: ${eventSync.expiration}`, - ); - continue; - } - - const watchDoc: Watch = { - _id: eventSync.channelId, - user: syncDoc.user, - resourceId: eventSync.resourceId, - expiration: expirationDate, - createdAt: new Date(), // Set current time as creation time for migration - }; - - watchDocuments.push(watchDoc); - } - - if (watchDocuments.length > 0) { - try { - // Use insertMany with ordered: false to continue on duplicates - const result = await mongoService.watch.insertMany(watchDocuments, { - ordered: false, - session, - }); - - migratedCount += result.insertedCount; - } catch (error: unknown) { - // Log errors but continue migration (some channels might already exist) - if (error?.writeErrors) { - const duplicateErrors = error.writeErrors.filter( - (err: any) => err.code === 11000, - ); - const otherErrors = error.writeErrors.filter( - (err: any) => err.code !== 11000, - ); - - if (duplicateErrors.length > 0) { - console.log( - `Skipped ${duplicateErrors.length} duplicate watch channels for user ${syncDoc.user}`, - ); - } - - if (otherErrors.length > 0) { - console.error( - `Errors inserting watch data for user ${syncDoc.user}:`, - otherErrors, - ); - } - - // Count successful inserts - const successCount = - watchDocuments.length - error.writeErrors.length; - migratedCount += successCount; - } else { - console.error( - `Unexpected error migrating watch data for user ${syncDoc.user}:`, - error, - ); - } - } - } - } - - console.log( - `Migrated ${migratedCount} events watch channels to watch collection`, - ); - } - - async down(): Promise { - // This is a non-destructive migration, we don't remove the data from watch collection - // because it might have been updated or new watches might have been added - console.log("Non-destructive migration: watch collection data left intact"); - } -} diff --git a/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-sync-watch-data.ts b/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-sync-watch-data.ts new file mode 100644 index 000000000..17320a4c8 --- /dev/null +++ b/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-sync-watch-data.ts @@ -0,0 +1,84 @@ +import { ObjectId, WithId } from "mongodb"; +import type { MigrationParams, RunnableMigration } from "umzug"; +import { MigrationContext } from "@scripts/common/cli.types"; +import { Schema_Watch, WatchSchema } from "@core/types/watch.types"; +import { getGcalClient } from "@backend/auth/services/google.auth.service"; +import gcalService from "@backend/common/services/gcal/gcal.service"; +import mongoService from "@backend/common/services/mongo.service"; +import syncService from "@backend/sync/services/sync.service"; +import { getChannelExpiration } from "@backend/sync/util/sync.util"; + +export default class Migration implements RunnableMigration { + readonly name: string = "2025.10.13T14.22.21.migrate-sync-watch-data"; + readonly path: string = "2025.10.13T14.22.21.migrate-sync-watch-data.ts"; + + async up(params: MigrationParams): Promise { + const { logger } = params.context; + const session = await mongoService.startSession(); + // This is a non-destructive migration to copy events watch data + // from sync collection to watch collection + + const cursor = mongoService.sync.find( + { "google.events": { $exists: true, $ne: [] } }, + { batchSize: 100, session }, + ); + + let migratedCount = 0; + + for await (const syncDoc of cursor) { + if (!syncDoc.google?.events?.length) continue; + + const watchDocuments: Array>> = []; + // we will not migrate calendarlist watches as we do not store resourceId + // for them currently and they are unused + const syncDocs = syncDoc.google.events; + const gcal = await getGcalClient(syncDoc.user); + const expiration = getChannelExpiration(); + + await Promise.allSettled([ + ...syncDocs.map(async (s) => { + await syncService + .stopWatch(syncDoc.user, s.channelId, s.resourceId, gcal) + .catch(logger.error); + + const _id = new ObjectId(); + const channelId = _id.toString(); + + const { watch } = await gcalService.watchEvents(gcal, { + channelId, + expiration, + gCalendarId: s.gCalendarId, + nextSyncToken: s.nextSyncToken, + }); + + watchDocuments.push( + WatchSchema.parse({ + _id, + user: syncDoc.user, + resourceId: watch.resourceId!, + expiration: new Date(parseInt(watch.expiration!)), + createdAt: new Date(), // Set current time as creation time for migration + }), + ); + }), + ]); + + const result = await mongoService.watch.insertMany(watchDocuments, { + session, + }); + + migratedCount += result.insertedCount; + } + + logger.info( + `Migrated ${migratedCount} events watch channels to watch collection`, + ); + } + + async down(params: MigrationParams): Promise { + const { logger } = params.context; + // This is a non-destructive migration, we don't remove the data from watch collection + // because it might have been updated or new watches might have been added + logger.info("Non-destructive migration: watch collection data left intact"); + } +} From 41936c8db21a6704d8e9fa53413339db679f562d Mon Sep 17 00:00:00 2001 From: Victor Enogwe <23452630+victor-enogwe@users.noreply.github.com> Date: Mon, 13 Oct 2025 23:06:44 +0100 Subject: [PATCH 6/6] Update packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-sync-watch-data.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../2025.10.13T14.22.21.migrate-sync-watch-data.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-sync-watch-data.ts b/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-sync-watch-data.ts index 17320a4c8..a97d2efd6 100644 --- a/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-sync-watch-data.ts +++ b/packages/scripts/src/migrations/2025.10.13T14.22.21.migrate-sync-watch-data.ts @@ -63,11 +63,12 @@ export default class Migration implements RunnableMigration { }), ]); - const result = await mongoService.watch.insertMany(watchDocuments, { - session, - }); - - migratedCount += result.insertedCount; + if (watchDocuments.length > 0) { + const result = await mongoService.watch.insertMany(watchDocuments, { + session, + }); + migratedCount += result.insertedCount; + } } logger.info(