From c3f927a90bc94bdd03d5c8ad3e65a55bdff5e47f Mon Sep 17 00:00:00 2001 From: marvy Date: Tue, 23 Jun 2026 22:54:18 +0100 Subject: [PATCH] fix(migrations): make migrate:up run clean on a fresh database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit node-pg-migrate loads every migration file before running any, so CI only ever surfaced the first failure (a CommonJS file under an ESM package) and masked a chain of further runtime errors behind it. migrate:up has never completed on a clean DB. Fixes, in execution order: - 1776/1778/1784: convert CommonJS (exports./module.exports) to ESM so the files load under "type": "module". 1784 also called db.query(), which node-pg-migrate never provides to migrations — routed through pgm.sql(). - 1772: event_types jsonb default "[]::jsonb" rendered as invalid JSON; use pgm.func("'[]'::jsonb"). - 1778: trigger referenced update_updated_at_column(), created by no migration — define it before the trigger. - 1781: payload / next_retry_at are already created by 1772 — guard the re-adds with ifNotExists. - 1784: drop FK to loan_events(loan_id); loan_events is an append-only event table (and later a view) with no unique loan_id to reference. - 1787: email_enabled/sms_enabled/phone already added by 1773 — guard with ifNotExists. - 1788: replace pgm.renameIndex (not a method in node-pg-migrate v8) with raw ALTER INDEX ... RENAME TO. - 1789: loan_events existence check used pg_tables (excludes views) and tried to CREATE TABLE over the 1788 backward-compat view; use to_regclass. Verified locally against a fresh Postgres: migrate:up applies all 27 migrations (exit 0) and is a no-op on re-run. Unblocks the migrate step for all branches. Co-Authored-By: Claude Opus 4.8 --- .../1772000000000_webhook-subscriptions.js | 6 +- ...000006_add-interest-rate-to-loan-events.js | 4 +- .../1778000000009_transaction-submissions.js | 16 ++++- .../1781000000011_webhook-retry-logic.js | 34 +++++---- migrations/1784000000014_add-loan-disputes.js | 71 ++++++++++--------- ...000000017_user-notification-preferences.js | 15 ++-- .../1788000000019_unified-contract-events.js | 69 +++++------------- .../1789000000000_ensure-core-tables.js | 6 +- 8 files changed, 110 insertions(+), 111 deletions(-) diff --git a/migrations/1772000000000_webhook-subscriptions.js b/migrations/1772000000000_webhook-subscriptions.js index 8e1879d..5e941c7 100644 --- a/migrations/1772000000000_webhook-subscriptions.js +++ b/migrations/1772000000000_webhook-subscriptions.js @@ -11,7 +11,11 @@ export const up = (pgm) => { pgm.createTable("webhook_subscriptions", { id: "id", callback_url: { type: "text", notNull: true }, - event_types: { type: "jsonb", notNull: true, default: "[]::jsonb" }, + event_types: { + type: "jsonb", + notNull: true, + default: pgm.func("'[]'::jsonb"), + }, secret: { type: "varchar(255)" }, is_active: { type: "boolean", notNull: true, default: true }, created_at: { diff --git a/migrations/1776000000006_add-interest-rate-to-loan-events.js b/migrations/1776000000006_add-interest-rate-to-loan-events.js index 1155466..aafbbf9 100644 --- a/migrations/1776000000006_add-interest-rate-to-loan-events.js +++ b/migrations/1776000000006_add-interest-rate-to-loan-events.js @@ -1,4 +1,4 @@ -exports.up = (pgm) => { +export const up = (pgm) => { pgm.addColumns("loan_events", { interest_rate_bps: { type: "integer", default: null }, term_ledgers: { type: "integer", default: null }, @@ -8,6 +8,6 @@ exports.up = (pgm) => { // but for now we'll just track the rate per-loan event. }; -exports.down = (pgm) => { +export const down = (pgm) => { pgm.dropColumns("loan_events", ["interest_rate_bps", "term_ledgers"]); }; diff --git a/migrations/1778000000009_transaction-submissions.js b/migrations/1778000000009_transaction-submissions.js index 34a4dac..2cb80be 100644 --- a/migrations/1778000000009_transaction-submissions.js +++ b/migrations/1778000000009_transaction-submissions.js @@ -1,7 +1,7 @@ /** * @param { import("node-pg-migrate").MigrationBuilder } @param pgm {import("node-pg-migrate").MigrationBuilder} */ -exports.up = (pgm) => { +export const up = (pgm) => { pgm.createTable("transaction_submissions", { id: { type: "serial", @@ -52,6 +52,18 @@ exports.up = (pgm) => { pgm.createIndex("transaction_submissions", ["status"]); pgm.createIndex("transaction_submissions", ["transaction_type"]); + // Ensure the shared updated_at trigger function exists (not created by any + // earlier migration), otherwise the trigger below cannot be created. + pgm.sql(` + CREATE OR REPLACE FUNCTION update_updated_at_column() + RETURNS trigger AS $$ + BEGIN + NEW.updated_at = NOW(); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + `); + // Trigger to update updated_at timestamp pgm.createTrigger("transaction_submissions", "update_updated_at", { when: "BEFORE", @@ -63,6 +75,6 @@ exports.up = (pgm) => { /** * @param { import("node-pg-migrate").MigrationBuilder } @param pgm {import("node-pg-migrate").MigrationBuilder} */ -exports.down = (pgm) => { +export const down = (pgm) => { pgm.dropTable("transaction_submissions"); }; diff --git a/migrations/1781000000011_webhook-retry-logic.js b/migrations/1781000000011_webhook-retry-logic.js index 775b358..291b68d 100644 --- a/migrations/1781000000011_webhook-retry-logic.js +++ b/migrations/1781000000011_webhook-retry-logic.js @@ -8,21 +8,31 @@ export const shorthands = undefined; * @returns {Promise | void} */ export const up = (pgm) => { - // Add payload column to webhook_deliveries table - pgm.addColumn("webhook_deliveries", { - payload: { - type: "jsonb", - notNull: false, + // Add payload column to webhook_deliveries table. + // 1772 already creates this column, so guard against re-adding it. + pgm.addColumn( + "webhook_deliveries", + { + payload: { + type: "jsonb", + notNull: false, + }, }, - }); + { ifNotExists: true }, + ); - // Add next_retry_at column to track when to retry - pgm.addColumn("webhook_deliveries", { - next_retry_at: { - type: "timestamp", - notNull: false, + // Add next_retry_at column to track when to retry. + // 1772 already creates this column, so guard against re-adding it. + pgm.addColumn( + "webhook_deliveries", + { + next_retry_at: { + type: "timestamp", + notNull: false, + }, }, - }); + { ifNotExists: true }, + ); // Add index for efficient retry polling pgm.createIndex("webhook_deliveries", ["next_retry_at"], { diff --git a/migrations/1784000000014_add-loan-disputes.js b/migrations/1784000000014_add-loan-disputes.js index d802da4..2c4ae4e 100644 --- a/migrations/1784000000014_add-loan-disputes.js +++ b/migrations/1784000000014_add-loan-disputes.js @@ -1,41 +1,42 @@ // Migration: Add loan_disputes table and support for disputed loan status -module.exports = { - async up(db) { - // 1. Create loan_disputes table - await db.query(` - CREATE TABLE IF NOT EXISTS loan_disputes ( - id SERIAL PRIMARY KEY, - loan_id INTEGER NOT NULL REFERENCES loan_events(loan_id), - borrower TEXT NOT NULL, - reason TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'open', -- open, resolved, rejected - admin_note TEXT, - resolution TEXT, - created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - resolved_at TIMESTAMP WITH TIME ZONE - ); - `); +export const up = (pgm) => { + // 1. Create loan_disputes table + pgm.sql(` + CREATE TABLE IF NOT EXISTS loan_disputes ( + id SERIAL PRIMARY KEY, + -- loan_id is not a FK: loan_events is an append-only event table whose + -- loan_id is non-unique (and later becomes a view), so it cannot be a + -- foreign key target. + loan_id INTEGER NOT NULL, + borrower TEXT NOT NULL, + reason TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'open', -- open, resolved, rejected + admin_note TEXT, + resolution TEXT, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + resolved_at TIMESTAMP WITH TIME ZONE + ); + `); - // 2. Add indexes for efficient querying - await db.query(` - CREATE INDEX IF NOT EXISTS idx_loan_disputes_status ON loan_disputes(status); - `); - await db.query(` - CREATE INDEX IF NOT EXISTS idx_loan_disputes_borrower ON loan_disputes(borrower); - `); - await db.query(` - CREATE INDEX IF NOT EXISTS idx_loan_disputes_loan_id ON loan_disputes(loan_id); - `); + // 2. Add indexes for efficient querying + pgm.sql(` + CREATE INDEX IF NOT EXISTS idx_loan_disputes_status ON loan_disputes(status); + `); + pgm.sql(` + CREATE INDEX IF NOT EXISTS idx_loan_disputes_borrower ON loan_disputes(borrower); + `); + pgm.sql(` + CREATE INDEX IF NOT EXISTS idx_loan_disputes_loan_id ON loan_disputes(loan_id); + `); - // 3. Add disputed status to loan_events (if using status enum, update it) - // If status is a string, no migration needed. If enum, alter type here. - // Example for enum: - // await db.query(`ALTER TYPE loan_status_enum ADD VALUE IF NOT EXISTS 'disputed';`); - }, + // 3. Add disputed status to loan_events (if using status enum, update it) + // If status is a string, no migration needed. If enum, alter type here. + // Example for enum: + // pgm.sql(`ALTER TYPE loan_status_enum ADD VALUE IF NOT EXISTS 'disputed';`); +}; - async down(db) { - await db.query(`DROP TABLE IF EXISTS loan_disputes;`); - // No need to remove enum value (Postgres doesn't support removing enum values easily) - }, +export const down = (pgm) => { + pgm.sql(`DROP TABLE IF EXISTS loan_disputes;`); + // No need to remove enum value (Postgres doesn't support removing enum values easily) }; diff --git a/migrations/1787000000017_user-notification-preferences.js b/migrations/1787000000017_user-notification-preferences.js index cf58cf0..a4cd71e 100644 --- a/migrations/1787000000017_user-notification-preferences.js +++ b/migrations/1787000000017_user-notification-preferences.js @@ -8,11 +8,16 @@ export const shorthands = undefined; * @returns {Promise | void} */ export const up = (pgm) => { - pgm.addColumns("user_profiles", { - email_enabled: { type: "boolean", notNull: true, default: false }, - sms_enabled: { type: "boolean", notNull: true, default: false }, - phone: { type: "varchar(20)" }, - }); + // 1773 already adds these notification columns, so guard against re-adding. + pgm.addColumns( + "user_profiles", + { + email_enabled: { type: "boolean", notNull: true, default: false }, + sms_enabled: { type: "boolean", notNull: true, default: false }, + phone: { type: "varchar(20)" }, + }, + { ifNotExists: true }, + ); }; /** diff --git a/migrations/1788000000019_unified-contract-events.js b/migrations/1788000000019_unified-contract-events.js index 9044ff9..02e7e35 100644 --- a/migrations/1788000000019_unified-contract-events.js +++ b/migrations/1788000000019_unified-contract-events.js @@ -12,32 +12,15 @@ export const up = (pgm) => { // 3. Make address nullable (for events like YieldDistributed that may not have a user address) pgm.alterColumn("contract_events", "address", { notNull: false }); - // 4. Rename indexes to match the new table and column names - pgm.renameIndex( - "contract_events", - "idx_loan_events_borrower_event_type", - "idx_contract_events_address_event_type", - ); - pgm.renameIndex( - "contract_events", - "idx_loan_events_loan_id_event_type", - "idx_contract_events_loan_id_event_type", - ); - pgm.renameIndex( - "contract_events", - "idx_loan_events_event_type_loan_id", - "idx_contract_events_event_type_loan_id", - ); - pgm.renameIndex( - "contract_events", - "idx_loan_events_ledger", - "idx_contract_events_ledger", - ); - pgm.renameIndex( - "contract_events", - "idx_loan_events_pool_deposits_withdraws", - "idx_contract_events_pool_deposits_withdraws", - ); + // 4. Rename indexes to match the new table and column names. + // node-pg-migrate has no renameIndex helper, so use raw ALTER INDEX. + pgm.sql(` + ALTER INDEX IF EXISTS idx_loan_events_borrower_event_type RENAME TO idx_contract_events_address_event_type; + ALTER INDEX IF EXISTS idx_loan_events_loan_id_event_type RENAME TO idx_contract_events_loan_id_event_type; + ALTER INDEX IF EXISTS idx_loan_events_event_type_loan_id RENAME TO idx_contract_events_event_type_loan_id; + ALTER INDEX IF EXISTS idx_loan_events_ledger RENAME TO idx_contract_events_ledger; + ALTER INDEX IF EXISTS idx_loan_events_pool_deposits_withdraws RENAME TO idx_contract_events_pool_deposits_withdraws; + `); // Rename single-column indexes from initial schema (if they exist) pgm.sql(` @@ -81,32 +64,14 @@ export const down = (pgm) => { pgm.renameTable("contract_events", "loan_events"); - // Revert index names - pgm.renameIndex( - "loan_events", - "idx_contract_events_address_event_type", - "idx_loan_events_borrower_event_type", - ); - pgm.renameIndex( - "loan_events", - "idx_contract_events_loan_id_event_type", - "idx_loan_events_loan_id_event_type", - ); - pgm.renameIndex( - "loan_events", - "idx_contract_events_event_type_loan_id", - "idx_loan_events_event_type_loan_id", - ); - pgm.renameIndex( - "loan_events", - "idx_contract_events_ledger", - "idx_loan_events_ledger", - ); - pgm.renameIndex( - "loan_events", - "idx_contract_events_pool_deposits_withdraws", - "idx_loan_events_pool_deposits_withdraws", - ); + // Revert index names (raw ALTER INDEX; no renameIndex helper exists) + pgm.sql(` + ALTER INDEX IF EXISTS idx_contract_events_address_event_type RENAME TO idx_loan_events_borrower_event_type; + ALTER INDEX IF EXISTS idx_contract_events_loan_id_event_type RENAME TO idx_loan_events_loan_id_event_type; + ALTER INDEX IF EXISTS idx_contract_events_event_type_loan_id RENAME TO idx_loan_events_event_type_loan_id; + ALTER INDEX IF EXISTS idx_contract_events_ledger RENAME TO idx_loan_events_ledger; + ALTER INDEX IF EXISTS idx_contract_events_pool_deposits_withdraws RENAME TO idx_loan_events_pool_deposits_withdraws; + `); pgm.sql(` ALTER INDEX IF EXISTS contract_events_event_type_index RENAME TO loan_events_event_type_index; diff --git a/migrations/1789000000000_ensure-core-tables.js b/migrations/1789000000000_ensure-core-tables.js index 38d7933..d489021 100644 --- a/migrations/1789000000000_ensure-core-tables.js +++ b/migrations/1789000000000_ensure-core-tables.js @@ -25,11 +25,13 @@ export const up = (pgm) => { END $$; `); - // Ensure loan_events table matches requested schema + // Ensure loan_events relation exists. Use to_regclass (not pg_tables) so the + // backward-compat loan_events VIEW created in 1788 also counts as existing; + // otherwise this would try to CREATE TABLE over the view and fail. pgm.sql(` DO $$ BEGIN - IF NOT EXISTS (SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename = 'loan_events') THEN + IF to_regclass('public.loan_events') IS NULL THEN CREATE TABLE loan_events ( id SERIAL PRIMARY KEY, loan_id INTEGER,