From dceb007bd95f671081ca19a78da516a0925b8861 Mon Sep 17 00:00:00 2001 From: "Andrei V. Lepikhov" Date: Wed, 1 Apr 2026 16:25:41 +0200 Subject: [PATCH] Add bidirectional large object migration between native and lolor storage Add migrate_from_native() and migrate_to_native() SQL functions that move large objects between PostgreSQL's native pg_catalog storage and lolor's replication-compatible tables, preserving OIDs, owners, ACLs, and data. The DROP EXTENSION event trigger now calls migrate_to_native() automatically before removing the extension, preventing data loss that previously occurred when lolor tables were dropped with the schema. Key details: - Forward migration uses bulk INSERT for performance - Reverse migration uses the native LO API (lo_create_orig/lowrite_orig) since direct INSERT into pg_catalog.pg_largeobject is not allowed - LOBLKSIZE derived at runtime from block_size GUC for non-default builds - Uses lo_lseek64_orig for large objects exceeding 2 GB - Both functions require superuser and verify lolor is enabled - Event trigger guards migrate_to_native() with pg_proc existence check for backward compatibility with versions < 1.2.3 - Emits NOTICE during installation if streaming replicas are detected - Migration is safe only in master-replica configurations for now Add regression tests for forward migration, reverse migration via DROP EXTENSION, manual migrate_to_native, empty-database edge cases, and the 1.2.2 to 1.2.3 upgrade path. --- Makefile | 3 +- README.md | 24 ++- docs/index.md | 44 ++++- expected/lolor.out | 375 ++++++++++++++++++++++++++++++++++++++- lolor--1.2.2--1.2.3.sql | 197 ++++++++++++++++++++ lolor.control | 2 +- sql/lolor.sql | 149 +++++++++++++++- src/lolor.c | 41 ++++- t/006_promote_standby.pl | 90 ++++++++++ 9 files changed, 909 insertions(+), 16 deletions(-) create mode 100644 lolor--1.2.2--1.2.3.sql create mode 100644 t/006_promote_standby.pl diff --git a/Makefile b/Makefile index caf9241..ef279d8 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,8 @@ MODULE_big = lolor EXTENSION = lolor DATA = lolor--1.0.sql \ - lolor--1.0--1.2.1.sql lolor--1.2.1--1.2.2.sql + lolor--1.0--1.2.1.sql lolor--1.2.1--1.2.2.sql \ + lolor--1.2.2--1.2.3.sql PGFILEDESC = "lolor - drop in large objects replacement for logical replication" OBJS = src/lolor.o src/lolor_fsstubs.o src/lolor_inv_api.o src/lolor_largeobject.o diff --git a/README.md b/README.md index ba26ea7..3f91275 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,30 @@ the following commands to add the tables: ./pgedge spock repset-add-table spock_replication_set 'lolor.pg_largeobject_metadata' lolor_db ``` +### Migrating large objects + +Migration from native to lolor is **manual**; migration back is **automatic** +on `DROP EXTENSION` so no objects are ever lost. + +Migrate existing native large objects into lolor storage (requires superuser): + +```sql +CREATE EXTENSION lolor; +SELECT lolor.migrate_from_native(); +``` + +Reverse migration happens automatically when the extension is dropped, or can +be triggered manually: + +```sql +SELECT lolor.migrate_to_native(); -- manual +DROP EXTENSION lolor; -- automatic +``` + +Both directions preserve original OIDs, owners, ACLs, and data. + ### Limitations - Native large object functionality cannot be used while you are using the lolor extension. -- Native large object migration to the lolor feature is not available yet. - lolor does not support the following statements: `ALTER LARGE OBJECT`, `GRANT ON LARGE OBJECT`, `COMMENT ON LARGE OBJECT`, and `REVOKE ON LARGE OBJECT`. +- Migration procedures are currently safe only in master-replica configurations. Multi-master migration is not yet supported due to OID encoding constraints. diff --git a/docs/index.md b/docs/index.md index 2fd0768..91603fd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,8 +16,50 @@ Postgres large objects allow you to store large files within the database. Each Use of the `lolor` extension requires Postgres 16 or newer. +## Migrating large objects + +Migration from native to lolor storage is **manual** — you decide when to move +existing large objects. Migration back to native is **automatic** — dropping +the extension moves all objects back so nothing is lost. + +### Native to lolor (manual) + +If your database already contains native Postgres large objects, call +`migrate_from_native()` after enabling the extension. The migration preserves +original OIDs, owners, ACLs, and data, so existing application references remain +valid. + +```sql +CREATE EXTENSION lolor; +SELECT lolor.migrate_from_native(); +``` + +The function returns the number of large objects migrated. It is safe to call +when there are no native large objects. This step is intentionally not +automatic: it requires superuser privileges and should be performed during a +maintenance window. + +### Lolor to native (automatic on DROP EXTENSION) + +Large objects are automatically migrated back to native Postgres storage when the +extension is dropped: + +```sql +DROP EXTENSION lolor; +``` + +This ensures that no large objects are lost if the extension is removed. You +can also trigger the reverse migration manually while the extension is still +installed: + +```sql +SELECT lolor.migrate_to_native(); +``` + +Both paths preserve OIDs, owners, ACLs, and data. + ## Limitations - Native Postgres large object functionality cannot be used while you are using the lolor extension. -- Native large object migration to the lolor feature is not available yet. - lolor does not support the following statements: `ALTER LARGE OBJECT`, `GRANT ON LARGE OBJECT`, `COMMENT ON LARGE OBJECT`, and `REVOKE ON LARGE OBJECT`. +- The migration procedures (`migrate_from_native`, `migrate_to_native`) are currently safe only in master-replica configurations. In multi-master setups, migrated OIDs (which lack node-encoding) may conflict with OIDs on other nodes. diff --git a/expected/lolor.out b/expected/lolor.out index 1409abe..0bb6976 100644 --- a/expected/lolor.out +++ b/expected/lolor.out @@ -68,6 +68,7 @@ SELECT lo_close(:fd); END; DROP EXTENSION lolor; +NOTICE: migrated 1 large object(s) from lolor to native storage -- Check extension upgrade CREATE EXTENSION lolor VERSION '1.0'; SELECT lo_creat(-1) AS loid \gset @@ -91,6 +92,37 @@ SELECT convert_from(loread(:fd, 1024), 'UTF8'); (1 row) END; +ALTER EXTENSION lolor UPDATE TO '1.2.3'; +-- Verify migration functions are available after upgrade +SELECT lolor.migrate_to_native(); -- One LO object has been created before LOLOR +NOTICE: migrated 1 large object(s) from lolor to native storage + migrate_to_native +------------------- + 1 +(1 row) + +SELECT lolor.migrate_from_native(); -- two objects +NOTICE: migrated 2 large object(s) (2 data page(s)) from native to lolor storage + migrate_from_native +--------------------- + 2 +(1 row) + +-- Repeat conversion cycle - should see the same two objects +SELECT lolor.migrate_to_native(); +NOTICE: migrated 2 large object(s) from lolor to native storage + migrate_to_native +------------------- + 2 +(1 row) + +SELECT lolor.migrate_from_native(); +NOTICE: migrated 2 large object(s) (2 data page(s)) from native to lolor storage + migrate_from_native +--------------------- + 2 +(1 row) + -- -- Basic checks for enable/disable routines. -- @@ -155,12 +187,15 @@ SELECT lolor.enable(); -- Check that no tails existing after the extension drop in both enabled and -- disabled states. DROP EXTENSION lolor; +NOTICE: migrated 3 large object(s) from lolor to native storage SELECT oid, proname FROM pg_proc WHERE proname IN ('lo_open_orig', 'lolor_lo_open'); oid | proname -----+--------- (0 rows) +-- Check: we can't just delete LOLOR without LO migration in disabled mode. +-- XXX: should we introduce a 'forced' flag to allow this? CREATE EXTENSION lolor; SELECT lolor.disable(); disable @@ -169,9 +204,339 @@ SELECT lolor.disable(); (1 row) DROP EXTENSION lolor; -SELECT oid, proname FROM pg_proc WHERE proname IN ('lo_open_orig', - 'lolor_lo_open'); - oid | proname ------+--------- -(0 rows) +ERROR: lolor must be enabled before migration to native +SELECT extname FROM pg_extension; -- lolor is here + extname +--------- + plpgsql + lolor +(2 rows) + +SELECT lolor.enable(); + enable +-------- + t +(1 row) + +DROP EXTENSION lolor; +NOTICE: no lolor large objects to migrate +SELECT extname FROM pg_extension; -- check lolor removal + extname +--------- + plpgsql +(1 row) + +-- +-- Migration tests: migrate_from_native / migrate_to_native / DROP EXTENSION +-- +-- Start fresh: no extension, create native LOs +SELECT lo_from_bytea(0, 'Native object number one') AS native_oid1 \gset +SELECT lo_from_bytea(0, 'Native object number two') AS native_oid2 \gset +-- Forward migration: expect native_lo_count = 2 +SELECT count(*) AS native_lo_count FROM pg_catalog.pg_largeobject_metadata; + native_lo_count +----------------- + 6 +(1 row) + +-- Install lolor and migrate native LOs into lolor storage +CREATE EXTENSION lolor; +SELECT lolor.migrate_from_native(); +NOTICE: migrated 6 large object(s) (6 data page(s)) from native to lolor storage + migrate_from_native +--------------------- + 6 +(1 row) + +-- After forward migration: expect 0 native objects +SELECT count(*) AS native_after_migrate FROM pg_catalog.pg_largeobject_metadata; + native_after_migrate +---------------------- + 0 +(1 row) + +SELECT count(*) AS lolor_after_migrate FROM lolor.pg_largeobject_metadata; + lolor_after_migrate +--------------------- + 6 +(1 row) + +-- Data integrity: expect "Native object number one" +BEGIN; +SELECT lo_open(:'native_oid1'::oid, 262144) AS fd \gset +SELECT convert_from(loread(:fd, 1024), 'UTF8') AS obj1_data; + obj1_data +-------------------------- + Native object number one +(1 row) + +SELECT lo_close(:fd); + lo_close +---------- + 0 +(1 row) + +END; +-- Data integrity: expect "Native object number two" +BEGIN; +SELECT lo_open(:'native_oid2'::oid, 262144) AS fd \gset +SELECT convert_from(loread(:fd, 1024), 'UTF8') AS obj2_data; + obj2_data +-------------------------- + Native object number two +(1 row) + +SELECT lo_close(:fd); + lo_close +---------- + 0 +(1 row) + +END; +-- Create an additional LO directly in lolor storage +SELECT lo_from_bytea(0, 'Created directly in lolor') AS lolor_direct_oid \gset +-- Reverse migration via DROP EXTENSION +DROP EXTENSION lolor; +NOTICE: migrated 7 large object(s) from lolor to native storage +SELECT count(*) AS native_after_drop FROM pg_catalog.pg_largeobject_metadata; + native_after_drop +------------------- + 7 +(1 row) + +-- After DROP: expect "Native object number one" +SELECT convert_from(lo_get(:'native_oid1'::oid), 'UTF8') AS obj1_after_reverse; + obj1_after_reverse +-------------------------- + Native object number one +(1 row) + +-- After DROP: expect "Native object number two" +SELECT convert_from(lo_get(:'native_oid2'::oid), 'UTF8') AS obj2_after_reverse; + obj2_after_reverse +-------------------------- + Native object number two +(1 row) + +-- After DROP: expect "Created directly in lolor" +SELECT convert_from(lo_get(:'lolor_direct_oid'::oid), 'UTF8') AS obj3_after_reverse; + obj3_after_reverse +--------------------------- + Created directly in lolor +(1 row) + +-- Cleanup native LOs +SELECT lo_unlink(:'native_oid1'::oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT lo_unlink(:'native_oid2'::oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT lo_unlink(:'lolor_direct_oid'::oid); + lo_unlink +----------- + 1 +(1 row) + +CREATE EXTENSION lolor; +SELECT lolor.migrate_from_native(); +NOTICE: migrated 4 large object(s) (4 data page(s)) from native to lolor storage + migrate_from_native +--------------------- + 4 +(1 row) + +DROP EXTENSION lolor; +NOTICE: migrated 4 large object(s) from lolor to native storage +-- +-- Manual migrate_to_native (not via DROP EXTENSION) +-- +CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'Manual reverse test') AS manual_oid \gset +SELECT lolor.migrate_to_native(); +NOTICE: migrated 1 large object(s) from lolor to native storage + migrate_to_native +------------------- + 1 +(1 row) + +SELECT count(*) AS native_after_manual FROM pg_catalog.pg_largeobject_metadata; + native_after_manual +--------------------- + 5 +(1 row) + +SELECT count(*) AS lolor_after_manual FROM lolor.pg_largeobject_metadata; + lolor_after_manual +-------------------- + 0 +(1 row) + +-- After manual migration: expect "Manual reverse test" +BEGIN; +-- Disable lolor to read from native storage directly +SELECT lolor.disable(); + disable +--------- + t +(1 row) + +SELECT convert_from(lo_get(:'manual_oid'::oid), 'UTF8') AS manual_data; + manual_data +--------------------- + Manual reverse test +(1 row) + +END; +-- Cleanup +SELECT lo_unlink(:'manual_oid'::oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT lolor.enable(); + enable +-------- + t +(1 row) + +DROP EXTENSION lolor; +NOTICE: no lolor large objects to migrate +-- +-- OID conflict detection +-- +-- OID conflict: migrate_from_native should ERROR on duplicate OID +SELECT lo_from_bytea(0, 'Conflict test object') AS conflict_oid \gset +CREATE EXTENSION lolor; +-- HACK: Manually insert a row with the same OID into lolor storage +INSERT INTO lolor.pg_largeobject_metadata (oid, lomowner, lomacl) + VALUES (:'conflict_oid', (SELECT oid FROM pg_roles WHERE rolname = current_user), NULL); +-- This should fail with OID conflict +SELECT lolor.migrate_from_native(); +ERROR: OID conflict: some native large objects already exist in lolor storage +-- Cleanup: remove the conflicting row and drop cleanly +DELETE FROM lolor.pg_largeobject_metadata WHERE oid = :'conflict_oid'; +DROP EXTENSION lolor; +NOTICE: no lolor large objects to migrate +SELECT lo_unlink(:'conflict_oid'::oid); + lo_unlink +----------- + 1 +(1 row) + +-- OID conflict: migrate_to_native should ERROR on duplicate OID +CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'Lolor side object') AS conflict_oid2 \gset +-- Disable lolor to create a native LO with the same OID +SELECT lolor.disable(); + disable +--------- + t +(1 row) + +SELECT lo_create(:'conflict_oid2') AS created_oid \gset +-- Verify native lo_create honored the explicit OID +SELECT :'created_oid' = :'conflict_oid2' AS oid_matches; + oid_matches +------------- + t +(1 row) + +SELECT lolor.enable(); + enable +-------- + t +(1 row) + +-- migrate_to_native should detect the collision +SELECT lolor.migrate_to_native(); +ERROR: OID conflict: some lolor large objects already exist in native storage +-- Cleanup: remove the native duplicate, then drop cleanly +SELECT lolor.disable(); + disable +--------- + t +(1 row) + +SELECT lo_unlink(:'conflict_oid2'::oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT lolor.enable(); + enable +-------- + t +(1 row) + +DROP EXTENSION lolor; +NOTICE: migrated 1 large object(s) from lolor to native storage +-- DROP EXTENSION should be rejected when migrate_to_native has OID conflict +CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'Drop conflict test') AS drop_conflict_oid \gset +-- Create a native LO with the same OID to force conflict at DROP time +SELECT lolor.disable(); + disable +--------- + t +(1 row) +SELECT lo_create(:'drop_conflict_oid'); + lo_create +----------- + 268385 +(1 row) + +SELECT lolor.enable(); + enable +-------- + t +(1 row) + +-- DROP EXTENSION should ERROR to prevent data loss +DROP EXTENSION lolor; +ERROR: OID conflict: some lolor large objects already exist in native storage +-- Extension should still be installed +SELECT extname FROM pg_extension WHERE extname = 'lolor'; + extname +--------- + lolor +(1 row) + +-- Objects should be in place +SELECT count(*) FROM lolor.pg_largeobject; + count +------- + 1 +(1 row) + +-- Resolve the conflict: remove the native duplicate, then retry +SELECT lolor.disable(); + disable +--------- + t +(1 row) + +SELECT lo_unlink(:'drop_conflict_oid'::oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT lolor.enable(); + enable +-------- + t +(1 row) + +-- Now DROP should succeed +DROP EXTENSION lolor; +NOTICE: migrated 1 large object(s) from lolor to native storage diff --git a/lolor--1.2.2--1.2.3.sql b/lolor--1.2.2--1.2.3.sql new file mode 100644 index 0000000..0123d6c --- /dev/null +++ b/lolor--1.2.2--1.2.3.sql @@ -0,0 +1,197 @@ +/* lolor--1.2.2--1.2.3.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION lolor UPDATE" to load this file. \quit + +-- Warn if there are active streaming replicas — they need lolor installed too +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_stat_replication WHERE state = 'streaming') THEN + RAISE NOTICE 'lolor: active streaming replica(s) detected. ' + 'Ensure the lolor extension is also installed on each replica, ' + 'otherwise large object operations will fail if a replica is promoted.'; + END IF; +END; +$$; + +/* + * lolor.migrate_from_native() + * + * Migrate all native PostgreSQL large objects from pg_catalog.pg_largeobject + * into lolor's storage, preserving original OIDs, owners, ACLs, and data. + * After migration, the native copies are removed. + * + * The entire operation is transactional: if anything fails, ROLLBACK undoes + * all changes and no data is lost. + * + * Returns the number of large objects migrated. + */ +CREATE FUNCTION lolor.migrate_from_native() +RETURNS bigint AS $$ +DECLARE + lo_count bigint; + inserted_count bigint; + page_count bigint; + native_page_count bigint; +BEGIN + -- Only superusers can read pg_largeobject.data and unlink others' objects + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = current_user AND rolsuper) THEN + RAISE EXCEPTION 'lolor.migrate_from_native() requires superuser privileges'; + END IF; + + -- Verify lolor is enabled (functions are replaced) + IF NOT lolor.is_enabled() THEN + RAISE EXCEPTION 'lolor must be enabled before migration'; + END IF; + + -- Check for OID conflicts: native LOs that already exist in lolor storage + IF EXISTS ( + SELECT 1 + FROM pg_catalog.pg_largeobject_metadata native + JOIN lolor.pg_largeobject_metadata lm ON lm.oid = native.oid + ) THEN + RAISE EXCEPTION 'OID conflict: some native large objects already exist in lolor storage'; + END IF; + + -- Count what we are about to migrate + SELECT count(*) INTO lo_count FROM pg_catalog.pg_largeobject_metadata; + + IF lo_count = 0 THEN + RAISE NOTICE 'no native large objects to migrate'; + RETURN 0; + END IF; + + SELECT count(*) INTO native_page_count + FROM pg_catalog.pg_largeobject; + + -- Copy metadata (preserving OIDs, owners, and ACLs) + INSERT INTO lolor.pg_largeobject_metadata (oid, lomowner, lomacl) + SELECT oid, lomowner, lomacl + FROM pg_catalog.pg_largeobject_metadata; + + GET DIAGNOSTICS inserted_count = ROW_COUNT; + IF inserted_count <> lo_count THEN + RAISE EXCEPTION 'metadata row count mismatch: expected %, inserted %', + lo_count, inserted_count; + END IF; + + -- Copy data pages + INSERT INTO lolor.pg_largeobject (loid, pageno, data) + SELECT loid, pageno, data FROM pg_catalog.pg_largeobject; + + GET DIAGNOSTICS page_count = ROW_COUNT; + IF page_count <> native_page_count THEN + RAISE EXCEPTION 'data page count mismatch: expected %, inserted %', + native_page_count, page_count; + END IF; + + -- Remove native large objects using the original (renamed) function. + -- Materialize the OID list first to avoid scanning the catalog while + -- lo_unlink_orig modifies it. + PERFORM pg_catalog.lo_unlink_orig(oid) + FROM (SELECT oid FROM pg_catalog.pg_largeobject_metadata) AS native_oids; + + RAISE NOTICE 'migrated % large object(s) (% data page(s)) from native to lolor storage', + lo_count, page_count; + + RETURN lo_count; +END; +$$ LANGUAGE plpgsql VOLATILE; + +/* + * lolor.migrate_to_native() + * + * Migrate all large objects from lolor storage back into native PostgreSQL + * storage, preserving original OIDs, owners, ACLs, and data. After + * migration, the lolor copies are removed. + * + * Called automatically by the DROP EXTENSION event trigger, but can also + * be invoked manually to revert to native large object storage. + * + * The _orig functions (native LO API) must be available, which means lolor + * must be in the enabled state. + * + * Returns the number of large objects migrated. + */ +CREATE FUNCTION lolor.migrate_to_native() +RETURNS bigint AS $$ +DECLARE + lo_count bigint; + loblksize bigint; + r_meta record; + r_data record; + fd integer; +BEGIN + -- Only superusers can UPDATE pg_catalog.pg_largeobject_metadata + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = current_user AND rolsuper) THEN + RAISE EXCEPTION 'lolor.migrate_to_native() requires superuser privileges'; + END IF; + + -- Verify lolor is enabled so _orig functions point to native API + IF NOT lolor.is_enabled() THEN + RAISE EXCEPTION 'lolor must be enabled before migration to native'; + END IF; + + -- Derive LOBLKSIZE at runtime. PostgreSQL defines it as BLCKSZ / 4. + -- Hard-coding 2048 would break on non-default block size builds. + loblksize := current_setting('block_size')::bigint / 4; + + -- Count what we are about to migrate + SELECT count(*) INTO lo_count FROM lolor.pg_largeobject_metadata; + + IF lo_count = 0 THEN + RAISE NOTICE 'no lolor large objects to migrate'; + RETURN 0; + END IF; + + -- Check for OID conflicts + IF EXISTS ( + SELECT 1 + FROM lolor.pg_largeobject_metadata lm + JOIN pg_catalog.pg_largeobject_metadata native ON native.oid = lm.oid + ) THEN + RAISE EXCEPTION 'OID conflict: some lolor large objects already exist in native storage'; + END IF; + + -- Migrate each object using the native LO API (_orig functions). + -- We cannot INSERT directly into pg_catalog.pg_largeobject from SQL, + -- so we use lo_create_orig + lo_open_orig + lowrite_orig. + -- + -- Zero-data-page LOs (metadata only) are handled correctly: lo_create_orig + -- creates an empty LO and the inner FOR loop simply does not execute. + -- + -- Note on sparse LOs: any gap between non-consecutive page numbers will + -- be filled with zeroes by the native LO write API. This preserves read + -- semantics (holes already returned zeroes) but may increase storage. + FOR r_meta IN SELECT oid, lomowner, lomacl FROM lolor.pg_largeobject_metadata + LOOP + -- Create native LO with the exact same OID + PERFORM pg_catalog.lo_create_orig(r_meta.oid); + + -- Write data pages through the native LO write API. + -- Use lo_lseek64_orig (bigint offset) to handle LOs larger than 2 GB. + fd := pg_catalog.lo_open_orig(r_meta.oid, x'60000'::int); + FOR r_data IN + SELECT pageno, data FROM lolor.pg_largeobject + WHERE loid = r_meta.oid ORDER BY pageno + LOOP + PERFORM pg_catalog.lo_lseek64_orig(fd, r_data.pageno::bigint * loblksize, 0); + PERFORM pg_catalog.lowrite_orig(fd, r_data.data); + END LOOP; + PERFORM pg_catalog.lo_close_orig(fd); + + -- Restore ownership and ACL (lo_create sets current user as owner) + UPDATE pg_catalog.pg_largeobject_metadata + SET lomowner = r_meta.lomowner, lomacl = r_meta.lomacl + WHERE pg_catalog.pg_largeobject_metadata.oid = r_meta.oid; + END LOOP; + + -- Clean lolor storage + DELETE FROM lolor.pg_largeobject; + DELETE FROM lolor.pg_largeobject_metadata; + + RAISE NOTICE 'migrated % large object(s) from lolor to native storage', lo_count; + + RETURN lo_count; +END; +$$ LANGUAGE plpgsql VOLATILE; diff --git a/lolor.control b/lolor.control index e8a0385..093079c 100644 --- a/lolor.control +++ b/lolor.control @@ -1,6 +1,6 @@ # lolor extension comment = 'Large Objects support for logical replication' -default_version = '1.2.2' +default_version = '1.2.3' module_pathname = '$libdir/lolor' relocatable = false trusted = true diff --git a/sql/lolor.sql b/sql/lolor.sql index 78546ee..c8b32e8 100644 --- a/sql/lolor.sql +++ b/sql/lolor.sql @@ -48,6 +48,14 @@ BEGIN; SELECT lo_open(:loid, 262144) AS fd \gset SELECT convert_from(loread(:fd, 1024), 'UTF8'); END; +ALTER EXTENSION lolor UPDATE TO '1.2.3'; +-- Verify migration functions are available after upgrade +SELECT lolor.migrate_to_native(); -- One LO object has been created before LOLOR +SELECT lolor.migrate_from_native(); -- two objects + +-- Repeat conversion cycle - should see the same two objects +SELECT lolor.migrate_to_native(); +SELECT lolor.migrate_from_native(); -- -- Basic checks for enable/disable routines. @@ -76,8 +84,145 @@ SELECT lolor.enable(); DROP EXTENSION lolor; SELECT oid, proname FROM pg_proc WHERE proname IN ('lo_open_orig', 'lolor_lo_open'); + +-- Check: we can't just delete LOLOR without LO migration in disabled mode. +-- XXX: should we introduce a 'forced' flag to allow this? CREATE EXTENSION lolor; SELECT lolor.disable(); DROP EXTENSION lolor; -SELECT oid, proname FROM pg_proc WHERE proname IN ('lo_open_orig', - 'lolor_lo_open'); +SELECT extname FROM pg_extension; -- lolor is here +SELECT lolor.enable(); +DROP EXTENSION lolor; +SELECT extname FROM pg_extension; -- check lolor removal + +-- +-- Migration tests: migrate_from_native / migrate_to_native / DROP EXTENSION +-- + +-- Start fresh: no extension, create native LOs +SELECT lo_from_bytea(0, 'Native object number one') AS native_oid1 \gset +SELECT lo_from_bytea(0, 'Native object number two') AS native_oid2 \gset + +-- Forward migration: expect native_lo_count = 2 +SELECT count(*) AS native_lo_count FROM pg_catalog.pg_largeobject_metadata; + +-- Install lolor and migrate native LOs into lolor storage +CREATE EXTENSION lolor; +SELECT lolor.migrate_from_native(); + +-- After forward migration: expect 0 native objects +SELECT count(*) AS native_after_migrate FROM pg_catalog.pg_largeobject_metadata; +SELECT count(*) AS lolor_after_migrate FROM lolor.pg_largeobject_metadata; + +-- Data integrity: expect "Native object number one" +BEGIN; +SELECT lo_open(:'native_oid1'::oid, 262144) AS fd \gset +SELECT convert_from(loread(:fd, 1024), 'UTF8') AS obj1_data; +SELECT lo_close(:fd); +END; + +-- Data integrity: expect "Native object number two" +BEGIN; +SELECT lo_open(:'native_oid2'::oid, 262144) AS fd \gset +SELECT convert_from(loread(:fd, 1024), 'UTF8') AS obj2_data; +SELECT lo_close(:fd); +END; + +-- Create an additional LO directly in lolor storage +SELECT lo_from_bytea(0, 'Created directly in lolor') AS lolor_direct_oid \gset + +-- Reverse migration via DROP EXTENSION +DROP EXTENSION lolor; + +SELECT count(*) AS native_after_drop FROM pg_catalog.pg_largeobject_metadata; + +-- After DROP: expect "Native object number one" +SELECT convert_from(lo_get(:'native_oid1'::oid), 'UTF8') AS obj1_after_reverse; +-- After DROP: expect "Native object number two" +SELECT convert_from(lo_get(:'native_oid2'::oid), 'UTF8') AS obj2_after_reverse; +-- After DROP: expect "Created directly in lolor" +SELECT convert_from(lo_get(:'lolor_direct_oid'::oid), 'UTF8') AS obj3_after_reverse; + +-- Cleanup native LOs +SELECT lo_unlink(:'native_oid1'::oid); +SELECT lo_unlink(:'native_oid2'::oid); +SELECT lo_unlink(:'lolor_direct_oid'::oid); + +CREATE EXTENSION lolor; +SELECT lolor.migrate_from_native(); +DROP EXTENSION lolor; + +-- +-- Manual migrate_to_native (not via DROP EXTENSION) +-- +CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'Manual reverse test') AS manual_oid \gset +SELECT lolor.migrate_to_native(); +SELECT count(*) AS native_after_manual FROM pg_catalog.pg_largeobject_metadata; +SELECT count(*) AS lolor_after_manual FROM lolor.pg_largeobject_metadata; + +-- After manual migration: expect "Manual reverse test" +BEGIN; +-- Disable lolor to read from native storage directly +SELECT lolor.disable(); +SELECT convert_from(lo_get(:'manual_oid'::oid), 'UTF8') AS manual_data; +END; + +-- Cleanup +SELECT lo_unlink(:'manual_oid'::oid); +SELECT lolor.enable(); +DROP EXTENSION lolor; + +-- +-- OID conflict detection +-- + +-- OID conflict: migrate_from_native should ERROR on duplicate OID +SELECT lo_from_bytea(0, 'Conflict test object') AS conflict_oid \gset +CREATE EXTENSION lolor; +-- HACK: Manually insert a row with the same OID into lolor storage +INSERT INTO lolor.pg_largeobject_metadata (oid, lomowner, lomacl) + VALUES (:'conflict_oid', (SELECT oid FROM pg_roles WHERE rolname = current_user), NULL); +-- This should fail with OID conflict +SELECT lolor.migrate_from_native(); +-- Cleanup: remove the conflicting row and drop cleanly +DELETE FROM lolor.pg_largeobject_metadata WHERE oid = :'conflict_oid'; +DROP EXTENSION lolor; +SELECT lo_unlink(:'conflict_oid'::oid); + +-- OID conflict: migrate_to_native should ERROR on duplicate OID +CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'Lolor side object') AS conflict_oid2 \gset +-- Disable lolor to create a native LO with the same OID +SELECT lolor.disable(); +SELECT lo_create(:'conflict_oid2') AS created_oid \gset +-- Verify native lo_create honored the explicit OID +SELECT :'created_oid' = :'conflict_oid2' AS oid_matches; +SELECT lolor.enable(); +-- migrate_to_native should detect the collision +SELECT lolor.migrate_to_native(); +-- Cleanup: remove the native duplicate, then drop cleanly +SELECT lolor.disable(); +SELECT lo_unlink(:'conflict_oid2'::oid); +SELECT lolor.enable(); +DROP EXTENSION lolor; + +-- DROP EXTENSION should be rejected when migrate_to_native has OID conflict +CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'Drop conflict test') AS drop_conflict_oid \gset +-- Create a native LO with the same OID to force conflict at DROP time +SELECT lolor.disable(); +SELECT lo_create(:'drop_conflict_oid'); +SELECT lolor.enable(); +-- DROP EXTENSION should ERROR to prevent data loss +DROP EXTENSION lolor; +-- Extension should still be installed +SELECT extname FROM pg_extension WHERE extname = 'lolor'; +-- Objects should be in place +SELECT count(*) FROM lolor.pg_largeobject; +-- Resolve the conflict: remove the native duplicate, then retry +SELECT lolor.disable(); +SELECT lo_unlink(:'drop_conflict_oid'::oid); +SELECT lolor.enable(); +-- Now DROP should succeed +DROP EXTENSION lolor; diff --git a/src/lolor.c b/src/lolor.c index 66d5676..6b69810 100644 --- a/src/lolor.c +++ b/src/lolor.c @@ -240,15 +240,46 @@ lolor_on_drop_extension(PG_FUNCTION_ARGS) PG_RETURN_NULL(); /* - * OK, this is DROP EXTENSION lolor. Rename our own - * functions out of the way (they will later be dropped by the - * DROP EXTENSION itself, and rename the original PostgreSQL - * functions back to what they were. + * OK, this is DROP EXTENSION lolor. + * + * First, migrate any large objects stored in lolor tables back to + * native PostgreSQL storage. This must happen while lolor is still + * enabled so the _orig functions (native LO API) are available. + * The event trigger fires on ddl_command_start, so lolor tables + * still exist and are readable at this point. + * + * Then rename our replacement functions out of the way and restore + * the original PostgreSQL function names. The DROP EXTENSION itself + * will then drop the lolor schema and its objects. + * + * Guard the migrate_to_native() call with a pg_proc check so that + * upgrades from versions < 1.2.3 (where the function does not exist) + * do not fail. */ SPI_connect(); - SPI_execute("SELECT CASE WHEN lolor.is_enabled() THEN lolor.disable() ELSE 'true' END CASE", + if (SPI_execute("SELECT 1 FROM pg_proc p " + "JOIN pg_namespace n ON n.oid = p.pronamespace " + "WHERE n.nspname = 'lolor' " + "AND p.proname = 'migrate_to_native'", + true, 1) == SPI_OK_SELECT && + SPI_processed > 0) + { + /* + * If migrate_to_native() fails (e.g. OID conflict), the ERROR + * propagates and aborts the DROP EXTENSION. This is intentional: + * losing large objects silently is worse than a failed DROP. The + * user must resolve the conflict and retry. + */ + if (SPI_execute("SELECT lolor.migrate_to_native()", false, 0) != SPI_OK_SELECT) + ereport(ERROR, + (errmsg("lolor: failed to migrate large objects back to native storage"))); + } + + SPI_execute("SELECT CASE WHEN lolor.is_enabled() " + "THEN lolor.disable() ELSE true END", false, 0); + SPI_finish(); PG_RETURN_NULL(); diff --git a/t/006_promote_standby.pl b/t/006_promote_standby.pl new file mode 100644 index 0000000..147c0c3 --- /dev/null +++ b/t/006_promote_standby.pl @@ -0,0 +1,90 @@ +# Check lolor works after promoting a streaming standby +# +# Copyright (c) 2022-2026, pgEdge, Inc. +# + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +my ($result, $stdout, $stderr); + +# Setup primary node with lolor extension +$primary->init(allows_streaming => 1); +$primary->append_conf('postgresql.conf', qq{lolor.node = 1}); +$primary->start; +$primary->safe_psql('postgres', "CREATE EXTENSION lolor"); + +# Create lolor large objects on primary +$primary->safe_psql('postgres', + qq(SELECT lo_from_bytea(1, 'LO before standby'))); +$primary->safe_psql('postgres', + qq(SELECT lo_from_bytea(2, 'LO after standby setup'))); + +# Take a backup and create streaming standby +my $backup_name = 'my_backup'; +$primary->backup($backup_name); + +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, $backup_name, + has_streaming => 1); +$standby->start; + +# Create one more object on the primary and let it replicate +$primary->safe_psql('postgres', + qq(SELECT lo_from_bytea(3, 'LO streamed to standby'))); +$primary->wait_for_replay_catchup($standby); + +# Stop the primary to simulate a failover +$primary->stop; + +# Promote the standby +$standby->promote; + +# Verify all lolor objects are readable after promotion +$result = $standby->safe_psql('postgres', qq( + BEGIN; + SELECT lo_open(1, 262144) AS fd \\gset + SELECT convert_from(loread(:fd, 1024), 'UTF8'); + END; +)); +is($result, 'LO before standby', + "Pre-backup LO readable after promotion"); + +$result = $standby->safe_psql('postgres', qq( + BEGIN; + SELECT lo_open(2, 262144) AS fd \\gset + SELECT convert_from(loread(:fd, 1024), 'UTF8'); + END; +)); +is($result, 'LO after standby setup', + "Backup-time LO readable after promotion"); + +$result = $standby->safe_psql('postgres', qq( + BEGIN; + SELECT lo_open(3, 262144) AS fd \\gset + SELECT convert_from(loread(:fd, 1024), 'UTF8'); + END; +)); +is($result, 'LO streamed to standby', + "Streamed LO readable after promotion"); + +# Verify new lolor objects can be created on the promoted standby +$standby->safe_psql('postgres', + qq(SELECT lo_from_bytea(4, 'LO created after promotion'))); +$result = $standby->safe_psql('postgres', qq( + BEGIN; + SELECT lo_open(4, 262144) AS fd \\gset + SELECT convert_from(loread(:fd, 1024), 'UTF8'); + END; +)); +is($result, 'LO created after promotion', + "Can create and read new lolor objects after promotion"); + +$standby->stop; + +done_testing();