diff --git a/bin/installcheck b/bin/installcheck index 7496065..4855f17 100755 --- a/bin/installcheck +++ b/bin/installcheck @@ -41,7 +41,7 @@ REGRESS="${PGXS}/../test/regress/pg_regress" TESTS=$(ls ${TESTDIR}/sql | sed -e 's/\..*$//' | sort ) # Execute the test fixtures -psql -v ON_ERROR_STOP=1 -f sql/setup.sql -f sql/walrus--0.1.sql -f sql/walrus_migration_0001*.sql -f sql/walrus_migration_0002*.sql -f sql/walrus_migration_0003*.sql -f sql/walrus_migration_0004*.sql -f sql/walrus_migration_0005*.sql -f sql/walrus_migration_0006*.sql -f sql/walrus_migration_0007*.sql -f sql/walrus_migration_0008*.sql -f sql/walrus_migration_0009*.sql -f sql/walrus_migration_0010*.sql -f sql/walrus_migration_0011*.sql -f sql/walrus_migration_0012*.sql -f sql/walrus_migration_0013*.sql -f sql/walrus_migration_0014*.sql -f test/fixtures.sql -d contrib_regression +psql -v ON_ERROR_STOP=1 -f sql/setup.sql -f sql/walrus--0.1.sql -f sql/walrus_migration_0001*.sql -f sql/walrus_migration_0002*.sql -f sql/walrus_migration_0003*.sql -f sql/walrus_migration_0004*.sql -f sql/walrus_migration_0005*.sql -f sql/walrus_migration_0006*.sql -f sql/walrus_migration_0007*.sql -f sql/walrus_migration_0008*.sql -f sql/walrus_migration_0009*.sql -f sql/walrus_migration_0010*.sql -f sql/walrus_migration_0011*.sql -f sql/walrus_migration_0012*.sql -f sql/walrus_migration_0013*.sql -f sql/walrus_migration_0014*.sql -f sql/walrus_migration_0015*.sql -f test/fixtures.sql -d contrib_regression # Run tests ${REGRESS} --use-existing --dbname=contrib_regression --inputdir=${TESTDIR} ${TESTS} diff --git a/sql/walrus_migration_0015_like_ilike_is_not_ops.sql b/sql/walrus_migration_0015_like_ilike_is_not_ops.sql new file mode 100644 index 0000000..b976d11 --- /dev/null +++ b/sql/walrus_migration_0015_like_ilike_is_not_ops.sql @@ -0,0 +1,218 @@ +-- Fix 1: Drop old 4-arg overload and its dependent before creating the 5-arg version +drop function if exists realtime.is_visible_through_filters(realtime.wal_column[], realtime.user_defined_filter[]); +drop function if exists realtime.check_equality_op(realtime.equality_op, regtype, text, text); + +alter type realtime.equality_op add value 'like'; +alter type realtime.equality_op add value 'ilike'; +alter type realtime.equality_op add value 'is'; +alter type realtime.equality_op add value 'match'; +alter type realtime.equality_op add value 'imatch'; +alter type realtime.equality_op add value 'isdistinct'; + +alter type realtime.user_defined_filter add attribute negate boolean; + +drop function if exists realtime.check_equality_op(realtime.equality_op, regtype, text, text); + +create or replace function realtime.check_equality_op( + op realtime.equality_op, + type_ regtype, + val_1 text, + val_2 text, + negate boolean DEFAULT false +) + returns bool + stable -- Fix 2: was immutable, uses EXECUTE so must be stable + language plpgsql +as $$ +declare + op_symbol text; + res boolean; +begin + -- IS DISTINCT FROM / IS NOT DISTINCT FROM: infix, both sides typed literals + if op = 'isdistinct' then + execute format( + 'select %L::%s %s %L::%s', + val_1, + type_::text, + case when negate then 'IS NOT DISTINCT FROM' else 'IS DISTINCT FROM' end, + val_2, + type_::text + ) into res; + return res; + end if; + + -- IS requires a keyword RHS (NULL, TRUE, FALSE, UNKNOWN), not a typed literal + if op = 'is' then + if val_2 not in ('null', 'true', 'false', 'unknown') then + raise exception 'invalid value for is filter: must be null, true, false, or unknown'; + end if; + execute format( + 'select %L::%s %s %s', + val_1, + type_::text, + case when negate then 'IS NOT' else 'IS' end, + upper(val_2) + ) into res; + return res; + end if; + + op_symbol = case + when op = 'eq' then '=' + when op = 'neq' then '!=' + when op = 'lt' then '<' + when op = 'lte' then '<=' + when op = 'gt' then '>' + when op = 'gte' then '>=' + when op = 'in' then '= any' + when op = 'like' then 'LIKE' + when op = 'ilike' then 'ILIKE' + when op = 'match' then '~' + when op = 'imatch' then '~*' + else null + end; + + if op_symbol is null then + raise exception 'unsupported equality operator: %', op::text; + end if; + + execute format( + 'select %L::%s %s (%L::%s)', + val_1, + type_::text, + op_symbol, + val_2, + case when op = 'in' then type_::text || '[]' else type_::text end + ) into res; + + return case when negate then not res else res end; +end; +$$; + + +create or replace function realtime.is_visible_through_filters(columns realtime.wal_column[], filters realtime.user_defined_filter[]) + returns bool + language sql + stable -- Fix 2: was immutable, calls stable function so must be stable +as $$ + select + filters is null + or array_length(filters, 1) is null + or coalesce( + count(col.name) = count(1) + and sum( + realtime.check_equality_op( + op:=f.op, + type_:=coalesce(col.type_oid::regtype, col.type_name::regtype), + val_1:=col.value #>> '{}', + val_2:=f.value, + negate:=coalesce(f.negate, false) + )::int + ) filter (where col.name is not null) = count(col.name), + false + ) + from + unnest(filters) f + left join unnest(columns) col + on f.column_name = col.name; +$$; + + +create or replace function realtime.subscription_check_filters() + returns trigger + language plpgsql +as $$ +declare + col_names text[] = coalesce( + array_agg(c.column_name order by c.ordinal_position), + '{}'::text[] + ) + from + information_schema.columns c + where + format('%I.%I', c.table_schema, c.table_name)::regclass = new.entity + and pg_catalog.has_column_privilege( + (new.claims ->> 'role'), + format('%I.%I', c.table_schema, c.table_name)::regclass, + c.column_name, + 'SELECT' + ); + -- Fix 3: removed unused table_col_names declaration + filter realtime.user_defined_filter; + col_type regtype; + in_val jsonb; + selected_col text; +begin + for filter in select * from unnest(new.filters) loop + if not filter.column_name = any(col_names) then + raise exception 'invalid column for filter %', filter.column_name; + end if; + + col_type = ( + select atttypid::regtype + from pg_catalog.pg_attribute + where attrelid = new.entity + and attname = filter.column_name + ); + if col_type is null then + raise exception 'failed to lookup type for column %', filter.column_name; + end if; + + if filter.op = 'in'::realtime.equality_op then + in_val = realtime.cast(filter.value, (col_type::text || '[]')::regtype); + if coalesce(jsonb_array_length(in_val), 0) > 100 then + raise exception 'too many values for `in` filter. Maximum 100'; + end if; + elsif filter.op = 'is'::realtime.equality_op then + if filter.value not in ('null', 'true', 'false', 'unknown') then + raise exception 'invalid value for is filter: must be null, true, false, or unknown'; + end if; + -- Fix 4: validate like/ilike only applies to text-compatible columns + elsif filter.op in ('like'::realtime.equality_op, 'ilike'::realtime.equality_op) then + if not exists ( + select 1 from pg_catalog.pg_operator + where oprname = '~~' and oprleft = col_type + ) then + raise exception 'operator % requires a text-compatible column type, got %', filter.op::text, col_type::text; + end if; + -- Fix 5: validate match/imatch regex patterns eagerly + elsif filter.op in ('match'::realtime.equality_op, 'imatch'::realtime.equality_op) then + begin + perform '' ~ filter.value; + exception when others then + raise exception 'invalid regular expression for % filter: %', filter.op::text, sqlerrm; + end; + else + perform realtime.cast(filter.value, col_type); + end if; + end loop; + + if new.selected_columns = '{}' then + raise exception 'selected_columns cannot be empty. Remove the select parameter to capture all columns.'; + end if; + + if new.selected_columns is not null and array_position(new.selected_columns, null::text) is not null then + raise exception 'selected_columns cannot contain null values.'; + end if; + + if new.selected_columns is not null then + for selected_col in select * from unnest(new.selected_columns) loop + if not selected_col = any(col_names) then + raise exception 'invalid column for select %', selected_col; + end if; + end loop; + end if; + + -- Fix 6: include negate in the ORDER BY for deterministic normalization + new.filters = coalesce( + array_agg(f order by f.column_name, f.op, f.value, f.negate), + '{}' + ) from unnest(new.filters) f; + + new.selected_columns = ( + select array_agg(c order by c) + from unnest(new.selected_columns) c + ); + + return new; +end; +$$; diff --git a/test/expected/issue_40_quoted_regtype.out b/test/expected/issue_40_quoted_regtype.out index b6d5f4b..ed313de 100644 --- a/test/expected/issue_40_quoted_regtype.out +++ b/test/expected/issue_40_quoted_regtype.out @@ -25,7 +25,7 @@ select 'role', 'authenticated', 'sub', seed_uuid(2)::text ), - array[('primary_color', 'eq', 'RED')::realtime.user_defined_filter]; + array[('primary_color', 'eq', 'RED', null)::realtime.user_defined_filter]; insert into public.notes(id, primary_color) values (1, 'RED'), -- matches filter diff --git a/test/expected/issue_50_delete_filters.out b/test/expected/issue_50_delete_filters.out index 404e593..dfc2bd8 100644 --- a/test/expected/issue_50_delete_filters.out +++ b/test/expected/issue_50_delete_filters.out @@ -17,7 +17,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value)::realtime.user_defined_filter] + array[(column_name, op, value, null)::realtime.user_defined_filter] from ( values @@ -25,10 +25,10 @@ from (2 , 'id', 'eq', '2') ) f(id, column_name, op, value); select subscription_id, filters from realtime.subscription; - subscription_id | filters ---------------------------------------+------------------- - f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,eq,bbb)"} - 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(id,eq,2)"} + subscription_id | filters +--------------------------------------+-------------------- + f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,eq,bbb,)"} + 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(id,eq,2,)"} (2 rows) ---------------------------------------------------------------------------------------- @@ -160,6 +160,64 @@ from } | | | (2 rows) +---------------------------------------------------------------------------------------- +-- Multi-filter: missing column must not allow subscription to pass -- +---------------------------------------------------------------------------------------- +alter table public.notes replica identity default; +truncate table realtime.subscription; +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(3), + 'public.notes', + jsonb_build_object( + 'role', 'authenticated', + 'email', 'example@example.com', + 'sub', seed_uuid(3)::text + ), + array[ + ('id', 'eq', '1', null)::realtime.user_defined_filter, + ('body', 'eq', 'bbb', null)::realtime.user_defined_filter + ]; +insert into public.notes(id, body) values (1, 'bbb'); +select clear_wal(); + clear_wal +----------- + +(1 row) + +-- Non-full replica identity DELETE: only PK in WAL, body filter cannot be evaluated +-- Expect 0 subscriptions: body='bbb' filter is unverifiable, must fail closed +delete from public.notes; +select + rec, + is_rls_enabled, + subscription_ids, + errors +from + walrus; + rec | is_rls_enabled | subscription_ids | errors +----------------------------------------------------+----------------+------------------+-------- + { +| f | {} | {} + "type": "DELETE", +| | | + "table": "notes", +| | | + "schema": "public", +| | | + "columns": [ +| | | + { +| | | + "name": "id", +| | | + "type": "int4" +| | | + }, +| | | + { +| | | + "name": "body", +| | | + "type": "text" +| | | + } +| | | + ], +| | | + "old_record": { +| | | + "id": 1 +| | | + }, +| | | + "commit_timestamp": "2000-01-01T08:01:01.000Z"+| | | + } | | | +(1 row) + drop table public.notes; select pg_drop_replication_slot('realtime'); pg_drop_replication_slot diff --git a/test/expected/issue_55_null_passes_filters.out b/test/expected/issue_55_null_passes_filters.out index ce5c540..90288c1 100644 --- a/test/expected/issue_55_null_passes_filters.out +++ b/test/expected/issue_55_null_passes_filters.out @@ -17,7 +17,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(1)::text ), - array[('page_id', 'eq', '5')::realtime.user_defined_filter]; + array[('page_id', 'eq', '5', null)::realtime.user_defined_filter]; select clear_wal(); clear_wal ----------- diff --git a/test/expected/test_integration_filters.out b/test/expected/test_integration_filters.out index c3d74b7..a2bdf21 100644 --- a/test/expected/test_integration_filters.out +++ b/test/expected/test_integration_filters.out @@ -18,7 +18,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value)::realtime.user_defined_filter] + array[(column_name, op, value, null)::realtime.user_defined_filter] from ( values @@ -35,11 +35,11 @@ select clear_wal(); insert into public.notes(id, body) values (1, 'bbb'); delete from public.notes; select subscription_id, filters from realtime.subscription; - subscription_id | filters ---------------------------------------+-------------------- - f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,eq,bbb)"} - 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,eq,aaaa)"} - 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,eq,cc)"} + subscription_id | filters +--------------------------------------+--------------------- + f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,eq,bbb,)"} + 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,eq,aaaa,)"} + 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,eq,cc,)"} (3 rows) select diff --git a/test/expected/test_integration_in_filter.out b/test/expected/test_integration_in_filter.out index f2245d1..6edfd26 100644 --- a/test/expected/test_integration_in_filter.out +++ b/test/expected/test_integration_in_filter.out @@ -18,7 +18,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value)::realtime.user_defined_filter] + array[(column_name, op, value, null)::realtime.user_defined_filter] from ( values @@ -35,11 +35,11 @@ select clear_wal(); insert into public.notes(id, body) values (1, 'bbb'); delete from public.notes; select subscription_id, filters from realtime.subscription; - subscription_id | filters ---------------------------------------+--------------------------------- - f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,in,\"{aaa,bbb,ccc}\")"} - 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,in,\"{aaa,ccc}\")"} - 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,in,{})"} + subscription_id | filters +--------------------------------------+---------------------------------- + f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,in,\"{aaa,bbb,ccc}\",)"} + 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,in,\"{aaa,ccc}\",)"} + 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,in,{},)"} (3 rows) select @@ -103,7 +103,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(6)::text ), - array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1])::realtime.user_defined_filter]; + array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1], null)::realtime.user_defined_filter]; ERROR: too many values for `in` filter. Maximum 100 CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 41 at RAISE drop table public.notes; diff --git a/test/expected/test_integration_in_uuid_filter.out b/test/expected/test_integration_in_uuid_filter.out index bed1550..c327065 100644 --- a/test/expected/test_integration_in_uuid_filter.out +++ b/test/expected/test_integration_in_uuid_filter.out @@ -21,7 +21,8 @@ select array[( 'identifier', 'in', - '{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}' + '{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}', + null )::realtime.user_defined_filter]; select clear_wal(); clear_wal @@ -32,9 +33,9 @@ select clear_wal(); insert into public.notes(id, identifier) values (1, 'ace23052-568e-4951-acc8-fd510ec667f9'); delete from public.notes; select subscription_id, filters from realtime.subscription; - subscription_id | filters ---------------------------------------+----------------------------------------------------------------------------------------------------- - 3fa85983-bc94-5c16-8bc7-157b8152c678 | {"(identifier,in,\"{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}\")"} + subscription_id | filters +--------------------------------------+------------------------------------------------------------------------------------------------------ + 3fa85983-bc94-5c16-8bc7-157b8152c678 | {"(identifier,in,\"{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}\",)"} (1 row) select diff --git a/test/expected/test_integration_like_ilike_is_not_filters.out b/test/expected/test_integration_like_ilike_is_not_filters.out new file mode 100644 index 0000000..a1d9fcf --- /dev/null +++ b/test/expected/test_integration_like_ilike_is_not_filters.out @@ -0,0 +1,154 @@ +select 1 from pg_create_logical_replication_slot('realtime', 'wal2json', false); + ?column? +---------- + 1 +(1 row) + +create table public.notes( + id int primary key, + body text, + nullable_body text +); +alter table public.notes replica identity full; +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(id), + 'public.notes', + jsonb_build_object( + 'role', 'authenticated', + 'email', 'example@example.com', + 'sub', seed_uuid(id)::text + ), + array[(column_name, op, value, negate)::realtime.user_defined_filter] +from + ( + values + -- like: matches 'hello world' → visible + (1, 'body', 'like', '%world%', false), + -- like: does not match → not visible + (2, 'body', 'like', '%xyz%', false), + -- ilike: case-insensitive match → visible + (3, 'body', 'ilike', '%WORLD%', false), + -- NOT LIKE: row does not match pattern → visible + (4, 'body', 'like', '%xyz%', true), + -- NOT LIKE: row matches pattern → not visible + (5, 'body', 'like', '%world%', true), + -- NOT IN: 'hello world' outside the list → visible + (6, 'body', 'in', '{foo,bar}', true), + -- NOT IN: 'hello world' inside the list → not visible + (7, 'body', 'in', '{hello world,other}', true), + -- is null on nullable_body (null row) → visible + (8, 'nullable_body', 'is', 'null', false), + -- is not null on nullable_body (null row)→ not visible + (9, 'nullable_body', 'is', 'null', true) + ) f(id, column_name, op, value, negate); +select clear_wal(); + clear_wal +----------- + +(1 row) + +insert into public.notes(id, body, nullable_body) values (1, 'hello world', null); +delete from public.notes; +select subscription_id, filters from realtime.subscription order by subscription_id; + subscription_id | filters +--------------------------------------+----------------------------------------- + 0d2f6c5b-62a4-5cae-af96-780d1ff5441b | {"(nullable_body,is,null,t)"} + 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,ilike,%WORLD%,f)"} + 23407699-1647-5973-97ee-7848c18c8a4f | {"(nullable_body,is,null,f)"} + 2a5efacf-2c0a-5e59-861b-eff2079d1e2e | {"(body,in,\"{foo,bar}\",t)"} + 33b3e2e2-d91c-530c-9955-f89a94467a34 | {"(body,like,%xyz%,t)"} + 3fa85983-bc94-5c16-8bc7-157b8152c678 | {"(body,like,%world%,t)"} + 50737752-8d74-577e-9972-20ba646857af | {"(body,in,\"{hello world,other}\",t)"} + 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,like,%xyz%,f)"} + f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,like,%world%,f)"} +(9 rows) + +-- Expected visible subscription_ids: 1, 3, 4, 6, 8 +select + rec, + is_rls_enabled, + subscription_ids, + errors +from + walrus; + rec | is_rls_enabled | subscription_ids | errors +----------------------------------------------------+----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------- + { +| f | {f4539ebe-c779-5788-bbc1-2421ffaa8954,11955172-4e1d-5836-925f-2bcb7a287b87,33b3e2e2-d91c-530c-9955-f89a94467a34,2a5efacf-2c0a-5e59-861b-eff2079d1e2e,23407699-1647-5973-97ee-7848c18c8a4f} | {} + "type": "INSERT", +| | | + "table": "notes", +| | | + "record": { +| | | + "id": 1, +| | | + "body": "hello world", +| | | + "nullable_body": null +| | | + }, +| | | + "schema": "public", +| | | + "columns": [ +| | | + { +| | | + "name": "id", +| | | + "type": "int4" +| | | + }, +| | | + { +| | | + "name": "body", +| | | + "type": "text" +| | | + }, +| | | + { +| | | + "name": "nullable_body", +| | | + "type": "text" +| | | + } +| | | + ], +| | | + "commit_timestamp": "2000-01-01T08:01:01.000Z"+| | | + } | | | + { +| f | {f4539ebe-c779-5788-bbc1-2421ffaa8954,11955172-4e1d-5836-925f-2bcb7a287b87,33b3e2e2-d91c-530c-9955-f89a94467a34,2a5efacf-2c0a-5e59-861b-eff2079d1e2e,23407699-1647-5973-97ee-7848c18c8a4f} | {} + "type": "DELETE", +| | | + "table": "notes", +| | | + "schema": "public", +| | | + "columns": [ +| | | + { +| | | + "name": "id", +| | | + "type": "int4" +| | | + }, +| | | + { +| | | + "name": "body", +| | | + "type": "text" +| | | + }, +| | | + { +| | | + "name": "nullable_body", +| | | + "type": "text" +| | | + } +| | | + ], +| | | + "old_record": { +| | | + "id": 1, +| | | + "body": "hello world", +| | | + "nullable_body": null +| | | + }, +| | | + "commit_timestamp": "2000-01-01T08:01:01.000Z"+| | | + } | | | +(2 rows) + +-- Confirm is with invalid value is rejected at subscription time +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(10), + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'example@example.com', 'sub', seed_uuid(10)::text), + array[('body', 'is', 'invalid_value', false)::realtime.user_defined_filter]; +ERROR: invalid value for is filter: must be null, true, false, or unknown +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 45 at RAISE +-- Confirm in with more than 100 entries is rejected +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(11), + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'example@example.com', 'sub', seed_uuid(11)::text), + array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1], true)::realtime.user_defined_filter]; +ERROR: too many values for `in` filter. Maximum 100 +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 41 at RAISE +drop table public.notes; +select pg_drop_replication_slot('realtime'); + pg_drop_replication_slot +-------------------------- + +(1 row) + +truncate table realtime.subscription; diff --git a/test/expected/test_select_columns_empty.out b/test/expected/test_select_columns_empty.out index 20b8cf0..e5fd42d 100644 --- a/test/expected/test_select_columns_empty.out +++ b/test/expected/test_select_columns_empty.out @@ -16,6 +16,6 @@ select ), '{}'::text[]; ERROR: selected_columns cannot be empty. Remove the select parameter to capture all columns. -CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 52 at RAISE +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 68 at RAISE drop table public.notes; truncate table realtime.subscription; diff --git a/test/expected/test_select_columns_filter_not_in_select.out b/test/expected/test_select_columns_filter_not_in_select.out index 11ba40c..0aaa1a8 100644 --- a/test/expected/test_select_columns_filter_not_in_select.out +++ b/test/expected/test_select_columns_filter_not_in_select.out @@ -26,7 +26,7 @@ select 'sub', seed_uuid(1)::text ), array['body'], - array[('extra', 'eq', 'match')::realtime.user_defined_filter]; + array[('extra', 'eq', 'match', null)::realtime.user_defined_filter]; -- Matching row: filter on extra matches, but extra not in output select clear_wal(); clear_wal diff --git a/test/expected/test_select_columns_invalid.out b/test/expected/test_select_columns_invalid.out index 5a3901c..750a26c 100644 --- a/test/expected/test_select_columns_invalid.out +++ b/test/expected/test_select_columns_invalid.out @@ -16,6 +16,6 @@ select ), array['nonexistent_column']; ERROR: invalid column for select nonexistent_column -CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 64 at RAISE +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 78 at RAISE drop table public.notes; truncate table realtime.subscription; diff --git a/test/expected/test_select_columns_null_element.out b/test/expected/test_select_columns_null_element.out index 42b0357..fea7278 100644 --- a/test/expected/test_select_columns_null_element.out +++ b/test/expected/test_select_columns_null_element.out @@ -16,6 +16,6 @@ select ), array[null]::text[]; ERROR: selected_columns cannot contain null values. -CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 57 at RAISE +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 72 at RAISE drop table public.notes; truncate table realtime.subscription; diff --git a/test/expected/test_unit_like_ilike_is_not_filters.out b/test/expected/test_unit_like_ilike_is_not_filters.out new file mode 100644 index 0000000..82acf01 --- /dev/null +++ b/test/expected/test_unit_like_ilike_is_not_filters.out @@ -0,0 +1,230 @@ +-- like (negate=false) +select realtime.check_equality_op('like', 'text', 'hello world', '%world%', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('like', 'text', 'hello world', '%xyz%', false); + check_equality_op +------------------- + f +(1 row) + +-- like negated (NOT LIKE) +select realtime.check_equality_op('like', 'text', 'hello world', '%xyz%', true); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('like', 'text', 'hello world', '%world%', true); + check_equality_op +------------------- + f +(1 row) + +-- ilike (negate=false) +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%world%', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%xyz%', false); + check_equality_op +------------------- + f +(1 row) + +-- ilike negated (NOT ILIKE) +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%xyz%', true); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%world%', true); + check_equality_op +------------------- + f +(1 row) + +-- in (negate=false) +select realtime.check_equality_op('in', 'bigint', '2', '{1,2,3}', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('in', 'bigint', '4', '{1,2,3}', false); + check_equality_op +------------------- + f +(1 row) + +-- in negated (NOT IN) +select realtime.check_equality_op('in', 'bigint', '4', '{1,2,3}', true); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('in', 'bigint', '2', '{1,2,3}', true); + check_equality_op +------------------- + f +(1 row) + +-- is null +select realtime.check_equality_op('is', 'text', null, 'null', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('is', 'text', 'value', 'null', false); + check_equality_op +------------------- + f +(1 row) + +-- is not null (negate=true) +select realtime.check_equality_op('is', 'text', 'value', 'null', true); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('is', 'text', null, 'null', true); + check_equality_op +------------------- + f +(1 row) + +-- is true / false on boolean +select realtime.check_equality_op('is', 'boolean', 'true', 'true', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('is', 'boolean', 'false', 'true', false); + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('is', 'boolean', 'false', 'true', true); + check_equality_op +------------------- + t +(1 row) + +-- match (regex) +select realtime.check_equality_op('match', 'text', 'hello world', 'hel+o', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('match', 'text', 'hello world', '^world', false); + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('match', 'text', 'hello world', 'hel+o', true); + check_equality_op +------------------- + f +(1 row) + +-- imatch (case-insensitive regex) +select realtime.check_equality_op('imatch', 'text', 'Hello World', 'hel+o', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('imatch', 'text', 'Hello World', '^world', false); + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('imatch', 'text', 'Hello World', 'hel+o', true); + check_equality_op +------------------- + f +(1 row) + +-- isdistinct (IS DISTINCT FROM — differs from neq in NULL handling) +select realtime.check_equality_op('isdistinct', 'text', 'aaa', 'bbb', false); -- true + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('isdistinct', 'text', 'aaa', 'aaa', false); -- false + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('isdistinct', 'text', null, 'aaa', false); -- true (NULL IS DISTINCT FROM 'aaa') + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('isdistinct', 'text', null, 'aaa', true); -- false (IS NOT DISTINCT FROM) + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('isdistinct', 'text', null, null, false); -- false (NULL IS NOT DISTINCT FROM NULL) + check_equality_op +------------------- + f +(1 row) + +-- negate works on all existing operators +select realtime.check_equality_op('eq', 'text', 'aaa', 'aaa', true); -- NOT eq → false + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('neq', 'text', 'aaa', 'bbb', true); -- NOT neq → false + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('lt', 'bigint', '1', '2', true); -- NOT lt → false + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('gte', 'bigint', '5', '3', true); -- NOT gte → false + check_equality_op +------------------- + f +(1 row) + +-- negate defaults to false (existing callers unaffected) +select realtime.check_equality_op('eq', 'text', 'aaa', 'aaa'); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('in', 'bigint', '2', '{1,2,3}'); + check_equality_op +------------------- + t +(1 row) + diff --git a/test/sql/issue_40_quoted_regtype.sql b/test/sql/issue_40_quoted_regtype.sql index 02679fd..8a833b7 100644 --- a/test/sql/issue_40_quoted_regtype.sql +++ b/test/sql/issue_40_quoted_regtype.sql @@ -29,7 +29,7 @@ select 'role', 'authenticated', 'sub', seed_uuid(2)::text ), - array[('primary_color', 'eq', 'RED')::realtime.user_defined_filter]; + array[('primary_color', 'eq', 'RED', null)::realtime.user_defined_filter]; insert into public.notes(id, primary_color) values diff --git a/test/sql/issue_50_delete_filters.sql b/test/sql/issue_50_delete_filters.sql index 77d4253..daea76b 100644 --- a/test/sql/issue_50_delete_filters.sql +++ b/test/sql/issue_50_delete_filters.sql @@ -14,7 +14,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value)::realtime.user_defined_filter] + array[(column_name, op, value, null)::realtime.user_defined_filter] from ( values @@ -64,6 +64,45 @@ select clear_wal(); delete from public.notes; +select + rec, + is_rls_enabled, + subscription_ids, + errors +from + walrus; + + +---------------------------------------------------------------------------------------- +-- Multi-filter: missing column must not allow subscription to pass -- +---------------------------------------------------------------------------------------- + +alter table public.notes replica identity default; + +truncate table realtime.subscription; + +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(3), + 'public.notes', + jsonb_build_object( + 'role', 'authenticated', + 'email', 'example@example.com', + 'sub', seed_uuid(3)::text + ), + array[ + ('id', 'eq', '1', null)::realtime.user_defined_filter, + ('body', 'eq', 'bbb', null)::realtime.user_defined_filter + ]; + +insert into public.notes(id, body) values (1, 'bbb'); + +select clear_wal(); + +-- Non-full replica identity DELETE: only PK in WAL, body filter cannot be evaluated +-- Expect 0 subscriptions: body='bbb' filter is unverifiable, must fail closed +delete from public.notes; + select rec, is_rls_enabled, diff --git a/test/sql/issue_55_null_passes_filters.sql b/test/sql/issue_55_null_passes_filters.sql index adf2802..e3bbfe2 100644 --- a/test/sql/issue_55_null_passes_filters.sql +++ b/test/sql/issue_55_null_passes_filters.sql @@ -14,7 +14,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(1)::text ), - array[('page_id', 'eq', '5')::realtime.user_defined_filter]; + array[('page_id', 'eq', '5', null)::realtime.user_defined_filter]; select clear_wal(); diff --git a/test/sql/test_integration_filters.sql b/test/sql/test_integration_filters.sql index 63cbc83..2e37725 100644 --- a/test/sql/test_integration_filters.sql +++ b/test/sql/test_integration_filters.sql @@ -16,7 +16,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value)::realtime.user_defined_filter] + array[(column_name, op, value, null)::realtime.user_defined_filter] from ( values diff --git a/test/sql/test_integration_in_filter.sql b/test/sql/test_integration_in_filter.sql index 2608099..dab3eed 100644 --- a/test/sql/test_integration_in_filter.sql +++ b/test/sql/test_integration_in_filter.sql @@ -16,7 +16,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value)::realtime.user_defined_filter] + array[(column_name, op, value, null)::realtime.user_defined_filter] from ( values @@ -52,7 +52,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(6)::text ), - array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1])::realtime.user_defined_filter]; + array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1], null)::realtime.user_defined_filter]; drop table public.notes; select pg_drop_replication_slot('realtime'); diff --git a/test/sql/test_integration_in_uuid_filter.sql b/test/sql/test_integration_in_uuid_filter.sql index 5ac3385..9fb538e 100644 --- a/test/sql/test_integration_in_uuid_filter.sql +++ b/test/sql/test_integration_in_uuid_filter.sql @@ -19,7 +19,8 @@ select array[( 'identifier', 'in', - '{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}' + '{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}', + null )::realtime.user_defined_filter]; diff --git a/test/sql/test_integration_like_ilike_is_not_filters.sql b/test/sql/test_integration_like_ilike_is_not_filters.sql new file mode 100644 index 0000000..249c04c --- /dev/null +++ b/test/sql/test_integration_like_ilike_is_not_filters.sql @@ -0,0 +1,80 @@ +select 1 from pg_create_logical_replication_slot('realtime', 'wal2json', false); + +create table public.notes( + id int primary key, + body text, + nullable_body text +); + +alter table public.notes replica identity full; + +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(id), + 'public.notes', + jsonb_build_object( + 'role', 'authenticated', + 'email', 'example@example.com', + 'sub', seed_uuid(id)::text + ), + array[(column_name, op, value, negate)::realtime.user_defined_filter] +from + ( + values + -- like: matches 'hello world' → visible + (1, 'body', 'like', '%world%', false), + -- like: does not match → not visible + (2, 'body', 'like', '%xyz%', false), + -- ilike: case-insensitive match → visible + (3, 'body', 'ilike', '%WORLD%', false), + -- NOT LIKE: row does not match pattern → visible + (4, 'body', 'like', '%xyz%', true), + -- NOT LIKE: row matches pattern → not visible + (5, 'body', 'like', '%world%', true), + -- NOT IN: 'hello world' outside the list → visible + (6, 'body', 'in', '{foo,bar}', true), + -- NOT IN: 'hello world' inside the list → not visible + (7, 'body', 'in', '{hello world,other}', true), + -- is null on nullable_body (null row) → visible + (8, 'nullable_body', 'is', 'null', false), + -- is not null on nullable_body (null row)→ not visible + (9, 'nullable_body', 'is', 'null', true) + ) f(id, column_name, op, value, negate); + + +select clear_wal(); +insert into public.notes(id, body, nullable_body) values (1, 'hello world', null); + +delete from public.notes; + +select subscription_id, filters from realtime.subscription order by subscription_id; + +-- Expected visible subscription_ids: 1, 3, 4, 6, 8 +select + rec, + is_rls_enabled, + subscription_ids, + errors +from + walrus; + + +-- Confirm is with invalid value is rejected at subscription time +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(10), + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'example@example.com', 'sub', seed_uuid(10)::text), + array[('body', 'is', 'invalid_value', false)::realtime.user_defined_filter]; + +-- Confirm in with more than 100 entries is rejected +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(11), + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'example@example.com', 'sub', seed_uuid(11)::text), + array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1], true)::realtime.user_defined_filter]; + +drop table public.notes; +select pg_drop_replication_slot('realtime'); +truncate table realtime.subscription; diff --git a/test/sql/test_select_columns_filter_not_in_select.sql b/test/sql/test_select_columns_filter_not_in_select.sql index b927c78..77b8f28 100644 --- a/test/sql/test_select_columns_filter_not_in_select.sql +++ b/test/sql/test_select_columns_filter_not_in_select.sql @@ -24,7 +24,7 @@ select 'sub', seed_uuid(1)::text ), array['body'], - array[('extra', 'eq', 'match')::realtime.user_defined_filter]; + array[('extra', 'eq', 'match', null)::realtime.user_defined_filter]; -- Matching row: filter on extra matches, but extra not in output select clear_wal(); diff --git a/test/sql/test_unit_like_ilike_is_not_filters.sql b/test/sql/test_unit_like_ilike_is_not_filters.sql new file mode 100644 index 0000000..56ea901 --- /dev/null +++ b/test/sql/test_unit_like_ilike_is_not_filters.sql @@ -0,0 +1,63 @@ +-- like (negate=false) +select realtime.check_equality_op('like', 'text', 'hello world', '%world%', false); +select realtime.check_equality_op('like', 'text', 'hello world', '%xyz%', false); + +-- like negated (NOT LIKE) +select realtime.check_equality_op('like', 'text', 'hello world', '%xyz%', true); +select realtime.check_equality_op('like', 'text', 'hello world', '%world%', true); + +-- ilike (negate=false) +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%world%', false); +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%xyz%', false); + +-- ilike negated (NOT ILIKE) +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%xyz%', true); +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%world%', true); + +-- in (negate=false) +select realtime.check_equality_op('in', 'bigint', '2', '{1,2,3}', false); +select realtime.check_equality_op('in', 'bigint', '4', '{1,2,3}', false); + +-- in negated (NOT IN) +select realtime.check_equality_op('in', 'bigint', '4', '{1,2,3}', true); +select realtime.check_equality_op('in', 'bigint', '2', '{1,2,3}', true); + +-- is null +select realtime.check_equality_op('is', 'text', null, 'null', false); +select realtime.check_equality_op('is', 'text', 'value', 'null', false); + +-- is not null (negate=true) +select realtime.check_equality_op('is', 'text', 'value', 'null', true); +select realtime.check_equality_op('is', 'text', null, 'null', true); + +-- is true / false on boolean +select realtime.check_equality_op('is', 'boolean', 'true', 'true', false); +select realtime.check_equality_op('is', 'boolean', 'false', 'true', false); +select realtime.check_equality_op('is', 'boolean', 'false', 'true', true); + +-- match (regex) +select realtime.check_equality_op('match', 'text', 'hello world', 'hel+o', false); +select realtime.check_equality_op('match', 'text', 'hello world', '^world', false); +select realtime.check_equality_op('match', 'text', 'hello world', 'hel+o', true); + +-- imatch (case-insensitive regex) +select realtime.check_equality_op('imatch', 'text', 'Hello World', 'hel+o', false); +select realtime.check_equality_op('imatch', 'text', 'Hello World', '^world', false); +select realtime.check_equality_op('imatch', 'text', 'Hello World', 'hel+o', true); + +-- isdistinct (IS DISTINCT FROM — differs from neq in NULL handling) +select realtime.check_equality_op('isdistinct', 'text', 'aaa', 'bbb', false); -- true +select realtime.check_equality_op('isdistinct', 'text', 'aaa', 'aaa', false); -- false +select realtime.check_equality_op('isdistinct', 'text', null, 'aaa', false); -- true (NULL IS DISTINCT FROM 'aaa') +select realtime.check_equality_op('isdistinct', 'text', null, 'aaa', true); -- false (IS NOT DISTINCT FROM) +select realtime.check_equality_op('isdistinct', 'text', null, null, false); -- false (NULL IS NOT DISTINCT FROM NULL) + +-- negate works on all existing operators +select realtime.check_equality_op('eq', 'text', 'aaa', 'aaa', true); -- NOT eq → false +select realtime.check_equality_op('neq', 'text', 'aaa', 'bbb', true); -- NOT neq → false +select realtime.check_equality_op('lt', 'bigint', '1', '2', true); -- NOT lt → false +select realtime.check_equality_op('gte', 'bigint', '5', '3', true); -- NOT gte → false + +-- negate defaults to false (existing callers unaffected) +select realtime.check_equality_op('eq', 'text', 'aaa', 'aaa'); +select realtime.check_equality_op('in', 'bigint', '2', '{1,2,3}');