diff --git a/pg_lake_copy/tests/pytests/test_parquet_copy.py b/pg_lake_copy/tests/pytests/test_parquet_copy.py index f5521535e..85e895dd0 100644 --- a/pg_lake_copy/tests/pytests/test_parquet_copy.py +++ b/pg_lake_copy/tests/pytests/test_parquet_copy.py @@ -834,6 +834,76 @@ def test_md_array(pg_conn, duckdb_conn, tmp_path): assert result[0]["val"] == 2 +def test_copy_to_multidim_array_errors(pg_conn, tmp_path): + """ + Regression test for https://github.com/Snowflake-Labs/pg_lake/issues/407. + + COPY TO must raise a clear pg_lake error when a column + contains a multidimensional array value. Previously the value was + serialised into the intermediate temp CSV and handed to DuckDB, which + either returned a cryptic Conversion Error or crashed the shared + pgduck_server process (issue #408). + + The check lives in CopyOneRowTo (csv_writer.c) and fires before the CSV + is written, so the engine is never involved. + """ + parquet_path = tmp_path / "test_multidim_err.parquet" + + run_command( + """ + CREATE TABLE test_multidim_err (id bigint, v int[]); + INSERT INTO test_multidim_err VALUES (1, ARRAY[[1,2],[3,4]]); + """, + pg_conn, + ) + + error = run_command( + f"COPY test_multidim_err TO '{parquet_path}' WITH (format 'parquet')", + pg_conn, + raise_error=False, + ) + + assert error is not None, "Expected an error for multidimensional array in COPY TO" + assert ( + "multidimensional arrays are not supported" in error.lower() + ), f"Unexpected error message: {error}" + + pg_conn.rollback() + + +def test_copy_to_1d_array_succeeds(pg_conn, duckdb_conn, tmp_path): + """ + Regression guard: 1-D array columns must still round-trip correctly through + COPY TO after the multidim check is added. A 1-D value has ARR_NDIM == 1 + and must not be rejected. + """ + parquet_path = tmp_path / "test_1d_array.parquet" + + run_command( + f""" + CREATE TABLE test_1d_array (id bigint, tags text[]); + INSERT INTO test_1d_array VALUES + (1, ARRAY['a', 'b', 'c']), + (2, NULL), + (3, ARRAY['x']); + COPY test_1d_array TO '{parquet_path}' WITH (format 'parquet'); + """, + pg_conn, + ) + + duckdb_conn.execute( + "SELECT id, tags FROM read_parquet($1) ORDER BY id", [str(parquet_path)] + ) + rows = duckdb_conn.fetchall() + + assert len(rows) == 3 + assert rows[0] == (1, ["a", "b", "c"]) + assert rows[1] == (2, None) + assert rows[2] == (3, ["x"]) + + pg_conn.rollback() + + def test_copy_virtual_column(pg_conn, tmp_path): # virtual columns were introduced in PostgreSQL 18 if get_pg_version_num(pg_conn) < 180000: diff --git a/pg_lake_engine/src/csv/csv_writer.c b/pg_lake_engine/src/csv/csv_writer.c index 3b91870fe..f0945560b 100644 --- a/pg_lake_engine/src/csv/csv_writer.c +++ b/pg_lake_engine/src/csv/csv_writer.c @@ -43,6 +43,7 @@ #include "nodes/execnodes.h" #include "nodes/makefuncs.h" #include "port/pg_bswap.h" +#include "utils/array.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/memutils.h" @@ -770,6 +771,36 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) */ Form_pg_attribute attr = TupleDescAttr(slot->tts_tupleDescriptor, attnum - 1); + /* + * Reject multidimensional arrays before serialization. + * PostgreSQL cannot distinguish int[] from int[][] at the + * type level, so a value with ndim > 1 would be serialised as + * "[[1,2],[3,4]]" and DuckDB cannot cast that string back to + * a flat LIST(T). For Iceberg tables this is handled + * upstream by IcebergErrorOrClampDatum; for plain COPY TO + * there is no such guard, so we raise here. Check before + * serialisation so we do not pay the cost of PGDuckSerialize + * on a value we will reject. + */ + if (get_element_type(attr->atttypid) != InvalidOid && + cstate->targetFormat != DATA_FORMAT_ICEBERG) + { + ArrayType *arr = DatumGetArrayTypeP(value); + + if (ARR_NDIM(arr) > 1) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("multidimensional arrays are not supported" + " in COPY TO"), + errdetail("Column \"%s\" contains a" + " %d-dimensional array value.", + NameStr(attr->attname), + ARR_NDIM(arr)), + errhint("Flatten the array to one dimension before" + " exporting, or write to an Iceberg table" + " with out_of_range_values = 'clamp'."))); + } + if (ShouldUseDuckSerialization(cstate->targetFormat, MakePGType(attr->atttypid, attr->atttypmod))) { /* diff --git a/pg_lake_engine/src/pgduck/write_data.c b/pg_lake_engine/src/pgduck/write_data.c index 3f91dc1f6..b00003eae 100644 --- a/pg_lake_engine/src/pgduck/write_data.c +++ b/pg_lake_engine/src/pgduck/write_data.c @@ -93,8 +93,11 @@ ConvertCSVFileTo(char *csvFilePath, TupleDesc csvTupleDesc, int maxLineSize, bool queryHasRowIds = false; /* - * CSV data is already clamped by WriteInsertRecord and converted to - * struct for Iceberg + * When reached from the Iceberg FDW INSERT path, the CSV data has already + * been clamped by WriteInsertRecord (via ClampAndCheckConstraints → + * IcebergErrorOrClampSlotInPlace). When reached from the plain COPY TO + * path (ProcessPgLakeCopyTo), multidimensional array values are rejected + * earlier in CopyOneRowTo, so they never appear in the CSV. */ return WriteQueryResultTo(command.data, destinationPath,