Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions pg_lake_copy/tests/pytests/test_parquet_copy.py
Original file line number Diff line number Diff line change
Expand Up @@ -834,6 +834,76 @@ def test_md_array(pg_conn, duckdb_conn, tmp_path):
assert result[0]["val"] == 2


def test_copy_to_multidim_array_errors(pg_conn, tmp_path):
"""
Regression test for https://github.com/Snowflake-Labs/pg_lake/issues/407.

COPY <table> TO <lake-file> must raise a clear pg_lake error when a column
contains a multidimensional array value. Previously the value was
serialised into the intermediate temp CSV and handed to DuckDB, which
either returned a cryptic Conversion Error or crashed the shared
pgduck_server process (issue #408).

The check lives in CopyOneRowTo (csv_writer.c) and fires before the CSV
is written, so the engine is never involved.
"""
parquet_path = tmp_path / "test_multidim_err.parquet"

run_command(
"""
CREATE TABLE test_multidim_err (id bigint, v int[]);
INSERT INTO test_multidim_err VALUES (1, ARRAY[[1,2],[3,4]]);
""",
pg_conn,
)

error = run_command(
f"COPY test_multidim_err TO '{parquet_path}' WITH (format 'parquet')",
pg_conn,
raise_error=False,
)

assert error is not None, "Expected an error for multidimensional array in COPY TO"
assert (
"multidimensional arrays are not supported" in error.lower()
), f"Unexpected error message: {error}"

pg_conn.rollback()


def test_copy_to_1d_array_succeeds(pg_conn, duckdb_conn, tmp_path):
"""
Regression guard: 1-D array columns must still round-trip correctly through
COPY TO after the multidim check is added. A 1-D value has ARR_NDIM == 1
and must not be rejected.
"""
parquet_path = tmp_path / "test_1d_array.parquet"

run_command(
f"""
CREATE TABLE test_1d_array (id bigint, tags text[]);
INSERT INTO test_1d_array VALUES
(1, ARRAY['a', 'b', 'c']),
(2, NULL),
(3, ARRAY['x']);
COPY test_1d_array TO '{parquet_path}' WITH (format 'parquet');
""",
pg_conn,
)

duckdb_conn.execute(
"SELECT id, tags FROM read_parquet($1) ORDER BY id", [str(parquet_path)]
)
rows = duckdb_conn.fetchall()

assert len(rows) == 3
assert rows[0] == (1, ["a", "b", "c"])
assert rows[1] == (2, None)
assert rows[2] == (3, ["x"])

pg_conn.rollback()


def test_copy_virtual_column(pg_conn, tmp_path):
# virtual columns were introduced in PostgreSQL 18
if get_pg_version_num(pg_conn) < 180000:
Expand Down
31 changes: 31 additions & 0 deletions pg_lake_engine/src/csv/csv_writer.c
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
#include "nodes/execnodes.h"
#include "nodes/makefuncs.h"
#include "port/pg_bswap.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
Expand Down Expand Up @@ -770,6 +771,36 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot)
*/
Form_pg_attribute attr = TupleDescAttr(slot->tts_tupleDescriptor, attnum - 1);

/*
* Reject multidimensional arrays before serialization.
* PostgreSQL cannot distinguish int[] from int[][] at the
* type level, so a value with ndim > 1 would be serialised as
* "[[1,2],[3,4]]" and DuckDB cannot cast that string back to
* a flat LIST(T). For Iceberg tables this is handled
* upstream by IcebergErrorOrClampDatum; for plain COPY TO
* there is no such guard, so we raise here. Check before
* serialisation so we do not pay the cost of PGDuckSerialize
* on a value we will reject.
*/
if (get_element_type(attr->atttypid) != InvalidOid &&
cstate->targetFormat != DATA_FORMAT_ICEBERG)
{
ArrayType *arr = DatumGetArrayTypeP(value);

if (ARR_NDIM(arr) > 1)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("multidimensional arrays are not supported"
" in COPY TO"),
errdetail("Column \"%s\" contains a"
" %d-dimensional array value.",
NameStr(attr->attname),
ARR_NDIM(arr)),
errhint("Flatten the array to one dimension before"
" exporting, or write to an Iceberg table"
" with out_of_range_values = 'clamp'.")));
}

if (ShouldUseDuckSerialization(cstate->targetFormat, MakePGType(attr->atttypid, attr->atttypmod)))
{
/*
Expand Down
7 changes: 5 additions & 2 deletions pg_lake_engine/src/pgduck/write_data.c
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,11 @@ ConvertCSVFileTo(char *csvFilePath, TupleDesc csvTupleDesc, int maxLineSize,
bool queryHasRowIds = false;

/*
* CSV data is already clamped by WriteInsertRecord and converted to
* struct for Iceberg
* When reached from the Iceberg FDW INSERT path, the CSV data has already
* been clamped by WriteInsertRecord (via ClampAndCheckConstraints →
* IcebergErrorOrClampSlotInPlace). When reached from the plain COPY TO
* path (ProcessPgLakeCopyTo), multidimensional array values are rejected
* earlier in CopyOneRowTo, so they never appear in the CSV.
*/
return WriteQueryResultTo(command.data,
destinationPath,
Expand Down
Loading