From 3faba6203380b785b72ffa1365329a5159c7e70e Mon Sep 17 00:00:00 2001 From: egolearner Date: Sat, 25 Apr 2026 22:18:50 +0800 Subject: [PATCH 1/7] fix(dataset): Propagate CSV parse delimiter to write options When writing a dataset with CsvFileFormat configured with a custom ParseOptions delimiter (e.g. delimiter=">"), the output CSV still used the default "," delimiter. This was because: - C++ CsvFileFormat::DefaultWriteOptions() always created WriteOptions::Defaults() with delimiter=',', ignoring the parse_options.delimiter stored on the format object. - Python CsvFileFormat.make_write_options() unconditionally overwrote the C++ write options with a fresh WriteOptions(**kwargs), discarding any C++-side default that might have been set. Fix the C++ side by propagating parse_options.delimiter into the write options in DefaultWriteOptions(). Fix the Python side by reading the delimiter from the C++ default when the caller does not explicitly specify one, preventing the overwrite from losing the propagated value. --- cpp/src/arrow/dataset/file_csv.cc | 1 + python/pyarrow/_dataset.pyx | 2 ++ python/pyarrow/tests/test_csv.py | 46 +++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/cpp/src/arrow/dataset/file_csv.cc b/cpp/src/arrow/dataset/file_csv.cc index cede2681070e..c0d85581f633 100644 --- a/cpp/src/arrow/dataset/file_csv.cc +++ b/cpp/src/arrow/dataset/file_csv.cc @@ -474,6 +474,7 @@ std::shared_ptr CsvFileFormat::DefaultWriteOptions() { new CsvFileWriteOptions(shared_from_this())); csv_options->write_options = std::make_shared(csv::WriteOptions::Defaults()); + csv_options->write_options->delimiter = parse_options.delimiter; return csv_options; } diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 666fd2c1cc50..d02d1345a46e 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -2253,6 +2253,8 @@ cdef class CsvFileFormat(FileFormat): """ cdef CsvFileWriteOptions opts = \ FileFormat.make_write_options(self) + if 'delimiter' not in kwargs: + kwargs['delimiter'] = opts.write_options.delimiter opts.write_options = WriteOptions(**kwargs) return opts diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index d608d2bee5eb..5dea0dc0e275 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -2149,3 +2149,49 @@ def test_write_csv_empty_batch_should_not_pollute_output(tables, expected): result = buf.read() assert result == expected + + +def test_write_csv_custom_delimiter_from_parse_options(): + import pyarrow.dataset as ds + + table = pa.table({ + "B": ["B1", "B2"], + "C": ["C1", "C2"], + }) + + # Verify that CsvFileFormat.make_write_options propagates the + # parse delimiter to write options when no explicit delimiter is given + for delimiter in [">", "|", "\t", ";"]: + csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=delimiter)) + write_opts = csv_format.make_write_options() + assert write_opts.write_options.delimiter == delimiter + + # Verify that an explicitly passed delimiter takes precedence + csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">")) + write_opts = csv_format.make_write_options(delimiter="|") + assert write_opts.write_options.delimiter == "|" + + # Verify the default delimiter is still "," when no parse options are given + csv_format = ds.CsvFileFormat() + write_opts = csv_format.make_write_options() + assert write_opts.write_options.delimiter == "," + + # Verify end-to-end: write_dataset with custom delimiter produces + # output using that delimiter, not the default "," + with tempfile.TemporaryDirectory() as tmpdir: + csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">")) + ds.write_dataset(table, tmpdir, format=csv_format) + + # Check that written CSV files use the custom delimiter + for root, dirs, files in os.walk(tmpdir): + for f in files: + with open(os.path.join(root, f)) as fh: + content = fh.read() + assert ">" in content, ( + f"Expected '>' delimiter in CSV output, got: {content!r}" + ) + + # Read back and verify roundtrip + read_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">")) + result = ds.dataset(tmpdir, format=read_format).to_table() + assert result.equals(table) From fa828d60a1ba0c657dae85f3a16c66d37891db5c Mon Sep 17 00:00:00 2001 From: egolearner Date: Thu, 30 Apr 2026 23:06:19 +0800 Subject: [PATCH 2/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- python/pyarrow/_dataset.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index d02d1345a46e..1fe553ae9e4d 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -2253,7 +2253,7 @@ cdef class CsvFileFormat(FileFormat): """ cdef CsvFileWriteOptions opts = \ FileFormat.make_write_options(self) - if 'delimiter' not in kwargs: + if kwargs.get('delimiter') is None: kwargs['delimiter'] = opts.write_options.delimiter opts.write_options = WriteOptions(**kwargs) return opts From e4704a8cd487b38e68f8f3aa4f53f699d8aba0f6 Mon Sep 17 00:00:00 2001 From: egolearner Date: Thu, 30 Apr 2026 23:17:08 +0800 Subject: [PATCH 3/7] add cpp csv delimiter test --- cpp/src/arrow/dataset/file_csv_test.cc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cpp/src/arrow/dataset/file_csv_test.cc b/cpp/src/arrow/dataset/file_csv_test.cc index e8e5838e6f93..e09ef49a4325 100644 --- a/cpp/src/arrow/dataset/file_csv_test.cc +++ b/cpp/src/arrow/dataset/file_csv_test.cc @@ -449,6 +449,24 @@ TEST_P(TestCsvFileFormat, WriteRecordBatchReaderCustomOptions) { ASSERT_EQ("0\n0\n0\n0\n0\n", written->ToString()); } +TEST_P(TestCsvFileFormat, WriteRecordBatchReaderDelimiterPropagation) { + // Verify that CsvFileFormat::DefaultWriteOptions() propagates the parse + // delimiter to write options + for (const char delim : {'>', '|', '\t', ';'}) { + auto csv_format = std::make_shared(); + csv_format->parse_options.delimiter = delim; + auto write_options = + checked_pointer_cast(csv_format->DefaultWriteOptions()); + ASSERT_EQ(write_options->write_options->delimiter, delim); + } + + // Verify that the default delimiter is ',' when no parse options are set + auto default_format = std::make_shared(); + auto default_write_options = + checked_pointer_cast(default_format->DefaultWriteOptions()); + ASSERT_EQ(default_write_options->write_options->delimiter, ','); +} + TEST_P(TestCsvFileFormat, CountRows) { TestCountRows(); } TEST_P(TestCsvFileFormat, FragmentEquals) { TestFragmentEquals(); } From 4c1f8aed8d41d5da00c1f07d5e91f36db29802f1 Mon Sep 17 00:00:00 2001 From: egolearner Date: Sat, 16 May 2026 21:04:26 +0800 Subject: [PATCH 4/7] resolve comments --- cpp/src/arrow/dataset/file_csv_test.cc | 12 +++--- python/pyarrow/tests/test_csv.py | 60 ++++++++++++-------------- 2 files changed, 32 insertions(+), 40 deletions(-) diff --git a/cpp/src/arrow/dataset/file_csv_test.cc b/cpp/src/arrow/dataset/file_csv_test.cc index e09ef49a4325..9ccc1140adbf 100644 --- a/cpp/src/arrow/dataset/file_csv_test.cc +++ b/cpp/src/arrow/dataset/file_csv_test.cc @@ -452,13 +452,11 @@ TEST_P(TestCsvFileFormat, WriteRecordBatchReaderCustomOptions) { TEST_P(TestCsvFileFormat, WriteRecordBatchReaderDelimiterPropagation) { // Verify that CsvFileFormat::DefaultWriteOptions() propagates the parse // delimiter to write options - for (const char delim : {'>', '|', '\t', ';'}) { - auto csv_format = std::make_shared(); - csv_format->parse_options.delimiter = delim; - auto write_options = - checked_pointer_cast(csv_format->DefaultWriteOptions()); - ASSERT_EQ(write_options->write_options->delimiter, delim); - } + auto csv_format = std::make_shared(); + csv_format->parse_options.delimiter = '|'; + auto write_options = + checked_pointer_cast(csv_format->DefaultWriteOptions()); + ASSERT_EQ(write_options->write_options->delimiter, '|'); // Verify that the default delimiter is ',' when no parse options are set auto default_format = std::make_shared(); diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index 5dea0dc0e275..2197b63b6d34 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -2151,47 +2151,41 @@ def test_write_csv_empty_batch_should_not_pollute_output(tables, expected): assert result == expected -def test_write_csv_custom_delimiter_from_parse_options(): +def test_csv_make_write_options_uses_parse_delimiter(): import pyarrow.dataset as ds - table = pa.table({ - "B": ["B1", "B2"], - "C": ["C1", "C2"], - }) - - # Verify that CsvFileFormat.make_write_options propagates the - # parse delimiter to write options when no explicit delimiter is given - for delimiter in [">", "|", "\t", ";"]: - csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=delimiter)) - write_opts = csv_format.make_write_options() - assert write_opts.write_options.delimiter == delimiter + # CsvFileFormat.make_write_options propagates the parse delimiter + # to write options when no explicit delimiter is given + csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter="|")) + write_opts = csv_format.make_write_options() + assert write_opts.write_options.delimiter == "|" - # Verify that an explicitly passed delimiter takes precedence + # An explicitly passed delimiter takes precedence csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">")) write_opts = csv_format.make_write_options(delimiter="|") assert write_opts.write_options.delimiter == "|" - # Verify the default delimiter is still "," when no parse options are given + # The default delimiter is still "," when no parse options are given csv_format = ds.CsvFileFormat() write_opts = csv_format.make_write_options() assert write_opts.write_options.delimiter == "," - # Verify end-to-end: write_dataset with custom delimiter produces - # output using that delimiter, not the default "," - with tempfile.TemporaryDirectory() as tmpdir: - csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">")) - ds.write_dataset(table, tmpdir, format=csv_format) - - # Check that written CSV files use the custom delimiter - for root, dirs, files in os.walk(tmpdir): - for f in files: - with open(os.path.join(root, f)) as fh: - content = fh.read() - assert ">" in content, ( - f"Expected '>' delimiter in CSV output, got: {content!r}" - ) - - # Read back and verify roundtrip - read_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">")) - result = ds.dataset(tmpdir, format=read_format).to_table() - assert result.equals(table) + +def test_write_dataset_uses_csv_parse_delimiter(tmp_path): + import pyarrow.dataset as ds + + table = pa.table({ + "B": ["B1", "B2"], + "C": ["C1", "C2"], + }) + + csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">")) + ds.write_dataset(table, tmp_path, format=csv_format) + + with open(tmp_path / "part-0.csv") as fh: + content = fh.read() + assert content == "B>C\nB1>C1\nB2>C2\n" + + # Roundtrip: reading back with the same delimiter recovers the table + result = ds.dataset(tmp_path, format=csv_format).to_table() + assert result.equals(table) From 52954d33aa01addd4aab8a2dc7ba0c54e04c9f4c Mon Sep 17 00:00:00 2001 From: egolearner Date: Sat, 16 May 2026 21:23:55 +0800 Subject: [PATCH 5/7] fix ut --- python/pyarrow/tests/test_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index 2197b63b6d34..3b8c5d5a119f 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -2184,7 +2184,7 @@ def test_write_dataset_uses_csv_parse_delimiter(tmp_path): with open(tmp_path / "part-0.csv") as fh: content = fh.read() - assert content == "B>C\nB1>C1\nB2>C2\n" + assert content == '"B">"C"\n"B1">"C1"\n"B2">"C2"\n' # Roundtrip: reading back with the same delimiter recovers the table result = ds.dataset(tmp_path, format=csv_format).to_table() From 74fd13bce3bdc833737aeee238c79af4ef83d244 Mon Sep 17 00:00:00 2001 From: egolearner Date: Wed, 20 May 2026 23:50:35 +0800 Subject: [PATCH 6/7] resolve comments --- python/pyarrow/_dataset.pyx | 11 ++++++++--- python/pyarrow/tests/test_csv.py | 20 -------------------- python/pyarrow/tests/test_dataset.py | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 1fe553ae9e4d..f0325b04b265 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -2253,9 +2253,14 @@ cdef class CsvFileFormat(FileFormat): """ cdef CsvFileWriteOptions opts = \ FileFormat.make_write_options(self) - if kwargs.get('delimiter') is None: - kwargs['delimiter'] = opts.write_options.delimiter - opts.write_options = WriteOptions(**kwargs) + # Start from the C++ defaults, which carry over fields from the + # format's parse_options (e.g. the delimiter), and apply caller + # overrides on top instead of replacing the WriteOptions object + # and discarding those defaults. + write_options = opts.write_options + for key, value in kwargs.items(): + setattr(write_options, key, value) + opts.write_options = write_options return opts @property diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index 19560beb69ce..eabe00eec8aa 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -2239,23 +2239,3 @@ def test_csv_make_write_options_uses_parse_delimiter(): csv_format = ds.CsvFileFormat() write_opts = csv_format.make_write_options() assert write_opts.write_options.delimiter == "," - - -def test_write_dataset_uses_csv_parse_delimiter(tmp_path): - import pyarrow.dataset as ds - - table = pa.table({ - "B": ["B1", "B2"], - "C": ["C1", "C2"], - }) - - csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">")) - ds.write_dataset(table, tmp_path, format=csv_format) - - with open(tmp_path / "part-0.csv") as fh: - content = fh.read() - assert content == '"B">"C"\n"B1">"C1"\n"B2">"C2"\n' - - # Roundtrip: reading back with the same delimiter recovers the table - result = ds.dataset(tmp_path, format=csv_format).to_table() - assert result.equals(table) diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index 3afe3281cbc3..7efe9b6fa285 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -4983,6 +4983,24 @@ def test_write_dataset_csv(tempdir): assert result.equals(table) +def test_write_dataset_uses_csv_parse_delimiter(tempdir): + table = pa.table({ + "B": ["B1", "B2"], + "C": ["C1", "C2"], + }) + + csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">")) + ds.write_dataset(table, tempdir, format=csv_format) + + with open(tempdir / "part-0.csv") as fh: + content = fh.read() + assert content == '"B">"C"\n"B1">"C1"\n"B2">"C2"\n' + + # Roundtrip: reading back with the same delimiter recovers the table + result = ds.dataset(tempdir, format=csv_format).to_table() + assert result.equals(table) + + @pytest.mark.parquet def test_write_dataset_parquet_file_visitor(tempdir): table = pa.table([ From c6aba6e1c8a0744feffc4771cc7c22bd501412c9 Mon Sep 17 00:00:00 2001 From: egolearner Date: Thu, 21 May 2026 09:10:23 +0800 Subject: [PATCH 7/7] resolve comments --- python/pyarrow/_dataset.pyx | 5 ++++- python/pyarrow/tests/test_csv.py | 20 -------------------- python/pyarrow/tests/test_dataset.py | 23 +++++++++++++++++++++++ 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index f0325b04b265..848bfee293c2 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -2256,9 +2256,12 @@ cdef class CsvFileFormat(FileFormat): # Start from the C++ defaults, which carry over fields from the # format's parse_options (e.g. the delimiter), and apply caller # overrides on top instead of replacing the WriteOptions object - # and discarding those defaults. + # and discarding those defaults. None is treated as "unspecified" + # to match the previous WriteOptions(**kwargs) semantics. write_options = opts.write_options for key, value in kwargs.items(): + if value is None: + continue setattr(write_options, key, value) opts.write_options = write_options return opts diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index eabe00eec8aa..ac9012ebdf63 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -2219,23 +2219,3 @@ def test_write_csv_empty_batch_should_not_pollute_output(tables, expected): result = buf.read() assert result == expected - - -def test_csv_make_write_options_uses_parse_delimiter(): - import pyarrow.dataset as ds - - # CsvFileFormat.make_write_options propagates the parse delimiter - # to write options when no explicit delimiter is given - csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter="|")) - write_opts = csv_format.make_write_options() - assert write_opts.write_options.delimiter == "|" - - # An explicitly passed delimiter takes precedence - csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">")) - write_opts = csv_format.make_write_options(delimiter="|") - assert write_opts.write_options.delimiter == "|" - - # The default delimiter is still "," when no parse options are given - csv_format = ds.CsvFileFormat() - write_opts = csv_format.make_write_options() - assert write_opts.write_options.delimiter == "," diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index 7efe9b6fa285..e5c173b139ab 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -4983,6 +4983,29 @@ def test_write_dataset_csv(tempdir): assert result.equals(table) +def test_csv_make_write_options_uses_parse_delimiter(): + # CsvFileFormat.make_write_options propagates the parse delimiter + # to write options when no explicit delimiter is given + csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter="|")) + write_opts = csv_format.make_write_options() + assert write_opts.write_options.delimiter == "|" + + # delimiter=None is treated as "unspecified" and the propagated + # value is preserved (matches WriteOptions(**kwargs) semantics) + write_opts = csv_format.make_write_options(delimiter=None) + assert write_opts.write_options.delimiter == "|" + + # An explicitly passed delimiter takes precedence + csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">")) + write_opts = csv_format.make_write_options(delimiter="|") + assert write_opts.write_options.delimiter == "|" + + # The default delimiter is still "," when no parse options are given + csv_format = ds.CsvFileFormat() + write_opts = csv_format.make_write_options() + assert write_opts.write_options.delimiter == "," + + def test_write_dataset_uses_csv_parse_delimiter(tempdir): table = pa.table({ "B": ["B1", "B2"],