diff --git a/cpp/src/arrow/dataset/file_csv.cc b/cpp/src/arrow/dataset/file_csv.cc index cede2681070..c0d85581f63 100644 --- a/cpp/src/arrow/dataset/file_csv.cc +++ b/cpp/src/arrow/dataset/file_csv.cc @@ -474,6 +474,7 @@ std::shared_ptr CsvFileFormat::DefaultWriteOptions() { new CsvFileWriteOptions(shared_from_this())); csv_options->write_options = std::make_shared(csv::WriteOptions::Defaults()); + csv_options->write_options->delimiter = parse_options.delimiter; return csv_options; } diff --git a/cpp/src/arrow/dataset/file_csv_test.cc b/cpp/src/arrow/dataset/file_csv_test.cc index e8e5838e6f9..9ccc1140adb 100644 --- a/cpp/src/arrow/dataset/file_csv_test.cc +++ b/cpp/src/arrow/dataset/file_csv_test.cc @@ -449,6 +449,22 @@ TEST_P(TestCsvFileFormat, WriteRecordBatchReaderCustomOptions) { ASSERT_EQ("0\n0\n0\n0\n0\n", written->ToString()); } +TEST_P(TestCsvFileFormat, WriteRecordBatchReaderDelimiterPropagation) { + // Verify that CsvFileFormat::DefaultWriteOptions() propagates the parse + // delimiter to write options + auto csv_format = std::make_shared(); + csv_format->parse_options.delimiter = '|'; + auto write_options = + checked_pointer_cast(csv_format->DefaultWriteOptions()); + ASSERT_EQ(write_options->write_options->delimiter, '|'); + + // Verify that the default delimiter is ',' when no parse options are set + auto default_format = std::make_shared(); + auto default_write_options = + checked_pointer_cast(default_format->DefaultWriteOptions()); + ASSERT_EQ(default_write_options->write_options->delimiter, ','); +} + TEST_P(TestCsvFileFormat, CountRows) { TestCountRows(); } TEST_P(TestCsvFileFormat, FragmentEquals) { TestFragmentEquals(); } diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 66d2cba27af..d40614a61fc 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -2255,7 +2255,17 @@ cdef class CsvFileFormat(FileFormat): """ cdef CsvFileWriteOptions opts = \ FileFormat.make_write_options(self) - opts.write_options = WriteOptions(**kwargs) + # Start from the C++ defaults, which carry over fields from the + # format's parse_options (e.g. the delimiter), and apply caller + # overrides on top instead of replacing the WriteOptions object + # and discarding those defaults. None is treated as "unspecified" + # to match the previous WriteOptions(**kwargs) semantics. + write_options = opts.write_options + for key, value in kwargs.items(): + if value is None: + continue + setattr(write_options, key, value) + opts.write_options = write_options return opts @property diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index 63c537eae5f..b12a1bce601 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -4987,6 +4987,47 @@ def test_write_dataset_csv(tempdir): assert result.equals(table) +def test_csv_make_write_options_uses_parse_delimiter(): + # CsvFileFormat.make_write_options propagates the parse delimiter + # to write options when no explicit delimiter is given + csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter="|")) + write_opts = csv_format.make_write_options() + assert write_opts.write_options.delimiter == "|" + + # delimiter=None is treated as "unspecified" and the propagated + # value is preserved (matches WriteOptions(**kwargs) semantics) + write_opts = csv_format.make_write_options(delimiter=None) + assert write_opts.write_options.delimiter == "|" + + # An explicitly passed delimiter takes precedence + csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">")) + write_opts = csv_format.make_write_options(delimiter="|") + assert write_opts.write_options.delimiter == "|" + + # The default delimiter is still "," when no parse options are given + csv_format = ds.CsvFileFormat() + write_opts = csv_format.make_write_options() + assert write_opts.write_options.delimiter == "," + + +def test_write_dataset_uses_csv_parse_delimiter(tempdir): + table = pa.table({ + "B": ["B1", "B2"], + "C": ["C1", "C2"], + }) + + csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">")) + ds.write_dataset(table, tempdir, format=csv_format) + + with open(tempdir / "part-0.csv") as fh: + content = fh.read() + assert content == '"B">"C"\n"B1">"C1"\n"B2">"C2"\n' + + # Roundtrip: reading back with the same delimiter recovers the table + result = ds.dataset(tempdir, format=csv_format).to_table() + assert result.equals(table) + + @pytest.mark.parquet def test_write_dataset_parquet_file_visitor(tempdir): table = pa.table([