Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cpp/src/arrow/dataset/file_csv.cc
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,7 @@ std::shared_ptr<FileWriteOptions> CsvFileFormat::DefaultWriteOptions() {
new CsvFileWriteOptions(shared_from_this()));
csv_options->write_options =
std::make_shared<csv::WriteOptions>(csv::WriteOptions::Defaults());
csv_options->write_options->delimiter = parse_options.delimiter;
return csv_options;
}

Expand Down
16 changes: 16 additions & 0 deletions cpp/src/arrow/dataset/file_csv_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,22 @@ TEST_P(TestCsvFileFormat, WriteRecordBatchReaderCustomOptions) {
ASSERT_EQ("0\n0\n0\n0\n0\n", written->ToString());
}

TEST_P(TestCsvFileFormat, WriteRecordBatchReaderDelimiterPropagation) {
// Verify that CsvFileFormat::DefaultWriteOptions() propagates the parse
// delimiter to write options
auto csv_format = std::make_shared<CsvFileFormat>();
csv_format->parse_options.delimiter = '|';
auto write_options =
checked_pointer_cast<CsvFileWriteOptions>(csv_format->DefaultWriteOptions());
ASSERT_EQ(write_options->write_options->delimiter, '|');

// Verify that the default delimiter is ',' when no parse options are set
auto default_format = std::make_shared<CsvFileFormat>();
auto default_write_options =
checked_pointer_cast<CsvFileWriteOptions>(default_format->DefaultWriteOptions());
ASSERT_EQ(default_write_options->write_options->delimiter, ',');
}

TEST_P(TestCsvFileFormat, CountRows) { TestCountRows(); }

TEST_P(TestCsvFileFormat, FragmentEquals) { TestFragmentEquals(); }
Expand Down
12 changes: 11 additions & 1 deletion python/pyarrow/_dataset.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -2255,7 +2255,17 @@ cdef class CsvFileFormat(FileFormat):
"""
cdef CsvFileWriteOptions opts = \
<CsvFileWriteOptions> FileFormat.make_write_options(self)
opts.write_options = WriteOptions(**kwargs)
# Start from the C++ defaults, which carry over fields from the
# format's parse_options (e.g. the delimiter), and apply caller
# overrides on top instead of replacing the WriteOptions object
# and discarding those defaults. None is treated as "unspecified"
# to match the previous WriteOptions(**kwargs) semantics.
write_options = opts.write_options
for key, value in kwargs.items():
if value is None:
continue
setattr(write_options, key, value)
Comment thread
egolearner marked this conversation as resolved.
opts.write_options = write_options
return opts

@property
Expand Down
41 changes: 41 additions & 0 deletions python/pyarrow/tests/test_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -4987,6 +4987,47 @@ def test_write_dataset_csv(tempdir):
assert result.equals(table)


def test_csv_make_write_options_uses_parse_delimiter():
# CsvFileFormat.make_write_options propagates the parse delimiter
# to write options when no explicit delimiter is given
csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter="|"))
write_opts = csv_format.make_write_options()
assert write_opts.write_options.delimiter == "|"

# delimiter=None is treated as "unspecified" and the propagated
# value is preserved (matches WriteOptions(**kwargs) semantics)
write_opts = csv_format.make_write_options(delimiter=None)
assert write_opts.write_options.delimiter == "|"

# An explicitly passed delimiter takes precedence
csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">"))
write_opts = csv_format.make_write_options(delimiter="|")
assert write_opts.write_options.delimiter == "|"

# The default delimiter is still "," when no parse options are given
csv_format = ds.CsvFileFormat()
write_opts = csv_format.make_write_options()
assert write_opts.write_options.delimiter == ","


def test_write_dataset_uses_csv_parse_delimiter(tempdir):
table = pa.table({
"B": ["B1", "B2"],
"C": ["C1", "C2"],
})

csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">"))
ds.write_dataset(table, tempdir, format=csv_format)

with open(tempdir / "part-0.csv") as fh:
content = fh.read()
assert content == '"B">"C"\n"B1">"C1"\n"B2">"C2"\n'

# Roundtrip: reading back with the same delimiter recovers the table
result = ds.dataset(tempdir, format=csv_format).to_table()
assert result.equals(table)


@pytest.mark.parquet
def test_write_dataset_parquet_file_visitor(tempdir):
table = pa.table([
Expand Down
Loading