Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 36 additions & 14 deletions cpp/src/arrow/array/data.cc
Original file line number Diff line number Diff line change
Expand Up @@ -781,23 +781,28 @@ struct ViewDataImpl {
if (input_exhausted) {
return;
}
while (true) {
// Skip exhausted layout (might be empty layout)
while (in_buffer_idx >= in_layouts[in_layout_idx].buffers.size()) {
in_buffer_idx = 0;
++in_layout_idx;
if (in_layout_idx >= in_layouts.size()) {
input_exhausted = true;
return;
}
// Skip exhausted layout (might be empty layout)
while (in_buffer_idx >= in_layouts[in_layout_idx].buffers.size()) {
in_buffer_idx = 0;
++in_layout_idx;
if (in_layout_idx >= in_layouts.size()) {
input_exhausted = true;
return;
}
}
}

void SkipInputAlwaysNullBuffers() {
AdjustInputPointer();
while (!input_exhausted) {
const auto& in_spec = in_layouts[in_layout_idx].buffers[in_buffer_idx];
if (in_spec.kind != DataTypeLayout::ALWAYS_NULL) {
return;
}
// Skip always-null input buffers
// (e.g. buffer 0 of a null type or buffer 2 of a sparse union)
// (e.g. buffer 0 of a null, union, or run-end encoded type)
++in_buffer_idx;
AdjustInputPointer();
}
}

Expand All @@ -809,6 +814,7 @@ struct ViewDataImpl {
}

Status CheckInputExhausted() {
SkipInputAlwaysNullBuffers();
if (!input_exhausted) {
return InvalidView("too many buffers for view type");
}
Expand All @@ -829,19 +835,33 @@ struct ViewDataImpl {
const auto& out_type = out_field->type();
const auto out_layout = out_type->layout();

AdjustInputPointer();
// No type has a purely empty layout
DCHECK_GT(out_layout.buffers.size(), 0);

int64_t out_length = in_data_length;
int64_t out_offset = 0;
int64_t out_null_count;

AdjustInputPointer();
// Preserve metadata only when matching an input NullType with an output NullType.
if (out_type->id() == Type::NA && !input_exhausted &&
in_data[in_layout_idx]->type->id() == Type::NA) {
DCHECK_EQ(in_layouts[in_layout_idx].buffers[in_buffer_idx].kind,
DataTypeLayout::ALWAYS_NULL);
const auto& in_data_item = in_data[in_layout_idx];
out_length = in_data_item->length;
out_offset = in_data_item->offset;
++in_buffer_idx;
AdjustInputPointer();
} else {
SkipInputAlwaysNullBuffers();
}

std::shared_ptr<ArrayData> dictionary;
if (out_type->id() == Type::DICTIONARY) {
ARROW_ASSIGN_OR_RAISE(dictionary, GetDictionaryView(*out_type));
}

// No type has a purely empty layout
DCHECK_GT(out_layout.buffers.size(), 0);

std::vector<std::shared_ptr<Buffer>> out_buffers;

// Process null bitmap
Expand Down Expand Up @@ -880,13 +900,15 @@ struct ViewDataImpl {
}

// If input buffer is null bitmap, try to ignore it
SkipInputAlwaysNullBuffers();
while (in_buffer_idx == 0) {
RETURN_NOT_OK(CheckInputAvailable());
if (in_data[in_layout_idx]->GetNullCount() != 0) {
return InvalidView("cannot represent nested nulls");
}
++in_buffer_idx;
AdjustInputPointer();
SkipInputAlwaysNullBuffers();
}

RETURN_NOT_OK(CheckInputAvailable());
Expand Down
20 changes: 20 additions & 0 deletions cpp/src/arrow/compute/kernels/scalar_cast_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,26 @@ TEST(Cast, SameTypeZeroCopy) {
AssertBufferSame(*arr, *result, 1);
}

TEST(Cast, SameNestedTypeWithNullField) {
auto type = list(struct_({field("b", null()), field("g", utf8())}));
auto array =
ArrayFromJSON(type, R"([[{"b": null, "g": null}, {"b": null, "g": "moo"}]])");
auto null_field = std::make_shared<NullArray>(4)->Slice(2);
array->data()->child_data[0]->child_data[0] = null_field->data();
ASSERT_OK(array->ValidateFull());
auto input = std::make_shared<ChunkedArray>(ArrayVector{array});

ASSERT_OK_AND_ASSIGN(Datum result, Cast(input, type));
ASSERT_EQ(result.kind(), Datum::CHUNKED_ARRAY);
ASSERT_OK(result.chunked_array()->ValidateFull());
AssertChunkedEqual(*input, *result.chunked_array());

const auto& result_null_field =
result.chunked_array()->chunk(0)->data()->child_data[0]->child_data[0];
ASSERT_EQ(result_null_field->length, 2);
ASSERT_EQ(result_null_field->offset, 2);
}

TEST(Cast, ZeroChunks) {
auto chunked_i32 = std::make_shared<ChunkedArray>(ArrayVector{}, int32());
ASSERT_OK_AND_ASSIGN(Datum result, Cast(chunked_i32, utf8()));
Expand Down