diff --git a/ruby/red-arrow/ext/arrow/converters.cpp b/ruby/red-arrow/ext/arrow/converters.cpp index bbabe60b7dd9..657665cb29af 100644 --- a/ruby/red-arrow/ext/arrow/converters.cpp +++ b/ruby/red-arrow/ext/arrow/converters.cpp @@ -30,6 +30,11 @@ namespace red_arrow { return large_list_array_value_converter_->convert(array, i); } + VALUE ArrayValueConverter::convert(const arrow::FixedSizeListArray& array, + const int64_t i) { + return fixed_size_list_array_value_converter_->convert(array, i); + } + VALUE ArrayValueConverter::convert(const arrow::StructArray& array, const int64_t i) { return struct_array_value_converter_->convert(array, i); diff --git a/ruby/red-arrow/ext/arrow/converters.hpp b/ruby/red-arrow/ext/arrow/converters.hpp index 099aa916863b..37559fb3104a 100644 --- a/ruby/red-arrow/ext/arrow/converters.hpp +++ b/ruby/red-arrow/ext/arrow/converters.hpp @@ -29,6 +29,7 @@ namespace red_arrow { class ListArrayValueConverter; class LargeListArrayValueConverter; + class FixedSizeListArrayValueConverter; class StructArrayValueConverter; class MapArrayValueConverter; class UnionArrayValueConverter; @@ -40,6 +41,7 @@ namespace red_arrow { : decimal_buffer_(), list_array_value_converter_(nullptr), large_list_array_value_converter_(nullptr), + fixed_size_list_array_value_converter_(nullptr), struct_array_value_converter_(nullptr), map_array_value_converter_(nullptr), union_array_value_converter_(nullptr), @@ -48,12 +50,14 @@ namespace red_arrow { inline void set_sub_value_converters(ListArrayValueConverter* list_array_value_converter, LargeListArrayValueConverter* large_list_array_value_converter, + FixedSizeListArrayValueConverter* fixed_size_list_array_value_converter, StructArrayValueConverter* struct_array_value_converter, MapArrayValueConverter* map_array_value_converter, UnionArrayValueConverter* union_array_value_converter, DictionaryArrayValueConverter* dictionary_array_value_converter) { list_array_value_converter_ = list_array_value_converter; large_list_array_value_converter_ = large_list_array_value_converter; + fixed_size_list_array_value_converter_ = fixed_size_list_array_value_converter; struct_array_value_converter_ = struct_array_value_converter; map_array_value_converter_ = map_array_value_converter; union_array_value_converter_ = union_array_value_converter; @@ -286,6 +290,9 @@ namespace red_arrow { VALUE convert(const arrow::LargeListArray& array, const int64_t i); + VALUE convert(const arrow::FixedSizeListArray& array, + const int64_t i); + VALUE convert(const arrow::StructArray& array, const int64_t i); @@ -322,6 +329,7 @@ namespace red_arrow { std::string decimal_buffer_; ListArrayValueConverter* list_array_value_converter_; LargeListArrayValueConverter* large_list_array_value_converter_; + FixedSizeListArrayValueConverter* fixed_size_list_array_value_converter_; StructArrayValueConverter* struct_array_value_converter_; MapArrayValueConverter* map_array_value_converter_; UnionArrayValueConverter* union_array_value_converter_; @@ -385,6 +393,7 @@ namespace red_arrow { VISIT(Duration) VISIT(List) VISIT(LargeList) + VISIT(FixedSizeList) VISIT(Struct) VISIT(Map) VISIT(SparseUnion) @@ -485,6 +494,108 @@ namespace red_arrow { VISIT(Duration) VISIT(List) VISIT(LargeList) + VISIT(FixedSizeList) + VISIT(Struct) + VISIT(Map) + VISIT(SparseUnion) + VISIT(DenseUnion) + VISIT(Dictionary) + VISIT(Decimal128) + VISIT(Decimal256) + // TODO + // VISIT(Extension) + +#undef VISIT + + private: + template + inline VALUE convert_value(const ArrayType& array, + const int64_t i) { + return array_value_converter_->convert(array, i); + } + + template + arrow::Status visit_value(const ArrayType& array) { + if (array.null_count() > 0) { + for (int64_t i = 0; i < length_; ++i) { + auto value = Qnil; + if (!array.IsNull(i + offset_)) { + value = convert_value(array, i + offset_); + } + rb_ary_push(result_, value); + } + } else { + for (int64_t i = 0; i < length_; ++i) { + rb_ary_push(result_, convert_value(array, i + offset_)); + } + } + return arrow::Status::OK(); + } + + ArrayValueConverter* array_value_converter_; + int32_t offset_; + int32_t length_; + VALUE result_; + }; + + class FixedSizeListArrayValueConverter : public arrow::ArrayVisitor { + public: + explicit FixedSizeListArrayValueConverter(ArrayValueConverter* converter) + : array_value_converter_(converter), + offset_(0), + length_(0), + result_(Qnil) {} + + VALUE convert(const arrow::FixedSizeListArray& array, const int64_t index) { + auto values = array.values().get(); + auto offset_keep = offset_; + auto length_keep = length_; + offset_ = array.value_offset(index); + length_ = array.value_length(index); + auto result_keep = result_; + result_ = rb_ary_new_capa(length_); + check_status(values->Accept(this), + "[raw-records][fixed-size-list-array]"); + offset_ = offset_keep; + length_ = length_keep; + auto result_return = result_; + result_ = result_keep; + return result_return; + } + +#define VISIT(TYPE) \ + arrow::Status Visit(const arrow::TYPE ## Array& array) override { \ + return visit_value(array); \ + } + + VISIT(Null) + VISIT(Boolean) + VISIT(Int8) + VISIT(Int16) + VISIT(Int32) + VISIT(Int64) + VISIT(UInt8) + VISIT(UInt16) + VISIT(UInt32) + VISIT(UInt64) + VISIT(HalfFloat) + VISIT(Float) + VISIT(Double) + VISIT(Binary) + VISIT(String) + VISIT(FixedSizeBinary) + VISIT(Date32) + VISIT(Date64) + VISIT(Time32) + VISIT(Time64) + VISIT(Timestamp) + VISIT(MonthInterval) + VISIT(DayTimeInterval) + VISIT(MonthDayNanoInterval) + VISIT(Duration) + VISIT(List) + VISIT(LargeList) + VISIT(FixedSizeList) VISIT(Struct) VISIT(Map) VISIT(SparseUnion) @@ -593,6 +704,7 @@ namespace red_arrow { VISIT(Duration) VISIT(List) VISIT(LargeList) + VISIT(FixedSizeList) VISIT(Struct) VISIT(Map) VISIT(SparseUnion) @@ -697,6 +809,7 @@ namespace red_arrow { VISIT(Duration) VISIT(List) VISIT(LargeList) + VISIT(FixedSizeList) VISIT(Struct) VISIT(Map) VISIT(SparseUnion) @@ -802,6 +915,7 @@ namespace red_arrow { VISIT(Duration) VISIT(List) VISIT(LargeList) + VISIT(FixedSizeList) VISIT(Struct) VISIT(Map) VISIT(SparseUnion) @@ -917,6 +1031,7 @@ namespace red_arrow { VISIT(Duration) VISIT(List) VISIT(LargeList) + VISIT(FixedSizeList) VISIT(Struct) VISIT(Map) VISIT(SparseUnion) @@ -947,6 +1062,7 @@ namespace red_arrow { : array_value_converter_(), list_array_value_converter_(&array_value_converter_), large_list_array_value_converter_(&array_value_converter_), + fixed_size_list_array_value_converter_(&array_value_converter_), struct_array_value_converter_(&array_value_converter_), map_array_value_converter_(&array_value_converter_), union_array_value_converter_(&array_value_converter_), @@ -954,6 +1070,7 @@ namespace red_arrow { array_value_converter_. set_sub_value_converters(&list_array_value_converter_, &large_list_array_value_converter_, + &fixed_size_list_array_value_converter_, &struct_array_value_converter_, &map_array_value_converter_, &union_array_value_converter_, @@ -969,6 +1086,7 @@ namespace red_arrow { ArrayValueConverter array_value_converter_; ListArrayValueConverter list_array_value_converter_; LargeListArrayValueConverter large_list_array_value_converter_; + FixedSizeListArrayValueConverter fixed_size_list_array_value_converter_; StructArrayValueConverter struct_array_value_converter_; MapArrayValueConverter map_array_value_converter_; UnionArrayValueConverter union_array_value_converter_; diff --git a/ruby/red-arrow/ext/arrow/raw-records.cpp b/ruby/red-arrow/ext/arrow/raw-records.cpp index 7f643bad4130..9b7b7ad3adef 100644 --- a/ruby/red-arrow/ext/arrow/raw-records.cpp +++ b/ruby/red-arrow/ext/arrow/raw-records.cpp @@ -102,6 +102,8 @@ namespace red_arrow { VISIT(MonthDayNanoInterval) VISIT(Duration) VISIT(List) + VISIT(LargeList) + VISIT(FixedSizeList) VISIT(Struct) VISIT(Map) VISIT(SparseUnion) @@ -241,6 +243,8 @@ namespace red_arrow { VISIT(MonthDayNanoInterval) VISIT(Duration) VISIT(List) + VISIT(LargeList) + VISIT(FixedSizeList) VISIT(Struct) VISIT(Map) VISIT(SparseUnion) diff --git a/ruby/red-arrow/ext/arrow/values.cpp b/ruby/red-arrow/ext/arrow/values.cpp index 0296f27398d8..61baa185d8b9 100644 --- a/ruby/red-arrow/ext/arrow/values.cpp +++ b/ruby/red-arrow/ext/arrow/values.cpp @@ -84,6 +84,7 @@ namespace red_arrow { VISIT(Duration) VISIT(List) VISIT(LargeList) + VISIT(FixedSizeList) VISIT(Struct) VISIT(Map) VISIT(SparseUnion) diff --git a/ruby/red-arrow/lib/arrow/fixed-size-list-data-type.rb b/ruby/red-arrow/lib/arrow/fixed-size-list-data-type.rb new file mode 100644 index 000000000000..36c265b871fc --- /dev/null +++ b/ruby/red-arrow/lib/arrow/fixed-size-list-data-type.rb @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class FixedSizeListDataType + include ListFieldResolvable + + alias_method :initialize_raw, :initialize + private :initialize_raw + + # Creates a new {Arrow::FixedSizeListDataType}. + # + # @overload initialize(field, size) + # + # @param field [Arrow::Field, Hash] The field of the fixed size + # list data type. You can also specify field description by + # `Hash`. + # + # See {Arrow::Field.new} how to specify field description. + # + # @param size [Integer] The number of values in each element. + # + # @example Create a fixed size list data type with {Arrow::Field} + # visible_field = Arrow::Field.new("visible", :boolean) + # size = 2 + # Arrow::FixedSizeListDataType.new(visible_field, size) + # + # @example Create a list data type with field description + # description = {name: "visible", type: :boolean} + # size = 2 + # Arrow::FixedSizeListDataType.new(description, size) + # + # @overload initialize(description) + # + # @param description [Hash] The description of the fixed size + # list data type. It must have `:field` value and `:size` + # value. + # + # @option description [Arrow::Field, Hash] :field The field of + # the list data type. You can also specify field description + # by `Hash`. + # + # See {Arrow::Field.new} how to specify field description. + # + # @option description [Integer] :size The number of values of + # each element of the fixed size list data type. + # + # @example Create a fixed size list data type with {Arrow::Field} + # visible_field = Arrow::Field.new("visible", :boolean) + # Arrow::FixedSizeListDataType.new(field: visible_field, size: 2) + # + # @example Create a fixed size list data type with field description + # Arrow::FixedSizeListDataType.new(field: { + # name: "visible", + # type: :boolean, + # }, + # size: 2) + # + # @overload initialize(data_type, size) + # + # @param data_type [Arrow::DataType, String, Symbol, + # ::Array, ::Array, Hash] The element data + # type of the fixed size list data type. A field is created + # with the default name `"item"` from the data type + # automatically. + # + # See {Arrow::DataType.resolve} how to specify data type. + # + # @param size [Integer] The number of values in each + # element. + # + # @example Create a fixed size list data type with {Arrow::DataType} + # size = 2 + # Arrow::FixedSizeListDataType.new(Arrow::BooleanDataType.new, + # size) + # + # @example Create a fixed size list data type with data type name as String + # size = 2 + # Arrow::FixedSizeListDataType.new("boolean", size) + # + # @example Create a fixed size list data type with data type name as Symbol + # size = 2 + # Arrow::FixedSizeListDataType.new(:boolean, size) + # + # @example Create a fixed size list data type with data type as Array + # size = 2 + # Arrow::FixedSizeListDataType.new([:time32, :milli], size) + def initialize(*args) + n_args = args.size + case n_args + when 1 + description = args[0] + size = description.delete(:size) + initialize_raw(resolve_field(description), size) + when 2 + field, size = args + initialize_raw(resolve_field(field), size) + else + message = "wrong number of arguments (given #{n_args}, expected 1..2)" + raise ArgumentError, message + end + end + end +end diff --git a/ruby/red-arrow/lib/arrow/libraries.rb b/ruby/red-arrow/lib/arrow/libraries.rb index a29a5588bbb5..f588883d663f 100644 --- a/ruby/red-arrow/lib/arrow/libraries.rb +++ b/ruby/red-arrow/lib/arrow/libraries.rb @@ -71,6 +71,7 @@ require_relative "fixed-size-binary-array" require_relative "fixed-size-binary-array-builder" require_relative "fixed-size-list-array-builder" +require_relative "fixed-size-list-data-type" require_relative "function" require_relative "group" require_relative "half-float" diff --git a/ruby/red-arrow/test/raw-records/test-fixed-size-list-array.rb b/ruby/red-arrow/test/raw-records/test-fixed-size-list-array.rb new file mode 100644 index 000000000000..168bdce595d5 --- /dev/null +++ b/ruby/red-arrow/test/raw-records/test-fixed-size-list-array.rb @@ -0,0 +1,761 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module RawRecordsFixedSizeListArrayTests + def build_schema(type) + field_description = { + name: :element, + } + if type.is_a?(Hash) + field_description = field_description.merge(type) + else + field_description[:type] = type + end + { + column: { + type: :fixed_size_list, + field: field_description, + size: 4, + }, + } + end + + def test_null + records = [ + [[nil, nil, nil, nil]], + [nil], + ] + target = build(:null, records) + assert_equal(records, actual_records(target)) + end + + def test_boolean + records = [ + [[true, nil, false, true]], + [nil], + ] + target = build(:boolean, records) + assert_equal(records, actual_records(target)) + end + + def test_int8 + records = [ + [[-(2 ** 7), nil, (2 ** 7) - 1, 0]], + [nil], + ] + target = build(:int8, records) + assert_equal(records, actual_records(target)) + end + + def test_uint8 + records = [ + [[0, nil, (2 ** 8) - 1, 0]], + [nil], + ] + target = build(:uint8, records) + assert_equal(records, actual_records(target)) + end + + def test_int16 + records = [ + [[-(2 ** 15), nil, (2 ** 15) - 1, 0]], + [nil], + ] + target = build(:int16, records) + assert_equal(records, actual_records(target)) + end + + def test_uint16 + records = [ + [[0, nil, (2 ** 16) - 1, 0]], + [nil], + ] + target = build(:uint16, records) + assert_equal(records, actual_records(target)) + end + + def test_int32 + records = [ + [[-(2 ** 31), nil, (2 ** 31) - 1, 0]], + [nil], + ] + target = build(:int32, records) + assert_equal(records, actual_records(target)) + end + + def test_uint32 + records = [ + [[0, nil, (2 ** 32) - 1, 0]], + [nil], + ] + target = build(:uint32, records) + assert_equal(records, actual_records(target)) + end + + def test_int64 + records = [ + [[-(2 ** 63), nil, (2 ** 63) - 1, 0]], + [nil], + ] + target = build(:int64, records) + assert_equal(records, actual_records(target)) + end + + def test_uint64 + records = [ + [[0, nil, (2 ** 64) - 1, 0]], + [nil], + ] + target = build(:uint64, records) + assert_equal(records, actual_records(target)) + end + + def test_float + records = [ + [[-1.0, nil, 1.0, 0.0]], + [nil], + ] + target = build(:float, records) + assert_equal(records, actual_records(target)) + end + + def test_double + records = [ + [[-1.0, nil, 1.0, 0.0]], + [nil], + ] + target = build(:double, records) + assert_equal(records, actual_records(target)) + end + + def test_binary + records = [ + [["\x00".b, nil, "\xff".b, "".b]], + [nil], + ] + target = build(:binary, records) + assert_equal(records, actual_records(target)) + end + + def test_string + records = [ + [ + [ + "Ruby", + nil, + "\u3042", # U+3042 HIRAGANA LETTER A + "", + ], + ], + [nil], + ] + target = build(:string, records) + assert_equal(records, actual_records(target)) + end + + def test_date32 + records = [ + [ + [ + Date.new(1960, 1, 1), + nil, + Date.new(2017, 8, 23), + Date.new(1970, 1, 1), + ], + ], + [nil], + ] + target = build(:date32, records) + assert_equal(records, actual_records(target)) + end + + def test_date64 + records = [ + [ + [ + DateTime.new(1960, 1, 1, 2, 9, 30), + nil, + DateTime.new(2017, 8, 23, 14, 57, 2), + DateTime.new(1970, 1, 1, 0, 0, 0), + ], + ], + [nil], + ] + target = build(:date64, records) + assert_equal(records, actual_records(target)) + end + + def test_timestamp_second + records = [ + [ + [ + Time.parse("1960-01-01T02:09:30Z"), + nil, + Time.parse("2017-08-23T14:57:02Z"), + Time.parse("1970-01-01T00:00:00Z"), + ], + ], + [nil], + ] + target = build({ + type: :timestamp, + unit: :second, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_timestamp_milli + records = [ + [ + [ + Time.parse("1960-01-01T02:09:30.123Z"), + nil, + Time.parse("2017-08-23T14:57:02.987Z"), + Time.parse("1970-01-01T00:00:00.000Z"), + ], + ], + [nil], + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_timestamp_micro + records = [ + [ + [ + Time.parse("1960-01-01T02:09:30.123456Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654Z"), + Time.parse("1970-01-01T00:00:00.000000Z"), + ], + ], + [nil], + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_timestamp_nano + records = [ + [ + [ + Time.parse("1960-01-01T02:09:30.123456789Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654321Z"), + Time.parse("1970-01-01T00:00:00.000000000Z"), + ], + ], + [nil], + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + records = [ + [ + [ + # 00:10:00 + Arrow::Time.new(unit, 60 * 10), + nil, + # 02:00:09 + Arrow::Time.new(unit, 60 * 60 * 2 + 9), + # 00:00:00 + Arrow::Time.new(unit, 0), + ], + ], + [nil], + ] + target = build({ + type: :time32, + unit: :second, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + records = [ + [ + [ + # 00:10:00.123 + Arrow::Time.new(unit, (60 * 10) * 1000 + 123), + nil, + # 02:00:09.987 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987), + # 00:00:00.000 + Arrow::Time.new(unit, 0), + ], + ], + [nil], + ] + target = build({ + type: :time32, + unit: :milli, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + records = [ + [ + [ + # 00:10:00.123456 + Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456), + nil, + # 02:00:09.987654 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654), + # 00:00:00.000000 + Arrow::Time.new(unit, 0), + ], + ], + [nil], + ] + target = build({ + type: :time64, + unit: :micro, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + records = [ + [ + [ + # 00:10:00.123456789 + Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789), + nil, + # 02:00:09.987654321 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321), + # 00:00:00.000000000 + Arrow::Time.new(unit, 0), + ], + ], + [nil], + ] + target = build({ + type: :time64, + unit: :nano, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_decimal128 + records = [ + [ + [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + BigDecimal("00.00"), + ], + ], + [nil], + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_decimal256 + records = [ + [ + [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + BigDecimal("00.00"), + ], + ], + [nil], + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_month_interval + records = [ + [[1, nil, 12, 0]], + [nil], + ] + target = build(:month_interval, records) + assert_equal(records, actual_records(target)) + end + + def test_day_time_interval + records = [ + [ + [ + {day: 1, millisecond: 100}, + nil, + {day: 2, millisecond: 300}, + {day: 0, millisecond: 0}, + ] + ], + [nil], + ] + target = build(:day_time_interval, records) + assert_equal(records, actual_records(target)) + end + + def test_month_day_nano_interval + records = [ + [ + [ + {month: 1, day: 1, nanosecond: 100}, + nil, + {month: 2, day: 3, nanosecond: 400}, + {month: 0, day: 0, nanosecond: 0}, + ] + ], + [nil], + ] + target = build(:month_day_nano_interval, records) + assert_equal(records, actual_records(target)) + end + + def test_list + records = [ + [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + [ + true, + false, + ], + ], + ], + [nil], + ] + target = build({ + type: :list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_large_list + records = [ + [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + [ + true, + false, + ], + ], + ], + [nil], + ] + target = build({ + type: :large_list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_fixed_size_list + records = [ + [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + [ + true, + false, + ], + ], + ], + [nil], + ] + target = build({ + type: :fixed_size_list, + field: { + name: :sub_element, + type: :boolean, + }, + size: 2, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_struct + records = [ + [ + [ + {"field" => true}, + nil, + {"field" => nil}, + {"field" => false}, + ], + ], + [nil], + ] + target = build({ + type: :struct, + fields: [ + { + name: :field, + type: :boolean, + }, + ], + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_map + records = [ + [ + [ + {"key1" => true, "key2" => nil}, + nil, + {"key1" => false}, + {"key3" => nil}, + ], + ], + [nil], + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + records) + assert_equal(records, actual_records(target)) + end + + def remove_union_field_names(records) + records.collect do |record| + record.collect do |column| + if column.nil? + column + else + column.collect do |value| + if value.nil? + value + else + value.values[0] + end + end + end + end + end + end + + def test_sparse_union + records = [ + [ + [ + {"field1" => true}, + nil, + {"field2" => 29}, + {"field2" => nil}, + ], + ], + [nil], + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(remove_union_field_names(records), + actual_records(target)) + end + + def test_dense_union + records = [ + [ + [ + {"field1" => true}, + nil, + {"field2" => 29}, + {"field2" => nil}, + ], + ], + [nil], + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(remove_union_field_names(records), + actual_records(target)) + end + + def test_dictionary + records = [ + [ + [ + "Ruby", + nil, + "GLib", + "Ruby", + ], + ], + [nil], + ] + target = build({ + type: :dictionary, + index_data_type: :int8, + value_data_type: :string, + ordered: false, + }, + records) + assert_equal(records, actual_records(target)) + end +end + +class EachRawRecordRecordBatchFixedSizeListArrayTest < Test::Unit::TestCase + include RawRecordsFixedSizeListArrayTests + + def build(type, records) + Arrow::RecordBatch.new(build_schema(type), records) + end + + def actual_records(target) + target.each_raw_record.to_a + end +end + +class EachRawRecordTableFixedSizeListArrayTest < Test::Unit::TestCase + include RawRecordsFixedSizeListArrayTests + + def build(type, records) + record_batch = Arrow::RecordBatch.new(build_schema(type), records) + # Multiple chunks + record_batches = [ + record_batch.slice(0, 2), + record_batch.slice(2, 0), # Empty chunk + record_batch.slice(2, record_batch.length - 2), + ] + Arrow::Table.new(record_batch.schema, record_batches) + end + + def actual_records(target) + target.each_raw_record.to_a + end +end + + +class RawRecordsRecordBatchFixedSizeListArrayTest < Test::Unit::TestCase + include RawRecordsFixedSizeListArrayTests + + def build(type, records) + Arrow::RecordBatch.new(build_schema(type), records) + end + + def actual_records(target) + target.raw_records + end +end + +class RawRecordsTableFixedSizeListArrayTest < Test::Unit::TestCase + include RawRecordsFixedSizeListArrayTests + + def build(type, records) + Arrow::Table.new(build_schema(type), records) + end + + def actual_records(target) + target.raw_records + end +end diff --git a/ruby/red-arrow/test/raw-records/test-large-list-array.rb b/ruby/red-arrow/test/raw-records/test-large-list-array.rb new file mode 100644 index 000000000000..4557decaa657 --- /dev/null +++ b/ruby/red-arrow/test/raw-records/test-large-list-array.rb @@ -0,0 +1,725 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module RawRecordsLargeListArrayTests + def build_schema(type) + field_description = { + name: :element, + } + if type.is_a?(Hash) + field_description = field_description.merge(type) + else + field_description[:type] = type + end + { + column: { + type: :large_list, + field: field_description, + }, + } + end + + def test_null + records = [ + [[nil, nil, nil]], + [nil], + ] + target = build(:null, records) + assert_equal(records, actual_records(target)) + end + + def test_boolean + records = [ + [[true, nil, false]], + [nil], + ] + target = build(:boolean, records) + assert_equal(records, actual_records(target)) + end + + def test_int8 + records = [ + [[-(2 ** 7), nil, (2 ** 7) - 1]], + [nil], + ] + target = build(:int8, records) + assert_equal(records, actual_records(target)) + end + + def test_uint8 + records = [ + [[0, nil, (2 ** 8) - 1]], + [nil], + ] + target = build(:uint8, records) + assert_equal(records, actual_records(target)) + end + + def test_int16 + records = [ + [[-(2 ** 15), nil, (2 ** 15) - 1]], + [nil], + ] + target = build(:int16, records) + assert_equal(records, actual_records(target)) + end + + def test_uint16 + records = [ + [[0, nil, (2 ** 16) - 1]], + [nil], + ] + target = build(:uint16, records) + assert_equal(records, actual_records(target)) + end + + def test_int32 + records = [ + [[-(2 ** 31), nil, (2 ** 31) - 1]], + [nil], + ] + target = build(:int32, records) + assert_equal(records, actual_records(target)) + end + + def test_uint32 + records = [ + [[0, nil, (2 ** 32) - 1]], + [nil], + ] + target = build(:uint32, records) + assert_equal(records, actual_records(target)) + end + + def test_int64 + records = [ + [[-(2 ** 63), nil, (2 ** 63) - 1]], + [nil], + ] + target = build(:int64, records) + assert_equal(records, actual_records(target)) + end + + def test_uint64 + records = [ + [[0, nil, (2 ** 64) - 1]], + [nil], + ] + target = build(:uint64, records) + assert_equal(records, actual_records(target)) + end + + def test_float + records = [ + [[-1.0, nil, 1.0]], + [nil], + ] + target = build(:float, records) + assert_equal(records, actual_records(target)) + end + + def test_double + records = [ + [[-1.0, nil, 1.0]], + [nil], + ] + target = build(:double, records) + assert_equal(records, actual_records(target)) + end + + def test_binary + records = [ + [["\x00".b, nil, "\xff".b]], + [nil], + ] + target = build(:binary, records) + assert_equal(records, actual_records(target)) + end + + def test_string + records = [ + [ + [ + "Ruby", + nil, + "\u3042", # U+3042 HIRAGANA LETTER A + ], + ], + [nil], + ] + target = build(:string, records) + assert_equal(records, actual_records(target)) + end + + def test_date32 + records = [ + [ + [ + Date.new(1960, 1, 1), + nil, + Date.new(2017, 8, 23), + ], + ], + [nil], + ] + target = build(:date32, records) + assert_equal(records, actual_records(target)) + end + + def test_date64 + records = [ + [ + [ + DateTime.new(1960, 1, 1, 2, 9, 30), + nil, + DateTime.new(2017, 8, 23, 14, 57, 2), + ], + ], + [nil], + ] + target = build(:date64, records) + assert_equal(records, actual_records(target)) + end + + def test_timestamp_second + records = [ + [ + [ + Time.parse("1960-01-01T02:09:30Z"), + nil, + Time.parse("2017-08-23T14:57:02Z"), + ], + ], + [nil], + ] + target = build({ + type: :timestamp, + unit: :second, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_timestamp_milli + records = [ + [ + [ + Time.parse("1960-01-01T02:09:30.123Z"), + nil, + Time.parse("2017-08-23T14:57:02.987Z"), + ], + ], + [nil], + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_timestamp_micro + records = [ + [ + [ + Time.parse("1960-01-01T02:09:30.123456Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654Z"), + ], + ], + [nil], + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_timestamp_nano + records = [ + [ + [ + Time.parse("1960-01-01T02:09:30.123456789Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654321Z"), + ], + ], + [nil], + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + records = [ + [ + [ + # 00:10:00 + Arrow::Time.new(unit, 60 * 10), + nil, + # 02:00:09 + Arrow::Time.new(unit, 60 * 60 * 2 + 9), + ], + ], + [nil], + ] + target = build({ + type: :time32, + unit: :second, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + records = [ + [ + [ + # 00:10:00.123 + Arrow::Time.new(unit, (60 * 10) * 1000 + 123), + nil, + # 02:00:09.987 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987), + ], + ], + [nil], + ] + target = build({ + type: :time32, + unit: :milli, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + records = [ + [ + [ + # 00:10:00.123456 + Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456), + nil, + # 02:00:09.987654 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654), + ], + ], + [nil], + ] + target = build({ + type: :time64, + unit: :micro, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + records = [ + [ + [ + # 00:10:00.123456789 + Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789), + nil, + # 02:00:09.987654321 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321), + ], + ], + [nil], + ] + target = build({ + type: :time64, + unit: :nano, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_decimal128 + records = [ + [ + [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + ], + ], + [nil], + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_decimal256 + records = [ + [ + [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + ], + ], + [nil], + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_month_interval + records = [ + [[1, nil, 12]], + [nil], + ] + target = build(:month_interval, records) + assert_equal(records, actual_records(target)) + end + + def test_day_time_interval + records = [ + [ + [ + {day: 1, millisecond: 100}, + nil, + {day: 2, millisecond: 300}, + ] + ], + [nil], + ] + target = build(:day_time_interval, records) + assert_equal(records, actual_records(target)) + end + + def test_month_day_nano_interval + records = [ + [ + [ + {month: 1, day: 1, nanosecond: 100}, + nil, + {month: 2, day: 3, nanosecond: 400}, + ] + ], + [nil], + ] + target = build(:month_day_nano_interval, records) + assert_equal(records, actual_records(target)) + end + + def test_list + records = [ + [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + ], + ], + [nil], + ] + target = build({ + type: :list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_large_list + records = [ + [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + ], + ], + [nil], + ] + target = build({ + type: :large_list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_fixed_size_list + records = [ + [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + ], + ], + [nil], + ] + target = build({ + type: :fixed_size_list, + field: { + name: :sub_element, + type: :boolean, + }, + size: 2, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_struct + records = [ + [ + [ + {"field" => true}, + nil, + {"field" => nil}, + ], + ], + [nil], + ] + target = build({ + type: :struct, + fields: [ + { + name: :field, + type: :boolean, + }, + ], + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_map + records = [ + [ + [ + {"key1" => true, "key2" => nil}, + nil, + ], + ], + [nil], + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + records) + assert_equal(records, actual_records(target)) + end + + def remove_union_field_names(records) + records.collect do |record| + record.collect do |column| + if column.nil? + column + else + column.collect do |value| + if value.nil? + value + else + value.values[0] + end + end + end + end + end + end + + def test_sparse_union + records = [ + [ + [ + {"field1" => true}, + nil, + {"field2" => 29}, + {"field2" => nil}, + ], + ], + [nil], + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(remove_union_field_names(records), + actual_records(target)) + end + + def test_dense_union + records = [ + [ + [ + {"field1" => true}, + nil, + {"field2" => 29}, + {"field2" => nil}, + ], + ], + [nil], + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + records) + assert_equal(remove_union_field_names(records), + actual_records(target)) + end + + def test_dictionary + records = [ + [ + [ + "Ruby", + nil, + "GLib", + ], + ], + [nil], + ] + target = build({ + type: :dictionary, + index_data_type: :int8, + value_data_type: :string, + ordered: false, + }, + records) + assert_equal(records, actual_records(target)) + end +end + +class EachRawRecordRecordBatchLargeListArrayTest < Test::Unit::TestCase + include RawRecordsLargeListArrayTests + + def build(type, records) + Arrow::RecordBatch.new(build_schema(type), records) + end + + def actual_records(target) + target.each_raw_record.to_a + end +end + +class EachRawRecordTableLargeListArrayTest < Test::Unit::TestCase + include RawRecordsLargeListArrayTests + + def build(type, records) + record_batch = Arrow::RecordBatch.new(build_schema(type), records) + # Multiple chunks + record_batches = [ + record_batch.slice(0, 2), + record_batch.slice(2, 0), # Empty chunk + record_batch.slice(2, record_batch.length - 2), + ] + Arrow::Table.new(record_batch.schema, record_batches) + end + + def actual_records(target) + target.each_raw_record.to_a + end +end + + +class RawRecordsRecordBatchLargeListArrayTest < Test::Unit::TestCase + include RawRecordsLargeListArrayTests + + def build(type, records) + Arrow::RecordBatch.new(build_schema(type), records) + end + + def actual_records(target) + target.raw_records + end +end + +class RawRecordsTableLargeListArrayTest < Test::Unit::TestCase + include RawRecordsLargeListArrayTests + + def build(type, records) + Arrow::Table.new(build_schema(type), records) + end + + def actual_records(target) + target.raw_records + end +end diff --git a/ruby/red-arrow/test/raw-records/test-list-array.rb b/ruby/red-arrow/test/raw-records/test-list-array.rb index d19b707d0f88..857be3be89c8 100644 --- a/ruby/red-arrow/test/raw-records/test-list-array.rb +++ b/ruby/red-arrow/test/raw-records/test-list-array.rb @@ -466,6 +466,63 @@ def test_list assert_equal(records, actual_records(target)) end + def test_large_list + records = [ + [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + ], + ], + [nil], + ] + target = build({ + type: :large_list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + records) + assert_equal(records, actual_records(target)) + end + + def test_fixed_size_list + records = [ + [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + ], + ], + [nil], + ] + target = build({ + type: :fixed_size_list, + field: { + name: :sub_element, + type: :boolean, + }, + size: 2, + }, + records) + assert_equal(records, actual_records(target)) + end + def test_struct records = [ [ diff --git a/ruby/red-arrow/test/values/test-fixed-size-list-array.rb b/ruby/red-arrow/test/values/test-fixed-size-list-array.rb new file mode 100644 index 000000000000..4e844b646ce7 --- /dev/null +++ b/ruby/red-arrow/test/values/test-fixed-size-list-array.rb @@ -0,0 +1,677 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module ValuesFixedSizeListArrayTests + def build_data_type(type) + field_description = { + name: :element, + } + if type.is_a?(Hash) + field_description = field_description.merge(type) + else + field_description[:type] = type + end + Arrow::FixedSizeListDataType.new(field: field_description, + size: 4) + end + + def build_array(type, values) + Arrow::FixedSizeListArray.new(build_data_type(type), values) + end + + def test_null + values = [ + [nil, nil, nil, nil], + nil, + ] + target = build(:null, values) + assert_equal(values, target.values) + end + + def test_boolean + values = [ + [true, nil, false, true], + nil, + ] + target = build(:boolean, values) + assert_equal(values, target.values) + end + + def test_int8 + values = [ + [-(2 ** 7), nil, (2 ** 7) - 1, 0], + nil, + ] + target = build(:int8, values) + assert_equal(values, target.values) + end + + def test_uint8 + values = [ + [0, nil, (2 ** 8) - 1, 0], + nil, + ] + target = build(:uint8, values) + assert_equal(values, target.values) + end + + def test_int16 + values = [ + [-(2 ** 15), nil, (2 ** 15) - 1, 0], + nil, + ] + target = build(:int16, values) + assert_equal(values, target.values) + end + + def test_uint16 + values = [ + [0, nil, (2 ** 16) - 1, 0], + nil, + ] + target = build(:uint16, values) + assert_equal(values, target.values) + end + + def test_int32 + values = [ + [-(2 ** 31), nil, (2 ** 31) - 1, 0], + nil, + ] + target = build(:int32, values) + assert_equal(values, target.values) + end + + def test_uint32 + values = [ + [0, nil, (2 ** 32) - 1, 0], + nil, + ] + target = build(:uint32, values) + assert_equal(values, target.values) + end + + def test_int64 + values = [ + [-(2 ** 63), nil, (2 ** 63) - 1, 0], + nil, + ] + target = build(:int64, values) + assert_equal(values, target.values) + end + + def test_uint64 + values = [ + [0, nil, (2 ** 64) - 1, 0], + nil, + ] + target = build(:uint64, values) + assert_equal(values, target.values) + end + + def test_float + values = [ + [-1.0, nil, 1.0, 0.0], + nil, + ] + target = build(:float, values) + assert_equal(values, target.values) + end + + def test_double + values = [ + [-1.0, nil, 1.0, 0.0], + nil, + ] + target = build(:double, values) + assert_equal(values, target.values) + end + + def test_binary + values = [ + ["\x00".b, nil, "\xff".b, "".b], + nil, + ] + target = build(:binary, values) + assert_equal(values, target.values) + end + + def test_string + values = [ + [ + "Ruby", + nil, + "\u3042", # U+3042 HIRAGANA LETTER A + "" + ], + nil, + ] + target = build(:string, values) + assert_equal(values, target.values) + end + + def test_date32 + values = [ + [ + Date.new(1960, 1, 1), + nil, + Date.new(2017, 8, 23), + Date.new(1970, 1, 1), + ], + nil, + ] + target = build(:date32, values) + assert_equal(values, target.values) + end + + def test_date64 + values = [ + [ + DateTime.new(1960, 1, 1, 2, 9, 30), + nil, + DateTime.new(2017, 8, 23, 14, 57, 2), + DateTime.new(1970, 1, 1, 0, 0, 0), + ], + nil, + ] + target = build(:date64, values) + assert_equal(values, target.values) + end + + def test_timestamp_second + values = [ + [ + Time.parse("1960-01-01T02:09:30Z"), + nil, + Time.parse("2017-08-23T14:57:02Z"), + Time.parse("1970-01-01T00:00:00Z"), + ], + nil, + ] + target = build({ + type: :timestamp, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_milli + values = [ + [ + Time.parse("1960-01-01T02:09:30.123Z"), + nil, + Time.parse("2017-08-23T14:57:02.987Z"), + Time.parse("1970-01-01T00:00:00.000Z"), + ], + nil, + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_micro + values = [ + [ + Time.parse("1960-01-01T02:09:30.123456Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654Z"), + Time.parse("1970-01-01T00:00:00.000000Z"), + ], + nil, + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_nano + values = [ + [ + Time.parse("1960-01-01T02:09:30.123456789Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654321Z"), + Time.parse("1970-01-01T00:00:00.000000000Z"), + ], + nil, + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + values = [ + [ + # 00:10:00 + Arrow::Time.new(unit, 60 * 10), + nil, + # 02:00:09 + Arrow::Time.new(unit, 60 * 60 * 2 + 9), + # 00:00:00 + Arrow::Time.new(unit, 0), + ], + nil, + ] + target = build({ + type: :time32, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + values = [ + [ + # 00:10:00.123 + Arrow::Time.new(unit, (60 * 10) * 1000 + 123), + nil, + # 02:00:09.987 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987), + # 00:00:00.000 + Arrow::Time.new(unit, 0), + ], + nil, + ] + target = build({ + type: :time32, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + values = [ + [ + # 00:10:00.123456 + Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456), + nil, + # 02:00:09.987654 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654), + # 00:00:00.000000 + Arrow::Time.new(unit, 0), + ], + nil, + ] + target = build({ + type: :time64, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + values = [ + [ + # 00:10:00.123456789 + Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789), + nil, + # 02:00:09.987654321 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321), + # 00:00:00.000000000 + Arrow::Time.new(unit, 0), + ], + nil, + ] + target = build({ + type: :time64, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal128 + values = [ + [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + BigDecimal("00.00"), + ], + nil, + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal256 + values = [ + [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + BigDecimal("00.00"), + ], + nil, + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_month_interval + values = [ + [ + 1, + nil, + 12, + 0, + ], + nil, + ] + target = build(:month_interval, values) + assert_equal(values, target.values) + end + + def test_day_time_interval + values = [ + [ + {day: 1, millisecond: 100}, + nil, + {day: 2, millisecond: 300}, + {day: 0, millisecond: 0}, + ], + nil, + ] + target = build(:day_time_interval, values) + assert_equal(values, target.values) + end + + def test_month_day_nano_interval + values = [ + [ + {month: 1, day: 1, nanosecond: 100}, + nil, + {month: 2, day: 3, nanosecond: 400}, + {month: 0, day: 0, nanosecond: 0}, + ], + nil, + ] + target = build(:month_day_nano_interval, values) + assert_equal(values, target.values) + end + + def test_list + values = [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + [ + true, + false, + ], + ], + nil, + ] + target = build({ + type: :list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + values) + assert_equal(values, target.values) + end + + def test_large_list + values = [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + [ + true, + false, + ], + ], + nil, + ] + target = build({ + type: :large_list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + values) + assert_equal(values, target.values) + end + + def test_fixed_size_list + values = [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + [ + true, + false, + ], + ], + nil, + ] + target = build({ + type: :fixed_size_list, + field: { + name: :sub_element, + type: :boolean, + }, + size: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_struct + values = [ + [ + {"field" => true}, + nil, + {"field" => nil}, + {"field" => false}, + ], + nil, + ] + target = build({ + type: :struct, + fields: [ + { + name: :field, + type: :boolean, + }, + ], + }, + values) + assert_equal(values, target.values) + end + + def test_map + values = [ + [ + {"key1" => true, "key2" => nil}, + nil, + {"key1" => false}, + {"key3" => nil}, + ], + nil, + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + values) + assert_equal(values, target.values) + end + + def remove_union_field_names(values) + values.collect do |value| + if value.nil? + value + else + value.collect do |v| + if v.nil? + v + else + v.values[0] + end + end + end + end + end + + def test_sparse_union + values = [ + [ + {"field1" => true}, + nil, + {"field2" => 29}, + {"field2" => nil}, + ], + nil, + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(remove_union_field_names(values), + target.values) + end + + def test_dense_union + values = [ + [ + {"field1" => true}, + nil, + {"field2" => 29}, + {"field2" => nil}, + ], + nil, + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(remove_union_field_names(values), + target.values) + end + + def test_dictionary + values = [ + [ + "Ruby", + nil, + "GLib", + "Ruby", + ], + nil, + ] + target = build({ + type: :dictionary, + index_data_type: :int8, + value_data_type: :string, + ordered: false, + }, + values) + assert_equal(values, target.values) + end +end + +class ValuesArrayFixedSizeListArrayTest < Test::Unit::TestCase + include ValuesFixedSizeListArrayTests + + def build(type, values) + build_array(type, values) + end +end + +class ValuesChunkedArrayFixedSizeListArrayTest < Test::Unit::TestCase + include ValuesFixedSizeListArrayTests + + def build(type, values) + Arrow::ChunkedArray.new([build_array(type, values)]) + end +end diff --git a/ruby/red-arrow/test/values/test-large-list-array.rb b/ruby/red-arrow/test/values/test-large-list-array.rb new file mode 100644 index 000000000000..8971e1fadbce --- /dev/null +++ b/ruby/red-arrow/test/values/test-large-list-array.rb @@ -0,0 +1,640 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module ValuesLargeListArrayTests + def build_data_type(type) + field_description = { + name: :element, + } + if type.is_a?(Hash) + field_description = field_description.merge(type) + else + field_description[:type] = type + end + Arrow::LargeListDataType.new(field: field_description) + end + + def build_array(type, values) + Arrow::LargeListArray.new(build_data_type(type), values) + end + + def test_null + values = [ + [nil, nil, nil], + nil, + ] + target = build(:null, values) + assert_equal(values, target.values) + end + + def test_boolean + values = [ + [true, nil, false], + nil, + ] + target = build(:boolean, values) + assert_equal(values, target.values) + end + + def test_int8 + values = [ + [-(2 ** 7), nil, (2 ** 7) - 1], + nil, + ] + target = build(:int8, values) + assert_equal(values, target.values) + end + + def test_uint8 + values = [ + [0, nil, (2 ** 8) - 1], + nil, + ] + target = build(:uint8, values) + assert_equal(values, target.values) + end + + def test_int16 + values = [ + [-(2 ** 15), nil, (2 ** 15) - 1], + nil, + ] + target = build(:int16, values) + assert_equal(values, target.values) + end + + def test_uint16 + values = [ + [0, nil, (2 ** 16) - 1], + nil, + ] + target = build(:uint16, values) + assert_equal(values, target.values) + end + + def test_int32 + values = [ + [-(2 ** 31), nil, (2 ** 31) - 1], + nil, + ] + target = build(:int32, values) + assert_equal(values, target.values) + end + + def test_uint32 + values = [ + [0, nil, (2 ** 32) - 1], + nil, + ] + target = build(:uint32, values) + assert_equal(values, target.values) + end + + def test_int64 + values = [ + [-(2 ** 63), nil, (2 ** 63) - 1], + nil, + ] + target = build(:int64, values) + assert_equal(values, target.values) + end + + def test_uint64 + values = [ + [0, nil, (2 ** 64) - 1], + nil, + ] + target = build(:uint64, values) + assert_equal(values, target.values) + end + + def test_float + values = [ + [-1.0, nil, 1.0], + nil, + ] + target = build(:float, values) + assert_equal(values, target.values) + end + + def test_double + values = [ + [-1.0, nil, 1.0], + nil, + ] + target = build(:double, values) + assert_equal(values, target.values) + end + + def test_binary + values = [ + ["\x00".b, nil, "\xff".b], + nil, + ] + target = build(:binary, values) + assert_equal(values, target.values) + end + + def test_string + values = [ + [ + "Ruby", + nil, + "\u3042", # U+3042 HIRAGANA LETTER A + ], + nil, + ] + target = build(:string, values) + assert_equal(values, target.values) + end + + def test_date32 + values = [ + [ + Date.new(1960, 1, 1), + nil, + Date.new(2017, 8, 23), + ], + nil, + ] + target = build(:date32, values) + assert_equal(values, target.values) + end + + def test_date64 + values = [ + [ + DateTime.new(1960, 1, 1, 2, 9, 30), + nil, + DateTime.new(2017, 8, 23, 14, 57, 2), + ], + nil, + ] + target = build(:date64, values) + assert_equal(values, target.values) + end + + def test_timestamp_second + values = [ + [ + Time.parse("1960-01-01T02:09:30Z"), + nil, + Time.parse("2017-08-23T14:57:02Z"), + ], + nil, + ] + target = build({ + type: :timestamp, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_milli + values = [ + [ + Time.parse("1960-01-01T02:09:30.123Z"), + nil, + Time.parse("2017-08-23T14:57:02.987Z"), + ], + nil, + ] + target = build({ + type: :timestamp, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_micro + values = [ + [ + Time.parse("1960-01-01T02:09:30.123456Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654Z"), + ], + nil, + ] + target = build({ + type: :timestamp, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_timestamp_nano + values = [ + [ + Time.parse("1960-01-01T02:09:30.123456789Z"), + nil, + Time.parse("2017-08-23T14:57:02.987654321Z"), + ], + nil, + ] + target = build({ + type: :timestamp, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_second + unit = Arrow::TimeUnit::SECOND + values = [ + [ + # 00:10:00 + Arrow::Time.new(unit, 60 * 10), + nil, + # 02:00:09 + Arrow::Time.new(unit, 60 * 60 * 2 + 9), + ], + nil, + ] + target = build({ + type: :time32, + unit: :second, + }, + values) + assert_equal(values, target.values) + end + + def test_time32_milli + unit = Arrow::TimeUnit::MILLI + values = [ + [ + # 00:10:00.123 + Arrow::Time.new(unit, (60 * 10) * 1000 + 123), + nil, + # 02:00:09.987 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987), + ], + nil, + ] + target = build({ + type: :time32, + unit: :milli, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_micro + unit = Arrow::TimeUnit::MICRO + values = [ + [ + # 00:10:00.123456 + Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456), + nil, + # 02:00:09.987654 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654), + ], + nil, + ] + target = build({ + type: :time64, + unit: :micro, + }, + values) + assert_equal(values, target.values) + end + + def test_time64_nano + unit = Arrow::TimeUnit::NANO + values = [ + [ + # 00:10:00.123456789 + Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789), + nil, + # 02:00:09.987654321 + Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321), + ], + nil, + ] + target = build({ + type: :time64, + unit: :nano, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal128 + values = [ + [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + ], + nil, + ] + target = build({ + type: :decimal128, + precision: 8, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_decimal256 + values = [ + [ + BigDecimal("92.92"), + nil, + BigDecimal("29.29"), + ], + nil, + ] + target = build({ + type: :decimal256, + precision: 38, + scale: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_month_interval + values = [ + [ + 1, + nil, + 12, + ], + nil, + ] + target = build(:month_interval, values) + assert_equal(values, target.values) + end + + def test_day_time_interval + values = [ + [ + {day: 1, millisecond: 100}, + nil, + {day: 2, millisecond: 300}, + ], + nil, + ] + target = build(:day_time_interval, values) + assert_equal(values, target.values) + end + + def test_month_day_nano_interval + values = [ + [ + {month: 1, day: 1, nanosecond: 100}, + nil, + {month: 2, day: 3, nanosecond: 400}, + ], + nil, + ] + target = build(:month_day_nano_interval, values) + assert_equal(values, target.values) + end + + def test_list + values = [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + ], + nil, + ] + target = build({ + type: :list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + values) + assert_equal(values, target.values) + end + + def test_large_list + values = [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + ], + nil, + ] + target = build({ + type: :large_list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + values) + assert_equal(values, target.values) + end + + def test_fixed_size_list + values = [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + ], + nil, + ] + target = build({ + type: :fixed_size_list, + field: { + name: :sub_element, + type: :boolean, + }, + size: 2, + }, + values) + assert_equal(values, target.values) + end + + def test_struct + values = [ + [ + {"field" => true}, + nil, + {"field" => nil}, + ], + nil, + ] + target = build({ + type: :struct, + fields: [ + { + name: :field, + type: :boolean, + }, + ], + }, + values) + assert_equal(values, target.values) + end + + def test_map + values = [ + [ + {"key1" => true, "key2" => nil}, + nil, + ], + nil, + ] + target = build({ + type: :map, + key: :string, + item: :boolean, + }, + values) + assert_equal(values, target.values) + end + + def remove_union_field_names(values) + values.collect do |value| + if value.nil? + value + else + value.collect do |v| + if v.nil? + v + else + v.values[0] + end + end + end + end + end + + def test_sparse_union + values = [ + [ + {"field1" => true}, + nil, + {"field2" => 29}, + {"field2" => nil}, + ], + nil, + ] + target = build({ + type: :sparse_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(remove_union_field_names(values), + target.values) + end + + def test_dense_union + values = [ + [ + {"field1" => true}, + nil, + {"field2" => 29}, + {"field2" => nil}, + ], + nil, + ] + target = build({ + type: :dense_union, + fields: [ + { + name: :field1, + type: :boolean, + }, + { + name: :field2, + type: :uint8, + }, + ], + type_codes: [0, 1], + }, + values) + assert_equal(remove_union_field_names(values), + target.values) + end + + def test_dictionary + values = [ + [ + "Ruby", + nil, + "GLib", + ], + nil, + ] + target = build({ + type: :dictionary, + index_data_type: :int8, + value_data_type: :string, + ordered: false, + }, + values) + assert_equal(values, target.values) + end +end + +class ValuesArrayLargeListArrayTest < Test::Unit::TestCase + include ValuesLargeListArrayTests + + def build(type, values) + build_array(type, values) + end +end + +class ValuesChunkedArrayLargeListArrayTest < Test::Unit::TestCase + include ValuesLargeListArrayTests + + def build(type, values) + Arrow::ChunkedArray.new([build_array(type, values)]) + end +end diff --git a/ruby/red-arrow/test/values/test-list-array.rb b/ruby/red-arrow/test/values/test-list-array.rb index 8c1bae9f9fe7..c9458b3666df 100644 --- a/ruby/red-arrow/test/values/test-list-array.rb +++ b/ruby/red-arrow/test/values/test-list-array.rb @@ -437,6 +437,59 @@ def test_list assert_equal(values, target.values) end + def test_large_list + values = [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + ], + nil, + ] + target = build({ + type: :large_list, + field: { + name: :sub_element, + type: :boolean, + }, + }, + values) + assert_equal(values, target.values) + end + + def test_fixed_size_list + values = [ + [ + [ + true, + nil, + ], + nil, + [ + nil, + false, + ], + ], + nil, + ] + target = build({ + type: :fixed_size_list, + field: { + name: :sub_element, + type: :boolean, + }, + size: 2, + }, + values) + assert_equal(values, target.values) + end + def test_struct values = [ [