diff --git a/ruby/red-arrow-format/lib/arrow-format/array.rb b/ruby/red-arrow-format/lib/arrow-format/array.rb index 5addc730e5f..37ed8135901 100644 --- a/ruby/red-arrow-format/lib/arrow-format/array.rb +++ b/ruby/red-arrow-format/lib/arrow-format/array.rb @@ -21,6 +21,8 @@ require_relative "bitmap-builder" module ArrowFormat + using FlatBuffers::AppendAsBytes if FlatBuffers.const_defined?(:AppendAsBytes) + class Array attr_reader :type attr_reader :size @@ -160,7 +162,25 @@ def to_a end class PrimitiveArray < Array - def initialize(type, size, validity_buffer, values_buffer) + include BufferAlignable + + def initialize(*args) + n_args = args.size + if self.class.respond_to?(:type) + type = self.class.type + expected_n_args = "1 or 3" + else + type = args.shift + expected_n_args = "2 or 4" + end + args = build_data(args[0], type) if args.size == 1 + if args.size != 3 + message = + "wrong number of arguments " + + "(given #{n_args}, expected #{expected_n_args})" + raise ArgumentError, message + end + size, validity_buffer, values_buffer = args super(type, size, validity_buffer) @values_buffer = values_buffer end @@ -185,20 +205,34 @@ def each_buffer def element_size IO::Buffer.size_of(@type.buffer_type) end + + def build_data(data, type) + n = 0 + validity_buffer_builder = nil + buffer = +"".b + pack_template = type.pack_template + data.each_with_index do |value, i| + if value.nil? + validity_buffer_builder ||= SparseBitmapBuilder.new + validity_buffer_builder.unset(i) + buffer.append_as_bytes([0].pack(pack_template)) + else + buffer.append_as_bytes([value].pack(pack_template)) + end + n += 1 + end + validity_buffer = validity_buffer_builder&.finish(n) + pad!(buffer, buffer_padding_size(buffer)) + buffer.freeze + return n, validity_buffer, IO::Buffer.for(buffer) + end end class BooleanArray < PrimitiveArray - def initialize(*args) - if args.size == 1 - args = build_data(args[0]) - end - n_args = args.size - if args.size != 3 - message = "wrong number of arguments (given #{n_args}, expected 1 or 3)" - raise ArgumentError, message + class << self + def type + BooleanType.singleton end - size, validity_buffer, values_buffer = args - super(BooleanType.singleton, size, validity_buffer, values_buffer) end def to_a @@ -222,7 +256,7 @@ def clear_cache @values_bitmap = nil end - def build_data(data) + def build_data(data, type) n = 0 validity_buffer_builder = nil values_buffer_builder = DenseBitmapBuilder.new @@ -244,9 +278,6 @@ def build_data(data) end class IntArray < PrimitiveArray - def initialize(size, validity_buffer, values_buffer) - super(self.class.type, size, validity_buffer, values_buffer) - end end class Int8Array < IntArray @@ -314,9 +345,6 @@ def type end class FloatingPointArray < PrimitiveArray - def initialize(size, validity_buffer, values_buffer) - super(self.class.type, size, validity_buffer, values_buffer) - end end class Float32Array < FloatingPointArray @@ -339,9 +367,6 @@ class TemporalArray < PrimitiveArray end class DateArray < TemporalArray - def initialize(size, validity_buffer, values_buffer) - super(self.class.type, size, validity_buffer, values_buffer) - end end class Date32Array < DateArray diff --git a/ruby/red-arrow-format/lib/arrow-format/bitmap-builder.rb b/ruby/red-arrow-format/lib/arrow-format/bitmap-builder.rb index 71e82e15d93..d550d058ac6 100644 --- a/ruby/red-arrow-format/lib/arrow-format/bitmap-builder.rb +++ b/ruby/red-arrow-format/lib/arrow-format/bitmap-builder.rb @@ -18,6 +18,8 @@ require_relative "buffer-alignable" module ArrowFormat + using FlatBuffers::AppendAsBytes if FlatBuffers.const_defined?(:AppendAsBytes) + class DenseBitmapBuilder include BufferAlignable @@ -36,8 +38,7 @@ def append(value) def finish flush if @n_bits > 0 - padding_size = buffer_padding_size(@buffer) - @buffer.append_as_bytes(padding(padding_size)) if padding_size > 0 + pad!(@buffer, buffer_padding_size(@buffer)) @buffer.freeze IO::Buffer.for(@buffer) end diff --git a/ruby/red-arrow-format/lib/arrow-format/integration/json-reader.rb b/ruby/red-arrow-format/lib/arrow-format/integration/json-reader.rb index b8d84a5a8e8..ddc8971c7e2 100644 --- a/ruby/red-arrow-format/lib/arrow-format/integration/json-reader.rb +++ b/ruby/red-arrow-format/lib/arrow-format/integration/json-reader.rb @@ -207,7 +207,7 @@ def read_schema def read_bitmap(bitmap) builder = DenseBitmapBuilder.new bitmap.each do |bit| - builder.append(bit == 1 ? true : false) + builder.append(bit == 1) end builder.finish end @@ -248,8 +248,6 @@ def read_hex_value(value) def read_values(data, type) case type - when BooleanType - read_bitmap(data.collect {|boolean| boolean ? 1 : 0}) when DayTimeIntervalType buffer_types = [type.buffer_type] * 2 size = IO::Buffer.size_of(buffer_types) @@ -314,14 +312,37 @@ def read_values(data, type) end end + def apply_validity(values, validity) + values.zip(validity).collect do |value, valid| + if valid == 1 + value + else + nil + end + end + end + def read_array(column, type) length = column["count"] case type when NullType type.build_array(length) - when BooleanType, - NumberType, - TemporalType, + when BooleanType + values = apply_validity(column["DATA"], column["VALIDITY"]) + type.build_array(values) + when NumberType + values = column["DATA"].collect do |value| + if value.is_a?(String) + # If the type is 64bit such as `Int64Type`, `value` is a + # string not integer to round-trip data through JSON. + Integer(value, 10) + else + value + end + end + values = apply_validity(values, column["VALIDITY"]) + type.build_array(values) + when TemporalType, FixedSizeBinaryType validity_buffer = read_bitmap(column["VALIDITY"]) values_buffer = read_values(column["DATA"], type) diff --git a/ruby/red-arrow-format/lib/arrow-format/type.rb b/ruby/red-arrow-format/lib/arrow-format/type.rb index 38523cf00bf..affd9db0243 100644 --- a/ruby/red-arrow-format/lib/arrow-format/type.rb +++ b/ruby/red-arrow-format/lib/arrow-format/type.rb @@ -32,8 +32,8 @@ def name "Null" end - def build_array(size) - NullArray.new(size) + def build_array(...) + NullArray.new(...) end def to_flatbuffers @@ -55,8 +55,8 @@ def name "Boolean" end - def build_array(size, validity_buffer, values_buffer) - BooleanArray.new(size, validity_buffer, values_buffer) + def build_array(...) + BooleanArray.new(...) end def to_flatbuffers @@ -106,8 +106,12 @@ def buffer_type :S8 end - def build_array(size, validity_buffer, values_buffer) - Int8Array.new(size, validity_buffer, values_buffer) + def pack_template + "c" + end + + def build_array(...) + Int8Array.new(...) end end @@ -130,8 +134,12 @@ def buffer_type :U8 end - def build_array(size, validity_buffer, values_buffer) - UInt8Array.new(size, validity_buffer, values_buffer) + def pack_template + "C" + end + + def build_array(...) + UInt8Array.new(...) end end @@ -154,8 +162,12 @@ def buffer_type :s16 end - def build_array(size, validity_buffer, values_buffer) - Int16Array.new(size, validity_buffer, values_buffer) + def pack_template + "s" + end + + def build_array(...) + Int16Array.new(...) end end @@ -178,8 +190,12 @@ def buffer_type :u16 end - def build_array(size, validity_buffer, values_buffer) - UInt16Array.new(size, validity_buffer, values_buffer) + def pack_template + "S" + end + + def build_array(...) + UInt16Array.new(...) end end @@ -202,8 +218,12 @@ def buffer_type :s32 end - def build_array(size, validity_buffer, values_buffer) - Int32Array.new(size, validity_buffer, values_buffer) + def pack_template + "l" + end + + def build_array(...) + Int32Array.new(...) end end @@ -226,8 +246,12 @@ def buffer_type :u32 end - def build_array(size, validity_buffer, values_buffer) - UInt32Array.new(size, validity_buffer, values_buffer) + def pack_template + "L" + end + + def build_array(...) + UInt32Array.new(...) end end @@ -250,8 +274,12 @@ def buffer_type :s64 end - def build_array(size, validity_buffer, values_buffer) - Int64Array.new(size, validity_buffer, values_buffer) + def pack_template + "q" + end + + def build_array(...) + Int64Array.new(...) end end @@ -274,8 +302,12 @@ def buffer_type :u64 end - def build_array(size, validity_buffer, values_buffer) - UInt64Array.new(size, validity_buffer, values_buffer) + def pack_template + "Q" + end + + def build_array(...) + UInt64Array.new(...) end end @@ -312,8 +344,12 @@ def buffer_type :f32 end - def build_array(size, validity_buffer, values_buffer) - Float32Array.new(size, validity_buffer, values_buffer) + def pack_template + "f" + end + + def build_array(...) + Float32Array.new(...) end end @@ -336,8 +372,12 @@ def buffer_type :f64 end - def build_array(size, validity_buffer, values_buffer) - Float64Array.new(size, validity_buffer, values_buffer) + def pack_template + "d" + end + + def build_array(...) + Float64Array.new(...) end end @@ -377,8 +417,8 @@ def buffer_type :s32 end - def build_array(size, validity_buffer, values_buffer) - Date32Array.new(size, validity_buffer, values_buffer) + def build_array(...) + Date32Array.new(...) end end @@ -401,8 +441,8 @@ def buffer_type :s64 end - def build_array(size, validity_buffer, values_buffer) - Date64Array.new(size, validity_buffer, values_buffer) + def build_array(...) + Date64Array.new(...) end end @@ -440,8 +480,8 @@ def buffer_type :s32 end - def build_array(size, validity_buffer, values_buffer) - Time32Array.new(self, size, validity_buffer, values_buffer) + def build_array(...) + Time32Array.new(self, ...) end end @@ -458,8 +498,8 @@ def buffer_type :s64 end - def build_array(size, validity_buffer, values_buffer) - Time64Array.new(self, size, validity_buffer, values_buffer) + def build_array(...) + Time64Array.new(self, ...) end end @@ -480,8 +520,8 @@ def buffer_type :s64 end - def build_array(size, validity_buffer, values_buffer) - TimestampArray.new(self, size, validity_buffer, values_buffer) + def build_array(...) + TimestampArray.new(self, ...) end def to_s @@ -531,8 +571,8 @@ def buffer_type :s32 end - def build_array(size, validity_buffer, values_buffer) - YearMonthIntervalArray.new(self, size, validity_buffer, values_buffer) + def build_array(...) + YearMonthIntervalArray.new(self, ...) end end @@ -549,8 +589,8 @@ def buffer_type :s32 end - def build_array(size, validity_buffer, values_buffer) - DayTimeIntervalArray.new(self, size, validity_buffer, values_buffer) + def build_array(...) + DayTimeIntervalArray.new(self, ...) end end @@ -567,11 +607,8 @@ def buffer_types @buffer_types ||= [:s32, :s32, :s64] end - def build_array(size, validity_buffer, values_buffer) - MonthDayNanoIntervalArray.new(self, - size, - validity_buffer, - values_buffer) + def build_array(...) + MonthDayNanoIntervalArray.new(self, ...) end end @@ -590,8 +627,8 @@ def buffer_type :s64 end - def build_array(size, validity_buffer, values_buffer) - DurationArray.new(self, size, validity_buffer, values_buffer) + def build_array(...) + DurationArray.new(self, ...) end def to_s @@ -627,11 +664,8 @@ def encoding Encoding::ASCII_8BIT end - def build_array(size, validity_buffer, offsets_buffer, values_buffer) - BinaryArray.new(size, - validity_buffer, - offsets_buffer, - values_buffer) + def build_array(...) + BinaryArray.new(...) end def to_flatbuffers @@ -658,11 +692,8 @@ def encoding Encoding::ASCII_8BIT end - def build_array(size, validity_buffer, offsets_buffer, values_buffer) - LargeBinaryArray.new(size, - validity_buffer, - offsets_buffer, - values_buffer) + def build_array(...) + LargeBinaryArray.new(...) end def to_flatbuffers @@ -689,8 +720,8 @@ def encoding Encoding::UTF_8 end - def build_array(size, validity_buffer, offsets_buffer, values_buffer) - UTF8Array.new(size, validity_buffer, offsets_buffer, values_buffer) + def build_array(...) + UTF8Array.new(...) end def to_flatbuffers @@ -717,11 +748,8 @@ def encoding Encoding::UTF_8 end - def build_array(size, validity_buffer, offsets_buffer, values_buffer) - LargeUTF8Array.new(size, - validity_buffer, - offsets_buffer, - values_buffer) + def build_array(...) + LargeUTF8Array.new(...) end def to_flatbuffers @@ -740,8 +768,8 @@ def name "FixedSizeBinary" end - def build_array(size, validity_buffer, values_buffer) - FixedSizeBinaryArray.new(self, size, validity_buffer, values_buffer) + def build_array(...) + FixedSizeBinaryArray.new(self, ...) end def to_s @@ -786,8 +814,8 @@ def name "Decimal128" end - def build_array(size, validity_buffer, values_buffer) - Decimal128Array.new(self, size, validity_buffer, values_buffer) + def build_array(...) + Decimal128Array.new(self, ...) end end @@ -800,8 +828,8 @@ def name "Decimal256" end - def build_array(size, validity_buffer, values_buffer) - Decimal256Array.new(self, size, validity_buffer, values_buffer) + def build_array(...) + Decimal256Array.new(self, ...) end end @@ -826,8 +854,8 @@ def offset_buffer_type :s32 # TODO: big endian support end - def build_array(size, validity_buffer, offsets_buffer, child) - ListArray.new(self, size, validity_buffer, offsets_buffer, child) + def build_array(...) + ListArray.new(self, ...) end def to_flatbuffers @@ -844,8 +872,8 @@ def offset_buffer_type :s64 # TODO: big endian support end - def build_array(size, validity_buffer, offsets_buffer, child) - LargeListArray.new(self, size, validity_buffer, offsets_buffer, child) + def build_array(...) + LargeListArray.new(self, ...) end def to_flatbuffers @@ -870,8 +898,8 @@ def to_s "#{super}<#{child.name}: #{child.type}>(#{@size})" end - def build_array(size, validity_buffer, child) - FixedSizeListArray.new(self, size, validity_buffer, child) + def build_array(...) + FixedSizeListArray.new(self, ...) end def to_flatbuffers @@ -892,8 +920,8 @@ def name "Struct" end - def build_array(size, validity_buffer, children) - StructArray.new(self, size, validity_buffer, children) + def build_array(...) + StructArray.new(self, ...) end def to_s @@ -942,8 +970,8 @@ def offset_buffer_type :s32 # TODO: big endian support end - def build_array(size, validity_buffer, offsets_buffer, child) - MapArray.new(self, size, validity_buffer, offsets_buffer, child) + def build_array(...) + MapArray.new(self, ...) end def to_s @@ -999,8 +1027,8 @@ def offset_buffer_type :s32 end - def build_array(size, types_buffer, offsets_buffer, children) - DenseUnionArray.new(self, size, types_buffer, offsets_buffer, children) + def build_array(...) + DenseUnionArray.new(self, ...) end end @@ -1013,8 +1041,8 @@ def name "SparseUnion" end - def build_array(size, types_buffer, children) - SparseUnionArray.new(self, size, types_buffer, children) + def build_array(...) + SparseUnionArray.new(self, ...) end end @@ -1038,12 +1066,8 @@ def name "Dictionary" end - def build_array(size, validity_buffer, indices_buffer, dictionaries) - DictionaryArray.new(self, - size, - validity_buffer, - indices_buffer, - dictionaries) + def build_array(...) + DictionaryArray.new(self, ...) end def build_fb_field(fb_field) diff --git a/ruby/red-arrow-format/test/test-boolean-array.rb b/ruby/red-arrow-format/test/test-boolean-array.rb deleted file mode 100644 index 1d80966ddc5..00000000000 --- a/ruby/red-arrow-format/test/test-boolean-array.rb +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -class TestBooleanArray < Test::Unit::TestCase - def test_mixed - assert_equal([true, nil, false], - ArrowFormat::BooleanArray.new([true, nil, false]).to_a) - end - - def test_no_null - assert_equal([true, false], - ArrowFormat::BooleanArray.new([true, false]).to_a) - end - - def test_more_8bits - values = [true] * 8 + [nil, false] - assert_equal(values, - ArrowFormat::BooleanArray.new(values).to_a) - end -end diff --git a/ruby/red-arrow-format/test/test-primitive-array.rb b/ruby/red-arrow-format/test/test-primitive-array.rb new file mode 100644 index 00000000000..7e09193ebfb --- /dev/null +++ b/ruby/red-arrow-format/test/test-primitive-array.rb @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestPrimitiveArray < Test::Unit::TestCase + def test_no_null + assert_equal([true, false], + ArrowFormat::BooleanArray.new([true, false]).to_a) + end + + def test_null_multiple_bytes + values = [true] * 8 + [nil, false] + assert_equal(values, + ArrowFormat::BooleanArray.new(values).to_a) + end + + def test_boolean + assert_equal([true, nil, false], + ArrowFormat::BooleanArray.new([true, nil, false]).to_a) + end + + def test_int8 + values = [-(2 ** 7), nil, (2 ** 7) - 1] + assert_equal(values, + ArrowFormat::Int8Array.new(values).to_a) + end + + def test_uint8 + values = [0, nil, (2 ** 8) - 1] + assert_equal(values, + ArrowFormat::UInt8Array.new(values).to_a) + end + + def test_int16 + values = [-(2 ** 15), nil, (2 ** 15) - 1] + assert_equal(values, + ArrowFormat::Int16Array.new(values).to_a) + end + + def test_uint16 + values = [0, nil, (2 ** 16) - 1] + assert_equal(values, + ArrowFormat::UInt16Array.new(values).to_a) + end + + def test_int32 + values = [-(2 ** 31), nil, (2 ** 31) - 1] + assert_equal(values, + ArrowFormat::Int32Array.new(values).to_a) + end + + def test_uint32 + values = [0, nil, (2 ** 32) - 1] + assert_equal(values, + ArrowFormat::UInt32Array.new(values).to_a) + end + + def test_int64 + values = [-(2 ** 63), nil, (2 ** 63) - 1] + assert_equal(values, + ArrowFormat::Int64Array.new(values).to_a) + end + + def test_uint64 + values = [0, nil, (2 ** 64) - 1] + assert_equal(values, + ArrowFormat::UInt64Array.new(values).to_a) + end + + def test_float32 + values = [-Float::INFINITY, -0.0, nil, +0.0, +Float::INFINITY] + assert_equal(values, + ArrowFormat::Float32Array.new(values).to_a) + end + + def test_float64 + values = [-Float::INFINITY, -0.0, nil, +0.0, +Float::INFINITY] + assert_equal(values, + ArrowFormat::Float64Array.new(values).to_a) + end +end