Skip to content

GH-50528: [Ruby] Add ArrowFormat::ArrayBuilder #50529

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kou wants to merge 1 commit into
base: main
Choose a base branch
from
Open

GH-50528: [Ruby] Add ArrowFormat::ArrayBuilder #50529

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 140 additions & 0 deletions ruby/red-arrow-format/lib/arrow-format/array-builder.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

module ArrowFormat
class ArrayBuilder
def initialize(values)
@values = values
end

def build
target = nil
@values.each do |value|
target = detect_target(value, target)
break if target and target[:detected]
end
if target
array_class = target[:array_class]
array_class ||= finalize_array_class(target)
type = target[:type]
if type
array_class.new(type, @values)
else
array_class.new(@values)
end
else
UTF8Array.new(@values)
end
end

private
def detect_target(value, target)
case value
when nil
target
when true, false
{
array_class: BooleanArray,
detected: true,
}
when String
{
array_class: UTF8Array,
detected: true,
}
when Float
{
array_class: Float64Array,
detected: true,
}
when Integer
target ||= {}
min = target[:min] || value
max = target[:max] || value
min = value if value < min
max = value if value > max

if target[:value_type] == :int or value < 0
{
value_type: :int,
min: min,
max: max,
}
else
{
value_type: :uint,
min: min,
max: max,
}
end
when Time
{
array_class: TimestampArray,
type: TimestampType.new(:nanosecond),
detected: true,
}
when Date
{
array_class: Date32Array,
detected: true,
}
else
{
array_class: UTF8Array,
detected: true,
}
end
end

def finalize_array_class(target)
value_type = target[:value_type]
case value_type
when :int
min = target[:min]
max = target[:max]

if Int8Type::MIN_VALUE <= min and max <= Int8Type::MAX_VALUE
Int8Array
elsif Int16Type::MIN_VALUE <= min and max <= Int16Type::MAX_VALUE
Int16Array
elsif Int32Type::MIN_VALUE <= min and max <= Int32Type::MAX_VALUE
Int32Array
elsif Int64Type::MIN_VALUE <= min and max <= Int64Type::MAX_VALUE
Int64Array
else
UTF8Array
end
when :uint
max = target[:max]

if max <= UInt8Type::MAX_VALUE
UInt8Array
elsif max <= UInt16Type::MAX_VALUE
UInt16Array
elsif max <= UInt32Type::MAX_VALUE
UInt32Array
elsif max <= UInt64Type::MAX_VALUE
UInt64Array
else
UTF8Array
end
else
UTF8Array
end
end
end
end
49 changes: 44 additions & 5 deletions ruby/red-arrow-format/lib/arrow-format/array.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

require "bigdecimal"

require_relative "array-builder"
require_relative "bitmap"
require_relative "bitmap-builder"

Expand All @@ -26,6 +27,12 @@ module ArrowFormat
class Array
include Enumerable

class << self
def build(values)
ArrayBuilder.new(values).build
end
end

attr_reader :type
attr_reader :size
alias_method :length, :size
Expand Down Expand Up @@ -271,9 +278,9 @@ def build_data(data, type)
if value.nil?
validity_buffer_builder ||= SparseBitmapBuilder.new
validity_buffer_builder.unset(i)
buffer.append_as_bytes(pack_value(nil, pack_template))
buffer.append_as_bytes(pack_value(nil, pack_template, type))
else
buffer.append_as_bytes(pack_value(value, pack_template))
buffer.append_as_bytes(pack_value(value, pack_template, type))
end
n += 1
end
Expand All @@ -283,7 +290,7 @@ def build_data(data, type)
return n, validity_buffer, IO::Buffer.for(buffer)
end

def pack_value(value, template)
def pack_value(value, template, type)
value = 0 if value.nil?
[value].pack(template)
end
Expand Down Expand Up @@ -455,6 +462,17 @@ def type
Date32Type.singleton
end
end

private
def pack_value(value, template, type)
if value.nil?
[0].pack(template)
elsif value.is_a?(Date)
[value.day].pack(template)
else
[value].pack(template)
end
end
end

class Date64Array < DateArray
Expand Down Expand Up @@ -490,6 +508,27 @@ def type_class
TimestampType
end
end

private
def pack_value(value, template, type)
if value.nil?
[0, 0].pack(template)
elsif value.is_a?(Time)
case type.unit
when :second
value = value.to_i
when :millisecond
value = (value.to_i * 1_000) + (value.nsec / 1_000_000)
when :microsecond
value = (value.to_i * 1_000_000) + (value.nsec / 1_000)
when :nanosecond
value = (value.to_i * 1_000_000_000) + value.nsec
end
[value].pack(template)
else
[value].pack(template)
end
end
end

class IntervalArray < TemporalArray
Expand Down Expand Up @@ -552,7 +591,7 @@ def element_size
super * 2
end

def pack_value(value, template)
def pack_value(value, template, type)
if value.nil?
[0, 0].pack(template)
elsif value.is_a?(Hash)
Expand Down Expand Up @@ -613,7 +652,7 @@ def element_size
IO::Buffer.size_of(@type.buffer_types)
end

def pack_value(value, template)
def pack_value(value, template, type)
if value.nil?
[0, 0, 0].pack(template)
elsif value.is_a?(Hash)
Expand Down
28 changes: 26 additions & 2 deletions ruby/red-arrow-format/lib/arrow-format/type.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ def to_flatbuffers
end

class Int8Type < IntType
MIN_VALUE = -(2 ** 7)
MAX_VALUE = (2 ** 7) - 1

class << self
def singleton
@singleton ||= new
Expand Down Expand Up @@ -116,6 +119,9 @@ def build_array(...)
end

class UInt8Type < IntType
MIN_VALUE = 0
MAX_VALUE = (2 ** 8) - 1

class << self
def singleton
@singleton ||= new
Expand Down Expand Up @@ -144,6 +150,9 @@ def build_array(...)
end

class Int16Type < IntType
MIN_VALUE = -(2 ** 15)
MAX_VALUE = (2 ** 15) - 1

class << self
def singleton
@singleton ||= new
Expand Down Expand Up @@ -172,6 +181,9 @@ def build_array(...)
end

class UInt16Type < IntType
MIN_VALUE = 0
MAX_VALUE = (2 ** 16) - 1

class << self
def singleton
@singleton ||= new
Expand Down Expand Up @@ -200,6 +212,9 @@ def build_array(...)
end

class Int32Type < IntType
MIN_VALUE = -(2 ** 31)
MAX_VALUE = (2 ** 31) - 1

class << self
def singleton
@singleton ||= new
Expand Down Expand Up @@ -228,6 +243,9 @@ def build_array(...)
end

class UInt32Type < IntType
MIN_VALUE = 0
MAX_VALUE = (2 ** 32) - 1

class << self
def singleton
@singleton ||= new
Expand Down Expand Up @@ -256,6 +274,9 @@ def build_array(...)
end

class Int64Type < IntType
MIN_VALUE = -(2 ** 63)
MAX_VALUE = (2 ** 63) - 1

class << self
def singleton
@singleton ||= new
Expand Down Expand Up @@ -284,6 +305,9 @@ def build_array(...)
end

class UInt64Type < IntType
MIN_VALUE = 0
MAX_VALUE = (2 ** 64) - 1

class << self
def singleton
@singleton ||= new
Expand Down Expand Up @@ -559,7 +583,7 @@ def try_convert(value)
case value
when Symbol
unit = value
new(unit, nil)
new(unit)
when ::Array
unit, time_zone = value
new(unit, time_zone)
Expand All @@ -573,7 +597,7 @@ def try_convert(value)

attr_reader :unit
attr_reader :time_zone
def initialize(unit, time_zone)
def initialize(unit, time_zone=nil)
super()
@unit = unit
@time_zone = time_zone
Expand Down
Loading
Loading