diff --git a/cpp/src/generated/parquet_types.cpp b/cpp/src/generated/parquet_types.cpp index cf8debb79e73..01b7b5b4a969 100644 --- a/cpp/src/generated/parquet_types.cpp +++ b/cpp/src/generated/parquet_types.cpp @@ -335,7 +335,8 @@ int _kFieldRepetitionTypeValues[] = { /** * The field is repeated and can contain 0 or more values */ - FieldRepetitionType::REPEATED + FieldRepetitionType::REPEATED, + FieldRepetitionType::VECTOR }; const char* _kFieldRepetitionTypeNames[] = { /** @@ -349,9 +350,10 @@ const char* _kFieldRepetitionTypeNames[] = { /** * The field is repeated and can contain 0 or more values */ - "REPEATED" + "REPEATED", + "VECTOR" }; -const std::map _FieldRepetitionType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kFieldRepetitionTypeValues, _kFieldRepetitionTypeNames), ::apache::thrift::TEnumIterator(-1, nullptr, nullptr)); +const std::map _FieldRepetitionType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(4, _kFieldRepetitionTypeValues, _kFieldRepetitionTypeNames), ::apache::thrift::TEnumIterator(-1, nullptr, nullptr)); std::ostream& operator<<(std::ostream& out, const FieldRepetitionType::type& val) { std::map::const_iterator it = _FieldRepetitionType_VALUES_TO_NAMES.find(val); @@ -2273,6 +2275,50 @@ void GeographyType::printTo(std::ostream& out) const { } +VectorType::~VectorType() noexcept { +} + +VectorType::VectorType() noexcept { +} +std::ostream& operator<<(std::ostream& out, const VectorType& obj) +{ + obj.printTo(out); + return out; +} + + +void swap(VectorType &a, VectorType &b) noexcept { + using ::std::swap; + (void) a; + (void) b; +} + +bool VectorType::operator==(const VectorType & /* rhs */) const +{ + return true; +} + +VectorType::VectorType(const VectorType& other119) noexcept { + (void) other119; +} +VectorType::VectorType(VectorType&& other120) noexcept { + (void) other120; +} +VectorType& VectorType::operator=(const VectorType& other121) noexcept { + (void) other121; + return *this; +} +VectorType& VectorType::operator=(VectorType&& other122) noexcept { + (void) other122; + return *this; +} +void VectorType::printTo(std::ostream& out) const { + using ::apache::thrift::to_string; + out << "VectorType("; + out << ")"; +} + + LogicalType::~LogicalType() noexcept { } @@ -2363,6 +2409,11 @@ void LogicalType::__set_GEOGRAPHY(const GeographyType& val) { this->GEOGRAPHY = val; __isset.GEOGRAPHY = true; } + +void LogicalType::__set_VECTOR(const VectorType& val) { + this->VECTOR = val; +__isset.VECTOR = true; +} std::ostream& operator<<(std::ostream& out, const LogicalType& obj) { obj.printTo(out); @@ -2389,6 +2440,7 @@ void swap(LogicalType &a, LogicalType &b) noexcept { swap(a.VARIANT, b.VARIANT); swap(a.GEOMETRY, b.GEOMETRY); swap(a.GEOGRAPHY, b.GEOGRAPHY); + swap(a.VECTOR, b.VECTOR); swap(a.__isset, b.__isset); } @@ -2462,89 +2514,97 @@ bool LogicalType::operator==(const LogicalType & rhs) const return false; else if (__isset.GEOGRAPHY && !(GEOGRAPHY == rhs.GEOGRAPHY)) return false; + if (__isset.VECTOR != rhs.__isset.VECTOR) + return false; + else if (__isset.VECTOR && !(VECTOR == rhs.VECTOR)) + return false; return true; } -LogicalType::LogicalType(const LogicalType& other119) { - STRING = other119.STRING; - MAP = other119.MAP; - LIST = other119.LIST; - ENUM = other119.ENUM; - DECIMAL = other119.DECIMAL; - DATE = other119.DATE; - TIME = other119.TIME; - TIMESTAMP = other119.TIMESTAMP; - INTEGER = other119.INTEGER; - UNKNOWN = other119.UNKNOWN; - JSON = other119.JSON; - BSON = other119.BSON; - UUID = other119.UUID; - FLOAT16 = other119.FLOAT16; - VARIANT = other119.VARIANT; - GEOMETRY = other119.GEOMETRY; - GEOGRAPHY = other119.GEOGRAPHY; - __isset = other119.__isset; -} -LogicalType::LogicalType(LogicalType&& other120) noexcept { - STRING = std::move(other120.STRING); - MAP = std::move(other120.MAP); - LIST = std::move(other120.LIST); - ENUM = std::move(other120.ENUM); - DECIMAL = std::move(other120.DECIMAL); - DATE = std::move(other120.DATE); - TIME = std::move(other120.TIME); - TIMESTAMP = std::move(other120.TIMESTAMP); - INTEGER = std::move(other120.INTEGER); - UNKNOWN = std::move(other120.UNKNOWN); - JSON = std::move(other120.JSON); - BSON = std::move(other120.BSON); - UUID = std::move(other120.UUID); - FLOAT16 = std::move(other120.FLOAT16); - VARIANT = std::move(other120.VARIANT); - GEOMETRY = std::move(other120.GEOMETRY); - GEOGRAPHY = std::move(other120.GEOGRAPHY); - __isset = other120.__isset; -} -LogicalType& LogicalType::operator=(const LogicalType& other121) { - STRING = other121.STRING; - MAP = other121.MAP; - LIST = other121.LIST; - ENUM = other121.ENUM; - DECIMAL = other121.DECIMAL; - DATE = other121.DATE; - TIME = other121.TIME; - TIMESTAMP = other121.TIMESTAMP; - INTEGER = other121.INTEGER; - UNKNOWN = other121.UNKNOWN; - JSON = other121.JSON; - BSON = other121.BSON; - UUID = other121.UUID; - FLOAT16 = other121.FLOAT16; - VARIANT = other121.VARIANT; - GEOMETRY = other121.GEOMETRY; - GEOGRAPHY = other121.GEOGRAPHY; - __isset = other121.__isset; +LogicalType::LogicalType(const LogicalType& other123) { + STRING = other123.STRING; + MAP = other123.MAP; + LIST = other123.LIST; + ENUM = other123.ENUM; + DECIMAL = other123.DECIMAL; + DATE = other123.DATE; + TIME = other123.TIME; + TIMESTAMP = other123.TIMESTAMP; + INTEGER = other123.INTEGER; + UNKNOWN = other123.UNKNOWN; + JSON = other123.JSON; + BSON = other123.BSON; + UUID = other123.UUID; + FLOAT16 = other123.FLOAT16; + VARIANT = other123.VARIANT; + GEOMETRY = other123.GEOMETRY; + GEOGRAPHY = other123.GEOGRAPHY; + VECTOR = other123.VECTOR; + __isset = other123.__isset; +} +LogicalType::LogicalType(LogicalType&& other124) noexcept { + STRING = std::move(other124.STRING); + MAP = std::move(other124.MAP); + LIST = std::move(other124.LIST); + ENUM = std::move(other124.ENUM); + DECIMAL = std::move(other124.DECIMAL); + DATE = std::move(other124.DATE); + TIME = std::move(other124.TIME); + TIMESTAMP = std::move(other124.TIMESTAMP); + INTEGER = std::move(other124.INTEGER); + UNKNOWN = std::move(other124.UNKNOWN); + JSON = std::move(other124.JSON); + BSON = std::move(other124.BSON); + UUID = std::move(other124.UUID); + FLOAT16 = std::move(other124.FLOAT16); + VARIANT = std::move(other124.VARIANT); + GEOMETRY = std::move(other124.GEOMETRY); + GEOGRAPHY = std::move(other124.GEOGRAPHY); + VECTOR = std::move(other124.VECTOR); + __isset = other124.__isset; +} +LogicalType& LogicalType::operator=(const LogicalType& other125) { + STRING = other125.STRING; + MAP = other125.MAP; + LIST = other125.LIST; + ENUM = other125.ENUM; + DECIMAL = other125.DECIMAL; + DATE = other125.DATE; + TIME = other125.TIME; + TIMESTAMP = other125.TIMESTAMP; + INTEGER = other125.INTEGER; + UNKNOWN = other125.UNKNOWN; + JSON = other125.JSON; + BSON = other125.BSON; + UUID = other125.UUID; + FLOAT16 = other125.FLOAT16; + VARIANT = other125.VARIANT; + GEOMETRY = other125.GEOMETRY; + GEOGRAPHY = other125.GEOGRAPHY; + VECTOR = other125.VECTOR; + __isset = other125.__isset; return *this; } -LogicalType& LogicalType::operator=(LogicalType&& other122) noexcept { - STRING = std::move(other122.STRING); - MAP = std::move(other122.MAP); - LIST = std::move(other122.LIST); - ENUM = std::move(other122.ENUM); - DECIMAL = std::move(other122.DECIMAL); - DATE = std::move(other122.DATE); - TIME = std::move(other122.TIME); - TIMESTAMP = std::move(other122.TIMESTAMP); - INTEGER = std::move(other122.INTEGER); - UNKNOWN = std::move(other122.UNKNOWN); - JSON = std::move(other122.JSON); - BSON = std::move(other122.BSON); - UUID = std::move(other122.UUID); - FLOAT16 = std::move(other122.FLOAT16); - VARIANT = std::move(other122.VARIANT); - GEOMETRY = std::move(other122.GEOMETRY); - GEOGRAPHY = std::move(other122.GEOGRAPHY); - __isset = other122.__isset; +LogicalType& LogicalType::operator=(LogicalType&& other126) noexcept { + STRING = std::move(other126.STRING); + MAP = std::move(other126.MAP); + LIST = std::move(other126.LIST); + ENUM = std::move(other126.ENUM); + DECIMAL = std::move(other126.DECIMAL); + DATE = std::move(other126.DATE); + TIME = std::move(other126.TIME); + TIMESTAMP = std::move(other126.TIMESTAMP); + INTEGER = std::move(other126.INTEGER); + UNKNOWN = std::move(other126.UNKNOWN); + JSON = std::move(other126.JSON); + BSON = std::move(other126.BSON); + UUID = std::move(other126.UUID); + FLOAT16 = std::move(other126.FLOAT16); + VARIANT = std::move(other126.VARIANT); + GEOMETRY = std::move(other126.GEOMETRY); + GEOGRAPHY = std::move(other126.GEOGRAPHY); + VECTOR = std::move(other126.VECTOR); + __isset = other126.__isset; return *this; } void LogicalType::printTo(std::ostream& out) const { @@ -2567,6 +2627,7 @@ void LogicalType::printTo(std::ostream& out) const { out << ", " << "VARIANT="; (__isset.VARIANT ? (out << to_string(VARIANT)) : (out << "")); out << ", " << "GEOMETRY="; (__isset.GEOMETRY ? (out << to_string(GEOMETRY)) : (out << "")); out << ", " << "GEOGRAPHY="; (__isset.GEOGRAPHY ? (out << to_string(GEOGRAPHY)) : (out << "")); + out << ", " << "VECTOR="; (__isset.VECTOR ? (out << to_string(VECTOR)) : (out << "")); out << ")"; } @@ -2583,7 +2644,8 @@ SchemaElement::SchemaElement() noexcept converted_type(static_cast(0)), scale(0), precision(0), - field_id(0) { + field_id(0), + vector_length(0) { } void SchemaElement::__set_type(const Type::type val) { @@ -2634,6 +2696,11 @@ void SchemaElement::__set_logicalType(const LogicalType& val) { this->logicalType = val; __isset.logicalType = true; } + +void SchemaElement::__set_vector_length(const int32_t val) { + this->vector_length = val; +__isset.vector_length = true; +} std::ostream& operator<<(std::ostream& out, const SchemaElement& obj) { obj.printTo(out); @@ -2653,6 +2720,7 @@ void swap(SchemaElement &a, SchemaElement &b) noexcept { swap(a.precision, b.precision); swap(a.field_id, b.field_id); swap(a.logicalType, b.logicalType); + swap(a.vector_length, b.vector_length); swap(a.__isset, b.__isset); } @@ -2696,61 +2764,69 @@ bool SchemaElement::operator==(const SchemaElement & rhs) const return false; else if (__isset.logicalType && !(logicalType == rhs.logicalType)) return false; + if (__isset.vector_length != rhs.__isset.vector_length) + return false; + else if (__isset.vector_length && !(vector_length == rhs.vector_length)) + return false; return true; } -SchemaElement::SchemaElement(const SchemaElement& other126) { - type = other126.type; - type_length = other126.type_length; - repetition_type = other126.repetition_type; - name = other126.name; - num_children = other126.num_children; - converted_type = other126.converted_type; - scale = other126.scale; - precision = other126.precision; - field_id = other126.field_id; - logicalType = other126.logicalType; - __isset = other126.__isset; -} -SchemaElement::SchemaElement(SchemaElement&& other127) noexcept { - type = other127.type; - type_length = other127.type_length; - repetition_type = other127.repetition_type; - name = std::move(other127.name); - num_children = other127.num_children; - converted_type = other127.converted_type; - scale = other127.scale; - precision = other127.precision; - field_id = other127.field_id; - logicalType = std::move(other127.logicalType); - __isset = other127.__isset; -} -SchemaElement& SchemaElement::operator=(const SchemaElement& other128) { - type = other128.type; - type_length = other128.type_length; - repetition_type = other128.repetition_type; - name = other128.name; - num_children = other128.num_children; - converted_type = other128.converted_type; - scale = other128.scale; - precision = other128.precision; - field_id = other128.field_id; - logicalType = other128.logicalType; - __isset = other128.__isset; +SchemaElement::SchemaElement(const SchemaElement& other130) { + type = other130.type; + type_length = other130.type_length; + repetition_type = other130.repetition_type; + name = other130.name; + num_children = other130.num_children; + converted_type = other130.converted_type; + scale = other130.scale; + precision = other130.precision; + field_id = other130.field_id; + logicalType = other130.logicalType; + vector_length = other130.vector_length; + __isset = other130.__isset; +} +SchemaElement::SchemaElement(SchemaElement&& other131) noexcept { + type = other131.type; + type_length = other131.type_length; + repetition_type = other131.repetition_type; + name = std::move(other131.name); + num_children = other131.num_children; + converted_type = other131.converted_type; + scale = other131.scale; + precision = other131.precision; + field_id = other131.field_id; + logicalType = std::move(other131.logicalType); + vector_length = other131.vector_length; + __isset = other131.__isset; +} +SchemaElement& SchemaElement::operator=(const SchemaElement& other132) { + type = other132.type; + type_length = other132.type_length; + repetition_type = other132.repetition_type; + name = other132.name; + num_children = other132.num_children; + converted_type = other132.converted_type; + scale = other132.scale; + precision = other132.precision; + field_id = other132.field_id; + logicalType = other132.logicalType; + vector_length = other132.vector_length; + __isset = other132.__isset; return *this; } -SchemaElement& SchemaElement::operator=(SchemaElement&& other129) noexcept { - type = other129.type; - type_length = other129.type_length; - repetition_type = other129.repetition_type; - name = std::move(other129.name); - num_children = other129.num_children; - converted_type = other129.converted_type; - scale = other129.scale; - precision = other129.precision; - field_id = other129.field_id; - logicalType = std::move(other129.logicalType); - __isset = other129.__isset; +SchemaElement& SchemaElement::operator=(SchemaElement&& other133) noexcept { + type = other133.type; + type_length = other133.type_length; + repetition_type = other133.repetition_type; + name = std::move(other133.name); + num_children = other133.num_children; + converted_type = other133.converted_type; + scale = other133.scale; + precision = other133.precision; + field_id = other133.field_id; + logicalType = std::move(other133.logicalType); + vector_length = other133.vector_length; + __isset = other133.__isset; return *this; } void SchemaElement::printTo(std::ostream& out) const { @@ -2766,6 +2842,7 @@ void SchemaElement::printTo(std::ostream& out) const { out << ", " << "precision="; (__isset.precision ? (out << to_string(precision)) : (out << "")); out << ", " << "field_id="; (__isset.field_id ? (out << to_string(field_id)) : (out << "")); out << ", " << "logicalType="; (__isset.logicalType ? (out << to_string(logicalType)) : (out << "")); + out << ", " << "vector_length="; (__isset.vector_length ? (out << to_string(vector_length)) : (out << "")); out << ")"; } @@ -2834,38 +2911,38 @@ bool DataPageHeader::operator==(const DataPageHeader & rhs) const return true; } -DataPageHeader::DataPageHeader(const DataPageHeader& other133) { - num_values = other133.num_values; - encoding = other133.encoding; - definition_level_encoding = other133.definition_level_encoding; - repetition_level_encoding = other133.repetition_level_encoding; - statistics = other133.statistics; - __isset = other133.__isset; -} -DataPageHeader::DataPageHeader(DataPageHeader&& other134) noexcept { - num_values = other134.num_values; - encoding = other134.encoding; - definition_level_encoding = other134.definition_level_encoding; - repetition_level_encoding = other134.repetition_level_encoding; - statistics = std::move(other134.statistics); - __isset = other134.__isset; -} -DataPageHeader& DataPageHeader::operator=(const DataPageHeader& other135) { - num_values = other135.num_values; - encoding = other135.encoding; - definition_level_encoding = other135.definition_level_encoding; - repetition_level_encoding = other135.repetition_level_encoding; - statistics = other135.statistics; - __isset = other135.__isset; +DataPageHeader::DataPageHeader(const DataPageHeader& other137) { + num_values = other137.num_values; + encoding = other137.encoding; + definition_level_encoding = other137.definition_level_encoding; + repetition_level_encoding = other137.repetition_level_encoding; + statistics = other137.statistics; + __isset = other137.__isset; +} +DataPageHeader::DataPageHeader(DataPageHeader&& other138) noexcept { + num_values = other138.num_values; + encoding = other138.encoding; + definition_level_encoding = other138.definition_level_encoding; + repetition_level_encoding = other138.repetition_level_encoding; + statistics = std::move(other138.statistics); + __isset = other138.__isset; +} +DataPageHeader& DataPageHeader::operator=(const DataPageHeader& other139) { + num_values = other139.num_values; + encoding = other139.encoding; + definition_level_encoding = other139.definition_level_encoding; + repetition_level_encoding = other139.repetition_level_encoding; + statistics = other139.statistics; + __isset = other139.__isset; return *this; } -DataPageHeader& DataPageHeader::operator=(DataPageHeader&& other136) noexcept { - num_values = other136.num_values; - encoding = other136.encoding; - definition_level_encoding = other136.definition_level_encoding; - repetition_level_encoding = other136.repetition_level_encoding; - statistics = std::move(other136.statistics); - __isset = other136.__isset; +DataPageHeader& DataPageHeader::operator=(DataPageHeader&& other140) noexcept { + num_values = other140.num_values; + encoding = other140.encoding; + definition_level_encoding = other140.definition_level_encoding; + repetition_level_encoding = other140.repetition_level_encoding; + statistics = std::move(other140.statistics); + __isset = other140.__isset; return *this; } void DataPageHeader::printTo(std::ostream& out) const { @@ -2903,18 +2980,18 @@ bool IndexPageHeader::operator==(const IndexPageHeader & /* rhs */) const return true; } -IndexPageHeader::IndexPageHeader(const IndexPageHeader& other137) noexcept { - (void) other137; +IndexPageHeader::IndexPageHeader(const IndexPageHeader& other141) noexcept { + (void) other141; } -IndexPageHeader::IndexPageHeader(IndexPageHeader&& other138) noexcept { - (void) other138; +IndexPageHeader::IndexPageHeader(IndexPageHeader&& other142) noexcept { + (void) other142; } -IndexPageHeader& IndexPageHeader::operator=(const IndexPageHeader& other139) noexcept { - (void) other139; +IndexPageHeader& IndexPageHeader::operator=(const IndexPageHeader& other143) noexcept { + (void) other143; return *this; } -IndexPageHeader& IndexPageHeader::operator=(IndexPageHeader&& other140) noexcept { - (void) other140; +IndexPageHeader& IndexPageHeader::operator=(IndexPageHeader&& other144) noexcept { + (void) other144; return *this; } void IndexPageHeader::printTo(std::ostream& out) const { @@ -2973,30 +3050,30 @@ bool DictionaryPageHeader::operator==(const DictionaryPageHeader & rhs) const return true; } -DictionaryPageHeader::DictionaryPageHeader(const DictionaryPageHeader& other142) noexcept { - num_values = other142.num_values; - encoding = other142.encoding; - is_sorted = other142.is_sorted; - __isset = other142.__isset; +DictionaryPageHeader::DictionaryPageHeader(const DictionaryPageHeader& other146) noexcept { + num_values = other146.num_values; + encoding = other146.encoding; + is_sorted = other146.is_sorted; + __isset = other146.__isset; } -DictionaryPageHeader::DictionaryPageHeader(DictionaryPageHeader&& other143) noexcept { - num_values = other143.num_values; - encoding = other143.encoding; - is_sorted = other143.is_sorted; - __isset = other143.__isset; +DictionaryPageHeader::DictionaryPageHeader(DictionaryPageHeader&& other147) noexcept { + num_values = other147.num_values; + encoding = other147.encoding; + is_sorted = other147.is_sorted; + __isset = other147.__isset; } -DictionaryPageHeader& DictionaryPageHeader::operator=(const DictionaryPageHeader& other144) noexcept { - num_values = other144.num_values; - encoding = other144.encoding; - is_sorted = other144.is_sorted; - __isset = other144.__isset; +DictionaryPageHeader& DictionaryPageHeader::operator=(const DictionaryPageHeader& other148) noexcept { + num_values = other148.num_values; + encoding = other148.encoding; + is_sorted = other148.is_sorted; + __isset = other148.__isset; return *this; } -DictionaryPageHeader& DictionaryPageHeader::operator=(DictionaryPageHeader&& other145) noexcept { - num_values = other145.num_values; - encoding = other145.encoding; - is_sorted = other145.is_sorted; - __isset = other145.__isset; +DictionaryPageHeader& DictionaryPageHeader::operator=(DictionaryPageHeader&& other149) noexcept { + num_values = other149.num_values; + encoding = other149.encoding; + is_sorted = other149.is_sorted; + __isset = other149.__isset; return *this; } void DictionaryPageHeader::printTo(std::ostream& out) const { @@ -3100,50 +3177,50 @@ bool DataPageHeaderV2::operator==(const DataPageHeaderV2 & rhs) const return true; } -DataPageHeaderV2::DataPageHeaderV2(const DataPageHeaderV2& other147) { - num_values = other147.num_values; - num_nulls = other147.num_nulls; - num_rows = other147.num_rows; - encoding = other147.encoding; - definition_levels_byte_length = other147.definition_levels_byte_length; - repetition_levels_byte_length = other147.repetition_levels_byte_length; - is_compressed = other147.is_compressed; - statistics = other147.statistics; - __isset = other147.__isset; -} -DataPageHeaderV2::DataPageHeaderV2(DataPageHeaderV2&& other148) noexcept { - num_values = other148.num_values; - num_nulls = other148.num_nulls; - num_rows = other148.num_rows; - encoding = other148.encoding; - definition_levels_byte_length = other148.definition_levels_byte_length; - repetition_levels_byte_length = other148.repetition_levels_byte_length; - is_compressed = other148.is_compressed; - statistics = std::move(other148.statistics); - __isset = other148.__isset; -} -DataPageHeaderV2& DataPageHeaderV2::operator=(const DataPageHeaderV2& other149) { - num_values = other149.num_values; - num_nulls = other149.num_nulls; - num_rows = other149.num_rows; - encoding = other149.encoding; - definition_levels_byte_length = other149.definition_levels_byte_length; - repetition_levels_byte_length = other149.repetition_levels_byte_length; - is_compressed = other149.is_compressed; - statistics = other149.statistics; - __isset = other149.__isset; +DataPageHeaderV2::DataPageHeaderV2(const DataPageHeaderV2& other151) { + num_values = other151.num_values; + num_nulls = other151.num_nulls; + num_rows = other151.num_rows; + encoding = other151.encoding; + definition_levels_byte_length = other151.definition_levels_byte_length; + repetition_levels_byte_length = other151.repetition_levels_byte_length; + is_compressed = other151.is_compressed; + statistics = other151.statistics; + __isset = other151.__isset; +} +DataPageHeaderV2::DataPageHeaderV2(DataPageHeaderV2&& other152) noexcept { + num_values = other152.num_values; + num_nulls = other152.num_nulls; + num_rows = other152.num_rows; + encoding = other152.encoding; + definition_levels_byte_length = other152.definition_levels_byte_length; + repetition_levels_byte_length = other152.repetition_levels_byte_length; + is_compressed = other152.is_compressed; + statistics = std::move(other152.statistics); + __isset = other152.__isset; +} +DataPageHeaderV2& DataPageHeaderV2::operator=(const DataPageHeaderV2& other153) { + num_values = other153.num_values; + num_nulls = other153.num_nulls; + num_rows = other153.num_rows; + encoding = other153.encoding; + definition_levels_byte_length = other153.definition_levels_byte_length; + repetition_levels_byte_length = other153.repetition_levels_byte_length; + is_compressed = other153.is_compressed; + statistics = other153.statistics; + __isset = other153.__isset; return *this; } -DataPageHeaderV2& DataPageHeaderV2::operator=(DataPageHeaderV2&& other150) noexcept { - num_values = other150.num_values; - num_nulls = other150.num_nulls; - num_rows = other150.num_rows; - encoding = other150.encoding; - definition_levels_byte_length = other150.definition_levels_byte_length; - repetition_levels_byte_length = other150.repetition_levels_byte_length; - is_compressed = other150.is_compressed; - statistics = std::move(other150.statistics); - __isset = other150.__isset; +DataPageHeaderV2& DataPageHeaderV2::operator=(DataPageHeaderV2&& other154) noexcept { + num_values = other154.num_values; + num_nulls = other154.num_nulls; + num_rows = other154.num_rows; + encoding = other154.encoding; + definition_levels_byte_length = other154.definition_levels_byte_length; + repetition_levels_byte_length = other154.repetition_levels_byte_length; + is_compressed = other154.is_compressed; + statistics = std::move(other154.statistics); + __isset = other154.__isset; return *this; } void DataPageHeaderV2::printTo(std::ostream& out) const { @@ -3184,18 +3261,18 @@ bool SplitBlockAlgorithm::operator==(const SplitBlockAlgorithm & /* rhs */) cons return true; } -SplitBlockAlgorithm::SplitBlockAlgorithm(const SplitBlockAlgorithm& other151) noexcept { - (void) other151; +SplitBlockAlgorithm::SplitBlockAlgorithm(const SplitBlockAlgorithm& other155) noexcept { + (void) other155; } -SplitBlockAlgorithm::SplitBlockAlgorithm(SplitBlockAlgorithm&& other152) noexcept { - (void) other152; +SplitBlockAlgorithm::SplitBlockAlgorithm(SplitBlockAlgorithm&& other156) noexcept { + (void) other156; } -SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(const SplitBlockAlgorithm& other153) noexcept { - (void) other153; +SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(const SplitBlockAlgorithm& other157) noexcept { + (void) other157; return *this; } -SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(SplitBlockAlgorithm&& other154) noexcept { - (void) other154; +SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(SplitBlockAlgorithm&& other158) noexcept { + (void) other158; return *this; } void SplitBlockAlgorithm::printTo(std::ostream& out) const { @@ -3237,22 +3314,22 @@ bool BloomFilterAlgorithm::operator==(const BloomFilterAlgorithm & rhs) const return true; } -BloomFilterAlgorithm::BloomFilterAlgorithm(const BloomFilterAlgorithm& other155) noexcept { - BLOCK = other155.BLOCK; - __isset = other155.__isset; +BloomFilterAlgorithm::BloomFilterAlgorithm(const BloomFilterAlgorithm& other159) noexcept { + BLOCK = other159.BLOCK; + __isset = other159.__isset; } -BloomFilterAlgorithm::BloomFilterAlgorithm(BloomFilterAlgorithm&& other156) noexcept { - BLOCK = std::move(other156.BLOCK); - __isset = other156.__isset; +BloomFilterAlgorithm::BloomFilterAlgorithm(BloomFilterAlgorithm&& other160) noexcept { + BLOCK = std::move(other160.BLOCK); + __isset = other160.__isset; } -BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(const BloomFilterAlgorithm& other157) noexcept { - BLOCK = other157.BLOCK; - __isset = other157.__isset; +BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(const BloomFilterAlgorithm& other161) noexcept { + BLOCK = other161.BLOCK; + __isset = other161.__isset; return *this; } -BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(BloomFilterAlgorithm&& other158) noexcept { - BLOCK = std::move(other158.BLOCK); - __isset = other158.__isset; +BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(BloomFilterAlgorithm&& other162) noexcept { + BLOCK = std::move(other162.BLOCK); + __isset = other162.__isset; return *this; } void BloomFilterAlgorithm::printTo(std::ostream& out) const { @@ -3286,18 +3363,18 @@ bool XxHash::operator==(const XxHash & /* rhs */) const return true; } -XxHash::XxHash(const XxHash& other159) noexcept { - (void) other159; +XxHash::XxHash(const XxHash& other163) noexcept { + (void) other163; } -XxHash::XxHash(XxHash&& other160) noexcept { - (void) other160; +XxHash::XxHash(XxHash&& other164) noexcept { + (void) other164; } -XxHash& XxHash::operator=(const XxHash& other161) noexcept { - (void) other161; +XxHash& XxHash::operator=(const XxHash& other165) noexcept { + (void) other165; return *this; } -XxHash& XxHash::operator=(XxHash&& other162) noexcept { - (void) other162; +XxHash& XxHash::operator=(XxHash&& other166) noexcept { + (void) other166; return *this; } void XxHash::printTo(std::ostream& out) const { @@ -3339,22 +3416,22 @@ bool BloomFilterHash::operator==(const BloomFilterHash & rhs) const return true; } -BloomFilterHash::BloomFilterHash(const BloomFilterHash& other163) noexcept { - XXHASH = other163.XXHASH; - __isset = other163.__isset; +BloomFilterHash::BloomFilterHash(const BloomFilterHash& other167) noexcept { + XXHASH = other167.XXHASH; + __isset = other167.__isset; } -BloomFilterHash::BloomFilterHash(BloomFilterHash&& other164) noexcept { - XXHASH = std::move(other164.XXHASH); - __isset = other164.__isset; +BloomFilterHash::BloomFilterHash(BloomFilterHash&& other168) noexcept { + XXHASH = std::move(other168.XXHASH); + __isset = other168.__isset; } -BloomFilterHash& BloomFilterHash::operator=(const BloomFilterHash& other165) noexcept { - XXHASH = other165.XXHASH; - __isset = other165.__isset; +BloomFilterHash& BloomFilterHash::operator=(const BloomFilterHash& other169) noexcept { + XXHASH = other169.XXHASH; + __isset = other169.__isset; return *this; } -BloomFilterHash& BloomFilterHash::operator=(BloomFilterHash&& other166) noexcept { - XXHASH = std::move(other166.XXHASH); - __isset = other166.__isset; +BloomFilterHash& BloomFilterHash::operator=(BloomFilterHash&& other170) noexcept { + XXHASH = std::move(other170.XXHASH); + __isset = other170.__isset; return *this; } void BloomFilterHash::printTo(std::ostream& out) const { @@ -3388,18 +3465,18 @@ bool Uncompressed::operator==(const Uncompressed & /* rhs */) const return true; } -Uncompressed::Uncompressed(const Uncompressed& other167) noexcept { - (void) other167; +Uncompressed::Uncompressed(const Uncompressed& other171) noexcept { + (void) other171; } -Uncompressed::Uncompressed(Uncompressed&& other168) noexcept { - (void) other168; +Uncompressed::Uncompressed(Uncompressed&& other172) noexcept { + (void) other172; } -Uncompressed& Uncompressed::operator=(const Uncompressed& other169) noexcept { - (void) other169; +Uncompressed& Uncompressed::operator=(const Uncompressed& other173) noexcept { + (void) other173; return *this; } -Uncompressed& Uncompressed::operator=(Uncompressed&& other170) noexcept { - (void) other170; +Uncompressed& Uncompressed::operator=(Uncompressed&& other174) noexcept { + (void) other174; return *this; } void Uncompressed::printTo(std::ostream& out) const { @@ -3441,22 +3518,22 @@ bool BloomFilterCompression::operator==(const BloomFilterCompression & rhs) cons return true; } -BloomFilterCompression::BloomFilterCompression(const BloomFilterCompression& other171) noexcept { - UNCOMPRESSED = other171.UNCOMPRESSED; - __isset = other171.__isset; +BloomFilterCompression::BloomFilterCompression(const BloomFilterCompression& other175) noexcept { + UNCOMPRESSED = other175.UNCOMPRESSED; + __isset = other175.__isset; } -BloomFilterCompression::BloomFilterCompression(BloomFilterCompression&& other172) noexcept { - UNCOMPRESSED = std::move(other172.UNCOMPRESSED); - __isset = other172.__isset; +BloomFilterCompression::BloomFilterCompression(BloomFilterCompression&& other176) noexcept { + UNCOMPRESSED = std::move(other176.UNCOMPRESSED); + __isset = other176.__isset; } -BloomFilterCompression& BloomFilterCompression::operator=(const BloomFilterCompression& other173) noexcept { - UNCOMPRESSED = other173.UNCOMPRESSED; - __isset = other173.__isset; +BloomFilterCompression& BloomFilterCompression::operator=(const BloomFilterCompression& other177) noexcept { + UNCOMPRESSED = other177.UNCOMPRESSED; + __isset = other177.__isset; return *this; } -BloomFilterCompression& BloomFilterCompression::operator=(BloomFilterCompression&& other174) noexcept { - UNCOMPRESSED = std::move(other174.UNCOMPRESSED); - __isset = other174.__isset; +BloomFilterCompression& BloomFilterCompression::operator=(BloomFilterCompression&& other178) noexcept { + UNCOMPRESSED = std::move(other178.UNCOMPRESSED); + __isset = other178.__isset; return *this; } void BloomFilterCompression::printTo(std::ostream& out) const { @@ -3517,30 +3594,30 @@ bool BloomFilterHeader::operator==(const BloomFilterHeader & rhs) const return true; } -BloomFilterHeader::BloomFilterHeader(const BloomFilterHeader& other175) noexcept { - numBytes = other175.numBytes; - algorithm = other175.algorithm; - hash = other175.hash; - compression = other175.compression; +BloomFilterHeader::BloomFilterHeader(const BloomFilterHeader& other179) noexcept { + numBytes = other179.numBytes; + algorithm = other179.algorithm; + hash = other179.hash; + compression = other179.compression; } -BloomFilterHeader::BloomFilterHeader(BloomFilterHeader&& other176) noexcept { - numBytes = other176.numBytes; - algorithm = std::move(other176.algorithm); - hash = std::move(other176.hash); - compression = std::move(other176.compression); +BloomFilterHeader::BloomFilterHeader(BloomFilterHeader&& other180) noexcept { + numBytes = other180.numBytes; + algorithm = std::move(other180.algorithm); + hash = std::move(other180.hash); + compression = std::move(other180.compression); } -BloomFilterHeader& BloomFilterHeader::operator=(const BloomFilterHeader& other177) noexcept { - numBytes = other177.numBytes; - algorithm = other177.algorithm; - hash = other177.hash; - compression = other177.compression; +BloomFilterHeader& BloomFilterHeader::operator=(const BloomFilterHeader& other181) noexcept { + numBytes = other181.numBytes; + algorithm = other181.algorithm; + hash = other181.hash; + compression = other181.compression; return *this; } -BloomFilterHeader& BloomFilterHeader::operator=(BloomFilterHeader&& other178) noexcept { - numBytes = other178.numBytes; - algorithm = std::move(other178.algorithm); - hash = std::move(other178.hash); - compression = std::move(other178.compression); +BloomFilterHeader& BloomFilterHeader::operator=(BloomFilterHeader&& other182) noexcept { + numBytes = other182.numBytes; + algorithm = std::move(other182.algorithm); + hash = std::move(other182.hash); + compression = std::move(other182.compression); return *this; } void BloomFilterHeader::printTo(std::ostream& out) const { @@ -3651,50 +3728,50 @@ bool PageHeader::operator==(const PageHeader & rhs) const return true; } -PageHeader::PageHeader(const PageHeader& other180) { - type = other180.type; - uncompressed_page_size = other180.uncompressed_page_size; - compressed_page_size = other180.compressed_page_size; - crc = other180.crc; - data_page_header = other180.data_page_header; - index_page_header = other180.index_page_header; - dictionary_page_header = other180.dictionary_page_header; - data_page_header_v2 = other180.data_page_header_v2; - __isset = other180.__isset; -} -PageHeader::PageHeader(PageHeader&& other181) noexcept { - type = other181.type; - uncompressed_page_size = other181.uncompressed_page_size; - compressed_page_size = other181.compressed_page_size; - crc = other181.crc; - data_page_header = std::move(other181.data_page_header); - index_page_header = std::move(other181.index_page_header); - dictionary_page_header = std::move(other181.dictionary_page_header); - data_page_header_v2 = std::move(other181.data_page_header_v2); - __isset = other181.__isset; -} -PageHeader& PageHeader::operator=(const PageHeader& other182) { - type = other182.type; - uncompressed_page_size = other182.uncompressed_page_size; - compressed_page_size = other182.compressed_page_size; - crc = other182.crc; - data_page_header = other182.data_page_header; - index_page_header = other182.index_page_header; - dictionary_page_header = other182.dictionary_page_header; - data_page_header_v2 = other182.data_page_header_v2; - __isset = other182.__isset; +PageHeader::PageHeader(const PageHeader& other184) { + type = other184.type; + uncompressed_page_size = other184.uncompressed_page_size; + compressed_page_size = other184.compressed_page_size; + crc = other184.crc; + data_page_header = other184.data_page_header; + index_page_header = other184.index_page_header; + dictionary_page_header = other184.dictionary_page_header; + data_page_header_v2 = other184.data_page_header_v2; + __isset = other184.__isset; +} +PageHeader::PageHeader(PageHeader&& other185) noexcept { + type = other185.type; + uncompressed_page_size = other185.uncompressed_page_size; + compressed_page_size = other185.compressed_page_size; + crc = other185.crc; + data_page_header = std::move(other185.data_page_header); + index_page_header = std::move(other185.index_page_header); + dictionary_page_header = std::move(other185.dictionary_page_header); + data_page_header_v2 = std::move(other185.data_page_header_v2); + __isset = other185.__isset; +} +PageHeader& PageHeader::operator=(const PageHeader& other186) { + type = other186.type; + uncompressed_page_size = other186.uncompressed_page_size; + compressed_page_size = other186.compressed_page_size; + crc = other186.crc; + data_page_header = other186.data_page_header; + index_page_header = other186.index_page_header; + dictionary_page_header = other186.dictionary_page_header; + data_page_header_v2 = other186.data_page_header_v2; + __isset = other186.__isset; return *this; } -PageHeader& PageHeader::operator=(PageHeader&& other183) noexcept { - type = other183.type; - uncompressed_page_size = other183.uncompressed_page_size; - compressed_page_size = other183.compressed_page_size; - crc = other183.crc; - data_page_header = std::move(other183.data_page_header); - index_page_header = std::move(other183.index_page_header); - dictionary_page_header = std::move(other183.dictionary_page_header); - data_page_header_v2 = std::move(other183.data_page_header_v2); - __isset = other183.__isset; +PageHeader& PageHeader::operator=(PageHeader&& other187) noexcept { + type = other187.type; + uncompressed_page_size = other187.uncompressed_page_size; + compressed_page_size = other187.compressed_page_size; + crc = other187.crc; + data_page_header = std::move(other187.data_page_header); + index_page_header = std::move(other187.index_page_header); + dictionary_page_header = std::move(other187.dictionary_page_header); + data_page_header_v2 = std::move(other187.data_page_header_v2); + __isset = other187.__isset; return *this; } void PageHeader::printTo(std::ostream& out) const { @@ -3753,26 +3830,26 @@ bool KeyValue::operator==(const KeyValue & rhs) const return true; } -KeyValue::KeyValue(const KeyValue& other184) { - key = other184.key; - value = other184.value; - __isset = other184.__isset; +KeyValue::KeyValue(const KeyValue& other188) { + key = other188.key; + value = other188.value; + __isset = other188.__isset; } -KeyValue::KeyValue(KeyValue&& other185) noexcept { - key = std::move(other185.key); - value = std::move(other185.value); - __isset = other185.__isset; +KeyValue::KeyValue(KeyValue&& other189) noexcept { + key = std::move(other189.key); + value = std::move(other189.value); + __isset = other189.__isset; } -KeyValue& KeyValue::operator=(const KeyValue& other186) { - key = other186.key; - value = other186.value; - __isset = other186.__isset; +KeyValue& KeyValue::operator=(const KeyValue& other190) { + key = other190.key; + value = other190.value; + __isset = other190.__isset; return *this; } -KeyValue& KeyValue::operator=(KeyValue&& other187) noexcept { - key = std::move(other187.key); - value = std::move(other187.value); - __isset = other187.__isset; +KeyValue& KeyValue::operator=(KeyValue&& other191) noexcept { + key = std::move(other191.key); + value = std::move(other191.value); + __isset = other191.__isset; return *this; } void KeyValue::printTo(std::ostream& out) const { @@ -3829,26 +3906,26 @@ bool SortingColumn::operator==(const SortingColumn & rhs) const return true; } -SortingColumn::SortingColumn(const SortingColumn& other188) noexcept { - column_idx = other188.column_idx; - descending = other188.descending; - nulls_first = other188.nulls_first; +SortingColumn::SortingColumn(const SortingColumn& other192) noexcept { + column_idx = other192.column_idx; + descending = other192.descending; + nulls_first = other192.nulls_first; } -SortingColumn::SortingColumn(SortingColumn&& other189) noexcept { - column_idx = other189.column_idx; - descending = other189.descending; - nulls_first = other189.nulls_first; +SortingColumn::SortingColumn(SortingColumn&& other193) noexcept { + column_idx = other193.column_idx; + descending = other193.descending; + nulls_first = other193.nulls_first; } -SortingColumn& SortingColumn::operator=(const SortingColumn& other190) noexcept { - column_idx = other190.column_idx; - descending = other190.descending; - nulls_first = other190.nulls_first; +SortingColumn& SortingColumn::operator=(const SortingColumn& other194) noexcept { + column_idx = other194.column_idx; + descending = other194.descending; + nulls_first = other194.nulls_first; return *this; } -SortingColumn& SortingColumn::operator=(SortingColumn&& other191) noexcept { - column_idx = other191.column_idx; - descending = other191.descending; - nulls_first = other191.nulls_first; +SortingColumn& SortingColumn::operator=(SortingColumn&& other195) noexcept { + column_idx = other195.column_idx; + descending = other195.descending; + nulls_first = other195.nulls_first; return *this; } void SortingColumn::printTo(std::ostream& out) const { @@ -3906,26 +3983,26 @@ bool PageEncodingStats::operator==(const PageEncodingStats & rhs) const return true; } -PageEncodingStats::PageEncodingStats(const PageEncodingStats& other194) noexcept { - page_type = other194.page_type; - encoding = other194.encoding; - count = other194.count; +PageEncodingStats::PageEncodingStats(const PageEncodingStats& other198) noexcept { + page_type = other198.page_type; + encoding = other198.encoding; + count = other198.count; } -PageEncodingStats::PageEncodingStats(PageEncodingStats&& other195) noexcept { - page_type = other195.page_type; - encoding = other195.encoding; - count = other195.count; +PageEncodingStats::PageEncodingStats(PageEncodingStats&& other199) noexcept { + page_type = other199.page_type; + encoding = other199.encoding; + count = other199.count; } -PageEncodingStats& PageEncodingStats::operator=(const PageEncodingStats& other196) noexcept { - page_type = other196.page_type; - encoding = other196.encoding; - count = other196.count; +PageEncodingStats& PageEncodingStats::operator=(const PageEncodingStats& other200) noexcept { + page_type = other200.page_type; + encoding = other200.encoding; + count = other200.count; return *this; } -PageEncodingStats& PageEncodingStats::operator=(PageEncodingStats&& other197) noexcept { - page_type = other197.page_type; - encoding = other197.encoding; - count = other197.count; +PageEncodingStats& PageEncodingStats::operator=(PageEncodingStats&& other201) noexcept { + page_type = other201.page_type; + encoding = other201.encoding; + count = other201.count; return *this; } void PageEncodingStats::printTo(std::ostream& out) const { @@ -4116,86 +4193,86 @@ bool ColumnMetaData::operator==(const ColumnMetaData & rhs) const return true; } -ColumnMetaData::ColumnMetaData(const ColumnMetaData& other225) { - type = other225.type; - encodings = other225.encodings; - path_in_schema = other225.path_in_schema; - codec = other225.codec; - num_values = other225.num_values; - total_uncompressed_size = other225.total_uncompressed_size; - total_compressed_size = other225.total_compressed_size; - key_value_metadata = other225.key_value_metadata; - data_page_offset = other225.data_page_offset; - index_page_offset = other225.index_page_offset; - dictionary_page_offset = other225.dictionary_page_offset; - statistics = other225.statistics; - encoding_stats = other225.encoding_stats; - bloom_filter_offset = other225.bloom_filter_offset; - bloom_filter_length = other225.bloom_filter_length; - size_statistics = other225.size_statistics; - geospatial_statistics = other225.geospatial_statistics; - __isset = other225.__isset; -} -ColumnMetaData::ColumnMetaData(ColumnMetaData&& other226) noexcept { - type = other226.type; - encodings = std::move(other226.encodings); - path_in_schema = std::move(other226.path_in_schema); - codec = other226.codec; - num_values = other226.num_values; - total_uncompressed_size = other226.total_uncompressed_size; - total_compressed_size = other226.total_compressed_size; - key_value_metadata = std::move(other226.key_value_metadata); - data_page_offset = other226.data_page_offset; - index_page_offset = other226.index_page_offset; - dictionary_page_offset = other226.dictionary_page_offset; - statistics = std::move(other226.statistics); - encoding_stats = std::move(other226.encoding_stats); - bloom_filter_offset = other226.bloom_filter_offset; - bloom_filter_length = other226.bloom_filter_length; - size_statistics = std::move(other226.size_statistics); - geospatial_statistics = std::move(other226.geospatial_statistics); - __isset = other226.__isset; -} -ColumnMetaData& ColumnMetaData::operator=(const ColumnMetaData& other227) { - type = other227.type; - encodings = other227.encodings; - path_in_schema = other227.path_in_schema; - codec = other227.codec; - num_values = other227.num_values; - total_uncompressed_size = other227.total_uncompressed_size; - total_compressed_size = other227.total_compressed_size; - key_value_metadata = other227.key_value_metadata; - data_page_offset = other227.data_page_offset; - index_page_offset = other227.index_page_offset; - dictionary_page_offset = other227.dictionary_page_offset; - statistics = other227.statistics; - encoding_stats = other227.encoding_stats; - bloom_filter_offset = other227.bloom_filter_offset; - bloom_filter_length = other227.bloom_filter_length; - size_statistics = other227.size_statistics; - geospatial_statistics = other227.geospatial_statistics; - __isset = other227.__isset; +ColumnMetaData::ColumnMetaData(const ColumnMetaData& other229) { + type = other229.type; + encodings = other229.encodings; + path_in_schema = other229.path_in_schema; + codec = other229.codec; + num_values = other229.num_values; + total_uncompressed_size = other229.total_uncompressed_size; + total_compressed_size = other229.total_compressed_size; + key_value_metadata = other229.key_value_metadata; + data_page_offset = other229.data_page_offset; + index_page_offset = other229.index_page_offset; + dictionary_page_offset = other229.dictionary_page_offset; + statistics = other229.statistics; + encoding_stats = other229.encoding_stats; + bloom_filter_offset = other229.bloom_filter_offset; + bloom_filter_length = other229.bloom_filter_length; + size_statistics = other229.size_statistics; + geospatial_statistics = other229.geospatial_statistics; + __isset = other229.__isset; +} +ColumnMetaData::ColumnMetaData(ColumnMetaData&& other230) noexcept { + type = other230.type; + encodings = std::move(other230.encodings); + path_in_schema = std::move(other230.path_in_schema); + codec = other230.codec; + num_values = other230.num_values; + total_uncompressed_size = other230.total_uncompressed_size; + total_compressed_size = other230.total_compressed_size; + key_value_metadata = std::move(other230.key_value_metadata); + data_page_offset = other230.data_page_offset; + index_page_offset = other230.index_page_offset; + dictionary_page_offset = other230.dictionary_page_offset; + statistics = std::move(other230.statistics); + encoding_stats = std::move(other230.encoding_stats); + bloom_filter_offset = other230.bloom_filter_offset; + bloom_filter_length = other230.bloom_filter_length; + size_statistics = std::move(other230.size_statistics); + geospatial_statistics = std::move(other230.geospatial_statistics); + __isset = other230.__isset; +} +ColumnMetaData& ColumnMetaData::operator=(const ColumnMetaData& other231) { + type = other231.type; + encodings = other231.encodings; + path_in_schema = other231.path_in_schema; + codec = other231.codec; + num_values = other231.num_values; + total_uncompressed_size = other231.total_uncompressed_size; + total_compressed_size = other231.total_compressed_size; + key_value_metadata = other231.key_value_metadata; + data_page_offset = other231.data_page_offset; + index_page_offset = other231.index_page_offset; + dictionary_page_offset = other231.dictionary_page_offset; + statistics = other231.statistics; + encoding_stats = other231.encoding_stats; + bloom_filter_offset = other231.bloom_filter_offset; + bloom_filter_length = other231.bloom_filter_length; + size_statistics = other231.size_statistics; + geospatial_statistics = other231.geospatial_statistics; + __isset = other231.__isset; return *this; } -ColumnMetaData& ColumnMetaData::operator=(ColumnMetaData&& other228) noexcept { - type = other228.type; - encodings = std::move(other228.encodings); - path_in_schema = std::move(other228.path_in_schema); - codec = other228.codec; - num_values = other228.num_values; - total_uncompressed_size = other228.total_uncompressed_size; - total_compressed_size = other228.total_compressed_size; - key_value_metadata = std::move(other228.key_value_metadata); - data_page_offset = other228.data_page_offset; - index_page_offset = other228.index_page_offset; - dictionary_page_offset = other228.dictionary_page_offset; - statistics = std::move(other228.statistics); - encoding_stats = std::move(other228.encoding_stats); - bloom_filter_offset = other228.bloom_filter_offset; - bloom_filter_length = other228.bloom_filter_length; - size_statistics = std::move(other228.size_statistics); - geospatial_statistics = std::move(other228.geospatial_statistics); - __isset = other228.__isset; +ColumnMetaData& ColumnMetaData::operator=(ColumnMetaData&& other232) noexcept { + type = other232.type; + encodings = std::move(other232.encodings); + path_in_schema = std::move(other232.path_in_schema); + codec = other232.codec; + num_values = other232.num_values; + total_uncompressed_size = other232.total_uncompressed_size; + total_compressed_size = other232.total_compressed_size; + key_value_metadata = std::move(other232.key_value_metadata); + data_page_offset = other232.data_page_offset; + index_page_offset = other232.index_page_offset; + dictionary_page_offset = other232.dictionary_page_offset; + statistics = std::move(other232.statistics); + encoding_stats = std::move(other232.encoding_stats); + bloom_filter_offset = other232.bloom_filter_offset; + bloom_filter_length = other232.bloom_filter_length; + size_statistics = std::move(other232.size_statistics); + geospatial_statistics = std::move(other232.geospatial_statistics); + __isset = other232.__isset; return *this; } void ColumnMetaData::printTo(std::ostream& out) const { @@ -4245,18 +4322,18 @@ bool EncryptionWithFooterKey::operator==(const EncryptionWithFooterKey & /* rhs return true; } -EncryptionWithFooterKey::EncryptionWithFooterKey(const EncryptionWithFooterKey& other229) noexcept { - (void) other229; +EncryptionWithFooterKey::EncryptionWithFooterKey(const EncryptionWithFooterKey& other233) noexcept { + (void) other233; } -EncryptionWithFooterKey::EncryptionWithFooterKey(EncryptionWithFooterKey&& other230) noexcept { - (void) other230; +EncryptionWithFooterKey::EncryptionWithFooterKey(EncryptionWithFooterKey&& other234) noexcept { + (void) other234; } -EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(const EncryptionWithFooterKey& other231) noexcept { - (void) other231; +EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(const EncryptionWithFooterKey& other235) noexcept { + (void) other235; return *this; } -EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(EncryptionWithFooterKey&& other232) noexcept { - (void) other232; +EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(EncryptionWithFooterKey&& other236) noexcept { + (void) other236; return *this; } void EncryptionWithFooterKey::printTo(std::ostream& out) const { @@ -4306,26 +4383,26 @@ bool EncryptionWithColumnKey::operator==(const EncryptionWithColumnKey & rhs) co return true; } -EncryptionWithColumnKey::EncryptionWithColumnKey(const EncryptionWithColumnKey& other239) { - path_in_schema = other239.path_in_schema; - key_metadata = other239.key_metadata; - __isset = other239.__isset; +EncryptionWithColumnKey::EncryptionWithColumnKey(const EncryptionWithColumnKey& other243) { + path_in_schema = other243.path_in_schema; + key_metadata = other243.key_metadata; + __isset = other243.__isset; } -EncryptionWithColumnKey::EncryptionWithColumnKey(EncryptionWithColumnKey&& other240) noexcept { - path_in_schema = std::move(other240.path_in_schema); - key_metadata = std::move(other240.key_metadata); - __isset = other240.__isset; +EncryptionWithColumnKey::EncryptionWithColumnKey(EncryptionWithColumnKey&& other244) noexcept { + path_in_schema = std::move(other244.path_in_schema); + key_metadata = std::move(other244.key_metadata); + __isset = other244.__isset; } -EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(const EncryptionWithColumnKey& other241) { - path_in_schema = other241.path_in_schema; - key_metadata = other241.key_metadata; - __isset = other241.__isset; +EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(const EncryptionWithColumnKey& other245) { + path_in_schema = other245.path_in_schema; + key_metadata = other245.key_metadata; + __isset = other245.__isset; return *this; } -EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(EncryptionWithColumnKey&& other242) noexcept { - path_in_schema = std::move(other242.path_in_schema); - key_metadata = std::move(other242.key_metadata); - __isset = other242.__isset; +EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(EncryptionWithColumnKey&& other246) noexcept { + path_in_schema = std::move(other246.path_in_schema); + key_metadata = std::move(other246.key_metadata); + __isset = other246.__isset; return *this; } void EncryptionWithColumnKey::printTo(std::ostream& out) const { @@ -4379,26 +4456,26 @@ bool ColumnCryptoMetaData::operator==(const ColumnCryptoMetaData & rhs) const return true; } -ColumnCryptoMetaData::ColumnCryptoMetaData(const ColumnCryptoMetaData& other243) { - ENCRYPTION_WITH_FOOTER_KEY = other243.ENCRYPTION_WITH_FOOTER_KEY; - ENCRYPTION_WITH_COLUMN_KEY = other243.ENCRYPTION_WITH_COLUMN_KEY; - __isset = other243.__isset; +ColumnCryptoMetaData::ColumnCryptoMetaData(const ColumnCryptoMetaData& other247) { + ENCRYPTION_WITH_FOOTER_KEY = other247.ENCRYPTION_WITH_FOOTER_KEY; + ENCRYPTION_WITH_COLUMN_KEY = other247.ENCRYPTION_WITH_COLUMN_KEY; + __isset = other247.__isset; } -ColumnCryptoMetaData::ColumnCryptoMetaData(ColumnCryptoMetaData&& other244) noexcept { - ENCRYPTION_WITH_FOOTER_KEY = std::move(other244.ENCRYPTION_WITH_FOOTER_KEY); - ENCRYPTION_WITH_COLUMN_KEY = std::move(other244.ENCRYPTION_WITH_COLUMN_KEY); - __isset = other244.__isset; +ColumnCryptoMetaData::ColumnCryptoMetaData(ColumnCryptoMetaData&& other248) noexcept { + ENCRYPTION_WITH_FOOTER_KEY = std::move(other248.ENCRYPTION_WITH_FOOTER_KEY); + ENCRYPTION_WITH_COLUMN_KEY = std::move(other248.ENCRYPTION_WITH_COLUMN_KEY); + __isset = other248.__isset; } -ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(const ColumnCryptoMetaData& other245) { - ENCRYPTION_WITH_FOOTER_KEY = other245.ENCRYPTION_WITH_FOOTER_KEY; - ENCRYPTION_WITH_COLUMN_KEY = other245.ENCRYPTION_WITH_COLUMN_KEY; - __isset = other245.__isset; +ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(const ColumnCryptoMetaData& other249) { + ENCRYPTION_WITH_FOOTER_KEY = other249.ENCRYPTION_WITH_FOOTER_KEY; + ENCRYPTION_WITH_COLUMN_KEY = other249.ENCRYPTION_WITH_COLUMN_KEY; + __isset = other249.__isset; return *this; } -ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(ColumnCryptoMetaData&& other246) noexcept { - ENCRYPTION_WITH_FOOTER_KEY = std::move(other246.ENCRYPTION_WITH_FOOTER_KEY); - ENCRYPTION_WITH_COLUMN_KEY = std::move(other246.ENCRYPTION_WITH_COLUMN_KEY); - __isset = other246.__isset; +ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(ColumnCryptoMetaData&& other250) noexcept { + ENCRYPTION_WITH_FOOTER_KEY = std::move(other250.ENCRYPTION_WITH_FOOTER_KEY); + ENCRYPTION_WITH_COLUMN_KEY = std::move(other250.ENCRYPTION_WITH_COLUMN_KEY); + __isset = other250.__isset; return *this; } void ColumnCryptoMetaData::printTo(std::ostream& out) const { @@ -4526,54 +4603,54 @@ bool ColumnChunk::operator==(const ColumnChunk & rhs) const return true; } -ColumnChunk::ColumnChunk(const ColumnChunk& other247) { - file_path = other247.file_path; - file_offset = other247.file_offset; - meta_data = other247.meta_data; - offset_index_offset = other247.offset_index_offset; - offset_index_length = other247.offset_index_length; - column_index_offset = other247.column_index_offset; - column_index_length = other247.column_index_length; - crypto_metadata = other247.crypto_metadata; - encrypted_column_metadata = other247.encrypted_column_metadata; - __isset = other247.__isset; -} -ColumnChunk::ColumnChunk(ColumnChunk&& other248) noexcept { - file_path = std::move(other248.file_path); - file_offset = other248.file_offset; - meta_data = std::move(other248.meta_data); - offset_index_offset = other248.offset_index_offset; - offset_index_length = other248.offset_index_length; - column_index_offset = other248.column_index_offset; - column_index_length = other248.column_index_length; - crypto_metadata = std::move(other248.crypto_metadata); - encrypted_column_metadata = std::move(other248.encrypted_column_metadata); - __isset = other248.__isset; -} -ColumnChunk& ColumnChunk::operator=(const ColumnChunk& other249) { - file_path = other249.file_path; - file_offset = other249.file_offset; - meta_data = other249.meta_data; - offset_index_offset = other249.offset_index_offset; - offset_index_length = other249.offset_index_length; - column_index_offset = other249.column_index_offset; - column_index_length = other249.column_index_length; - crypto_metadata = other249.crypto_metadata; - encrypted_column_metadata = other249.encrypted_column_metadata; - __isset = other249.__isset; +ColumnChunk::ColumnChunk(const ColumnChunk& other251) { + file_path = other251.file_path; + file_offset = other251.file_offset; + meta_data = other251.meta_data; + offset_index_offset = other251.offset_index_offset; + offset_index_length = other251.offset_index_length; + column_index_offset = other251.column_index_offset; + column_index_length = other251.column_index_length; + crypto_metadata = other251.crypto_metadata; + encrypted_column_metadata = other251.encrypted_column_metadata; + __isset = other251.__isset; +} +ColumnChunk::ColumnChunk(ColumnChunk&& other252) noexcept { + file_path = std::move(other252.file_path); + file_offset = other252.file_offset; + meta_data = std::move(other252.meta_data); + offset_index_offset = other252.offset_index_offset; + offset_index_length = other252.offset_index_length; + column_index_offset = other252.column_index_offset; + column_index_length = other252.column_index_length; + crypto_metadata = std::move(other252.crypto_metadata); + encrypted_column_metadata = std::move(other252.encrypted_column_metadata); + __isset = other252.__isset; +} +ColumnChunk& ColumnChunk::operator=(const ColumnChunk& other253) { + file_path = other253.file_path; + file_offset = other253.file_offset; + meta_data = other253.meta_data; + offset_index_offset = other253.offset_index_offset; + offset_index_length = other253.offset_index_length; + column_index_offset = other253.column_index_offset; + column_index_length = other253.column_index_length; + crypto_metadata = other253.crypto_metadata; + encrypted_column_metadata = other253.encrypted_column_metadata; + __isset = other253.__isset; return *this; } -ColumnChunk& ColumnChunk::operator=(ColumnChunk&& other250) noexcept { - file_path = std::move(other250.file_path); - file_offset = other250.file_offset; - meta_data = std::move(other250.meta_data); - offset_index_offset = other250.offset_index_offset; - offset_index_length = other250.offset_index_length; - column_index_offset = other250.column_index_offset; - column_index_length = other250.column_index_length; - crypto_metadata = std::move(other250.crypto_metadata); - encrypted_column_metadata = std::move(other250.encrypted_column_metadata); - __isset = other250.__isset; +ColumnChunk& ColumnChunk::operator=(ColumnChunk&& other254) noexcept { + file_path = std::move(other254.file_path); + file_offset = other254.file_offset; + meta_data = std::move(other254.meta_data); + offset_index_offset = other254.offset_index_offset; + offset_index_length = other254.offset_index_length; + column_index_offset = other254.column_index_offset; + column_index_length = other254.column_index_length; + crypto_metadata = std::move(other254.crypto_metadata); + encrypted_column_metadata = std::move(other254.encrypted_column_metadata); + __isset = other254.__isset; return *this; } void ColumnChunk::printTo(std::ostream& out) const { @@ -4680,46 +4757,46 @@ bool RowGroup::operator==(const RowGroup & rhs) const return true; } -RowGroup::RowGroup(const RowGroup& other263) { - columns = other263.columns; - total_byte_size = other263.total_byte_size; - num_rows = other263.num_rows; - sorting_columns = other263.sorting_columns; - file_offset = other263.file_offset; - total_compressed_size = other263.total_compressed_size; - ordinal = other263.ordinal; - __isset = other263.__isset; -} -RowGroup::RowGroup(RowGroup&& other264) noexcept { - columns = std::move(other264.columns); - total_byte_size = other264.total_byte_size; - num_rows = other264.num_rows; - sorting_columns = std::move(other264.sorting_columns); - file_offset = other264.file_offset; - total_compressed_size = other264.total_compressed_size; - ordinal = other264.ordinal; - __isset = other264.__isset; -} -RowGroup& RowGroup::operator=(const RowGroup& other265) { - columns = other265.columns; - total_byte_size = other265.total_byte_size; - num_rows = other265.num_rows; - sorting_columns = other265.sorting_columns; - file_offset = other265.file_offset; - total_compressed_size = other265.total_compressed_size; - ordinal = other265.ordinal; - __isset = other265.__isset; +RowGroup::RowGroup(const RowGroup& other267) { + columns = other267.columns; + total_byte_size = other267.total_byte_size; + num_rows = other267.num_rows; + sorting_columns = other267.sorting_columns; + file_offset = other267.file_offset; + total_compressed_size = other267.total_compressed_size; + ordinal = other267.ordinal; + __isset = other267.__isset; +} +RowGroup::RowGroup(RowGroup&& other268) noexcept { + columns = std::move(other268.columns); + total_byte_size = other268.total_byte_size; + num_rows = other268.num_rows; + sorting_columns = std::move(other268.sorting_columns); + file_offset = other268.file_offset; + total_compressed_size = other268.total_compressed_size; + ordinal = other268.ordinal; + __isset = other268.__isset; +} +RowGroup& RowGroup::operator=(const RowGroup& other269) { + columns = other269.columns; + total_byte_size = other269.total_byte_size; + num_rows = other269.num_rows; + sorting_columns = other269.sorting_columns; + file_offset = other269.file_offset; + total_compressed_size = other269.total_compressed_size; + ordinal = other269.ordinal; + __isset = other269.__isset; return *this; } -RowGroup& RowGroup::operator=(RowGroup&& other266) noexcept { - columns = std::move(other266.columns); - total_byte_size = other266.total_byte_size; - num_rows = other266.num_rows; - sorting_columns = std::move(other266.sorting_columns); - file_offset = other266.file_offset; - total_compressed_size = other266.total_compressed_size; - ordinal = other266.ordinal; - __isset = other266.__isset; +RowGroup& RowGroup::operator=(RowGroup&& other270) noexcept { + columns = std::move(other270.columns); + total_byte_size = other270.total_byte_size; + num_rows = other270.num_rows; + sorting_columns = std::move(other270.sorting_columns); + file_offset = other270.file_offset; + total_compressed_size = other270.total_compressed_size; + ordinal = other270.ordinal; + __isset = other270.__isset; return *this; } void RowGroup::printTo(std::ostream& out) const { @@ -4759,18 +4836,18 @@ bool TypeDefinedOrder::operator==(const TypeDefinedOrder & /* rhs */) const return true; } -TypeDefinedOrder::TypeDefinedOrder(const TypeDefinedOrder& other267) noexcept { - (void) other267; +TypeDefinedOrder::TypeDefinedOrder(const TypeDefinedOrder& other271) noexcept { + (void) other271; } -TypeDefinedOrder::TypeDefinedOrder(TypeDefinedOrder&& other268) noexcept { - (void) other268; +TypeDefinedOrder::TypeDefinedOrder(TypeDefinedOrder&& other272) noexcept { + (void) other272; } -TypeDefinedOrder& TypeDefinedOrder::operator=(const TypeDefinedOrder& other269) noexcept { - (void) other269; +TypeDefinedOrder& TypeDefinedOrder::operator=(const TypeDefinedOrder& other273) noexcept { + (void) other273; return *this; } -TypeDefinedOrder& TypeDefinedOrder::operator=(TypeDefinedOrder&& other270) noexcept { - (void) other270; +TypeDefinedOrder& TypeDefinedOrder::operator=(TypeDefinedOrder&& other274) noexcept { + (void) other274; return *this; } void TypeDefinedOrder::printTo(std::ostream& out) const { @@ -4803,18 +4880,18 @@ bool IEEE754TotalOrder::operator==(const IEEE754TotalOrder & /* rhs */) const return true; } -IEEE754TotalOrder::IEEE754TotalOrder(const IEEE754TotalOrder& other271) noexcept { - (void) other271; +IEEE754TotalOrder::IEEE754TotalOrder(const IEEE754TotalOrder& other275) noexcept { + (void) other275; } -IEEE754TotalOrder::IEEE754TotalOrder(IEEE754TotalOrder&& other272) noexcept { - (void) other272; +IEEE754TotalOrder::IEEE754TotalOrder(IEEE754TotalOrder&& other276) noexcept { + (void) other276; } -IEEE754TotalOrder& IEEE754TotalOrder::operator=(const IEEE754TotalOrder& other273) noexcept { - (void) other273; +IEEE754TotalOrder& IEEE754TotalOrder::operator=(const IEEE754TotalOrder& other277) noexcept { + (void) other277; return *this; } -IEEE754TotalOrder& IEEE754TotalOrder::operator=(IEEE754TotalOrder&& other274) noexcept { - (void) other274; +IEEE754TotalOrder& IEEE754TotalOrder::operator=(IEEE754TotalOrder&& other278) noexcept { + (void) other278; return *this; } void IEEE754TotalOrder::printTo(std::ostream& out) const { @@ -4866,26 +4943,26 @@ bool ColumnOrder::operator==(const ColumnOrder & rhs) const return true; } -ColumnOrder::ColumnOrder(const ColumnOrder& other275) noexcept { - TYPE_ORDER = other275.TYPE_ORDER; - IEEE_754_TOTAL_ORDER = other275.IEEE_754_TOTAL_ORDER; - __isset = other275.__isset; +ColumnOrder::ColumnOrder(const ColumnOrder& other279) noexcept { + TYPE_ORDER = other279.TYPE_ORDER; + IEEE_754_TOTAL_ORDER = other279.IEEE_754_TOTAL_ORDER; + __isset = other279.__isset; } -ColumnOrder::ColumnOrder(ColumnOrder&& other276) noexcept { - TYPE_ORDER = std::move(other276.TYPE_ORDER); - IEEE_754_TOTAL_ORDER = std::move(other276.IEEE_754_TOTAL_ORDER); - __isset = other276.__isset; +ColumnOrder::ColumnOrder(ColumnOrder&& other280) noexcept { + TYPE_ORDER = std::move(other280.TYPE_ORDER); + IEEE_754_TOTAL_ORDER = std::move(other280.IEEE_754_TOTAL_ORDER); + __isset = other280.__isset; } -ColumnOrder& ColumnOrder::operator=(const ColumnOrder& other277) noexcept { - TYPE_ORDER = other277.TYPE_ORDER; - IEEE_754_TOTAL_ORDER = other277.IEEE_754_TOTAL_ORDER; - __isset = other277.__isset; +ColumnOrder& ColumnOrder::operator=(const ColumnOrder& other281) noexcept { + TYPE_ORDER = other281.TYPE_ORDER; + IEEE_754_TOTAL_ORDER = other281.IEEE_754_TOTAL_ORDER; + __isset = other281.__isset; return *this; } -ColumnOrder& ColumnOrder::operator=(ColumnOrder&& other278) noexcept { - TYPE_ORDER = std::move(other278.TYPE_ORDER); - IEEE_754_TOTAL_ORDER = std::move(other278.IEEE_754_TOTAL_ORDER); - __isset = other278.__isset; +ColumnOrder& ColumnOrder::operator=(ColumnOrder&& other282) noexcept { + TYPE_ORDER = std::move(other282.TYPE_ORDER); + IEEE_754_TOTAL_ORDER = std::move(other282.IEEE_754_TOTAL_ORDER); + __isset = other282.__isset; return *this; } void ColumnOrder::printTo(std::ostream& out) const { @@ -4942,26 +5019,26 @@ bool PageLocation::operator==(const PageLocation & rhs) const return true; } -PageLocation::PageLocation(const PageLocation& other279) noexcept { - offset = other279.offset; - compressed_page_size = other279.compressed_page_size; - first_row_index = other279.first_row_index; +PageLocation::PageLocation(const PageLocation& other283) noexcept { + offset = other283.offset; + compressed_page_size = other283.compressed_page_size; + first_row_index = other283.first_row_index; } -PageLocation::PageLocation(PageLocation&& other280) noexcept { - offset = other280.offset; - compressed_page_size = other280.compressed_page_size; - first_row_index = other280.first_row_index; +PageLocation::PageLocation(PageLocation&& other284) noexcept { + offset = other284.offset; + compressed_page_size = other284.compressed_page_size; + first_row_index = other284.first_row_index; } -PageLocation& PageLocation::operator=(const PageLocation& other281) noexcept { - offset = other281.offset; - compressed_page_size = other281.compressed_page_size; - first_row_index = other281.first_row_index; +PageLocation& PageLocation::operator=(const PageLocation& other285) noexcept { + offset = other285.offset; + compressed_page_size = other285.compressed_page_size; + first_row_index = other285.first_row_index; return *this; } -PageLocation& PageLocation::operator=(PageLocation&& other282) noexcept { - offset = other282.offset; - compressed_page_size = other282.compressed_page_size; - first_row_index = other282.first_row_index; +PageLocation& PageLocation::operator=(PageLocation&& other286) noexcept { + offset = other286.offset; + compressed_page_size = other286.compressed_page_size; + first_row_index = other286.first_row_index; return *this; } void PageLocation::printTo(std::ostream& out) const { @@ -5013,26 +5090,26 @@ bool OffsetIndex::operator==(const OffsetIndex & rhs) const return true; } -OffsetIndex::OffsetIndex(const OffsetIndex& other295) { - page_locations = other295.page_locations; - unencoded_byte_array_data_bytes = other295.unencoded_byte_array_data_bytes; - __isset = other295.__isset; +OffsetIndex::OffsetIndex(const OffsetIndex& other299) { + page_locations = other299.page_locations; + unencoded_byte_array_data_bytes = other299.unencoded_byte_array_data_bytes; + __isset = other299.__isset; } -OffsetIndex::OffsetIndex(OffsetIndex&& other296) noexcept { - page_locations = std::move(other296.page_locations); - unencoded_byte_array_data_bytes = std::move(other296.unencoded_byte_array_data_bytes); - __isset = other296.__isset; +OffsetIndex::OffsetIndex(OffsetIndex&& other300) noexcept { + page_locations = std::move(other300.page_locations); + unencoded_byte_array_data_bytes = std::move(other300.unencoded_byte_array_data_bytes); + __isset = other300.__isset; } -OffsetIndex& OffsetIndex::operator=(const OffsetIndex& other297) { - page_locations = other297.page_locations; - unencoded_byte_array_data_bytes = other297.unencoded_byte_array_data_bytes; - __isset = other297.__isset; +OffsetIndex& OffsetIndex::operator=(const OffsetIndex& other301) { + page_locations = other301.page_locations; + unencoded_byte_array_data_bytes = other301.unencoded_byte_array_data_bytes; + __isset = other301.__isset; return *this; } -OffsetIndex& OffsetIndex::operator=(OffsetIndex&& other298) noexcept { - page_locations = std::move(other298.page_locations); - unencoded_byte_array_data_bytes = std::move(other298.unencoded_byte_array_data_bytes); - __isset = other298.__isset; +OffsetIndex& OffsetIndex::operator=(OffsetIndex&& other302) noexcept { + page_locations = std::move(other302.page_locations); + unencoded_byte_array_data_bytes = std::move(other302.unencoded_byte_array_data_bytes); + __isset = other302.__isset; return *this; } void OffsetIndex::printTo(std::ostream& out) const { @@ -5135,50 +5212,50 @@ bool ColumnIndex::operator==(const ColumnIndex & rhs) const return true; } -ColumnIndex::ColumnIndex(const ColumnIndex& other342) { - null_pages = other342.null_pages; - min_values = other342.min_values; - max_values = other342.max_values; - boundary_order = other342.boundary_order; - null_counts = other342.null_counts; - repetition_level_histograms = other342.repetition_level_histograms; - definition_level_histograms = other342.definition_level_histograms; - nan_counts = other342.nan_counts; - __isset = other342.__isset; -} -ColumnIndex::ColumnIndex(ColumnIndex&& other343) noexcept { - null_pages = std::move(other343.null_pages); - min_values = std::move(other343.min_values); - max_values = std::move(other343.max_values); - boundary_order = other343.boundary_order; - null_counts = std::move(other343.null_counts); - repetition_level_histograms = std::move(other343.repetition_level_histograms); - definition_level_histograms = std::move(other343.definition_level_histograms); - nan_counts = std::move(other343.nan_counts); - __isset = other343.__isset; -} -ColumnIndex& ColumnIndex::operator=(const ColumnIndex& other344) { - null_pages = other344.null_pages; - min_values = other344.min_values; - max_values = other344.max_values; - boundary_order = other344.boundary_order; - null_counts = other344.null_counts; - repetition_level_histograms = other344.repetition_level_histograms; - definition_level_histograms = other344.definition_level_histograms; - nan_counts = other344.nan_counts; - __isset = other344.__isset; +ColumnIndex::ColumnIndex(const ColumnIndex& other346) { + null_pages = other346.null_pages; + min_values = other346.min_values; + max_values = other346.max_values; + boundary_order = other346.boundary_order; + null_counts = other346.null_counts; + repetition_level_histograms = other346.repetition_level_histograms; + definition_level_histograms = other346.definition_level_histograms; + nan_counts = other346.nan_counts; + __isset = other346.__isset; +} +ColumnIndex::ColumnIndex(ColumnIndex&& other347) noexcept { + null_pages = std::move(other347.null_pages); + min_values = std::move(other347.min_values); + max_values = std::move(other347.max_values); + boundary_order = other347.boundary_order; + null_counts = std::move(other347.null_counts); + repetition_level_histograms = std::move(other347.repetition_level_histograms); + definition_level_histograms = std::move(other347.definition_level_histograms); + nan_counts = std::move(other347.nan_counts); + __isset = other347.__isset; +} +ColumnIndex& ColumnIndex::operator=(const ColumnIndex& other348) { + null_pages = other348.null_pages; + min_values = other348.min_values; + max_values = other348.max_values; + boundary_order = other348.boundary_order; + null_counts = other348.null_counts; + repetition_level_histograms = other348.repetition_level_histograms; + definition_level_histograms = other348.definition_level_histograms; + nan_counts = other348.nan_counts; + __isset = other348.__isset; return *this; } -ColumnIndex& ColumnIndex::operator=(ColumnIndex&& other345) noexcept { - null_pages = std::move(other345.null_pages); - min_values = std::move(other345.min_values); - max_values = std::move(other345.max_values); - boundary_order = other345.boundary_order; - null_counts = std::move(other345.null_counts); - repetition_level_histograms = std::move(other345.repetition_level_histograms); - definition_level_histograms = std::move(other345.definition_level_histograms); - nan_counts = std::move(other345.nan_counts); - __isset = other345.__isset; +ColumnIndex& ColumnIndex::operator=(ColumnIndex&& other349) noexcept { + null_pages = std::move(other349.null_pages); + min_values = std::move(other349.min_values); + max_values = std::move(other349.max_values); + boundary_order = other349.boundary_order; + null_counts = std::move(other349.null_counts); + repetition_level_histograms = std::move(other349.repetition_level_histograms); + definition_level_histograms = std::move(other349.definition_level_histograms); + nan_counts = std::move(other349.nan_counts); + __isset = other349.__isset; return *this; } void ColumnIndex::printTo(std::ostream& out) const { @@ -5251,30 +5328,30 @@ bool AesGcmV1::operator==(const AesGcmV1 & rhs) const return true; } -AesGcmV1::AesGcmV1(const AesGcmV1& other346) { - aad_prefix = other346.aad_prefix; - aad_file_unique = other346.aad_file_unique; - supply_aad_prefix = other346.supply_aad_prefix; - __isset = other346.__isset; +AesGcmV1::AesGcmV1(const AesGcmV1& other350) { + aad_prefix = other350.aad_prefix; + aad_file_unique = other350.aad_file_unique; + supply_aad_prefix = other350.supply_aad_prefix; + __isset = other350.__isset; } -AesGcmV1::AesGcmV1(AesGcmV1&& other347) noexcept { - aad_prefix = std::move(other347.aad_prefix); - aad_file_unique = std::move(other347.aad_file_unique); - supply_aad_prefix = other347.supply_aad_prefix; - __isset = other347.__isset; +AesGcmV1::AesGcmV1(AesGcmV1&& other351) noexcept { + aad_prefix = std::move(other351.aad_prefix); + aad_file_unique = std::move(other351.aad_file_unique); + supply_aad_prefix = other351.supply_aad_prefix; + __isset = other351.__isset; } -AesGcmV1& AesGcmV1::operator=(const AesGcmV1& other348) { - aad_prefix = other348.aad_prefix; - aad_file_unique = other348.aad_file_unique; - supply_aad_prefix = other348.supply_aad_prefix; - __isset = other348.__isset; +AesGcmV1& AesGcmV1::operator=(const AesGcmV1& other352) { + aad_prefix = other352.aad_prefix; + aad_file_unique = other352.aad_file_unique; + supply_aad_prefix = other352.supply_aad_prefix; + __isset = other352.__isset; return *this; } -AesGcmV1& AesGcmV1::operator=(AesGcmV1&& other349) noexcept { - aad_prefix = std::move(other349.aad_prefix); - aad_file_unique = std::move(other349.aad_file_unique); - supply_aad_prefix = other349.supply_aad_prefix; - __isset = other349.__isset; +AesGcmV1& AesGcmV1::operator=(AesGcmV1&& other353) noexcept { + aad_prefix = std::move(other353.aad_prefix); + aad_file_unique = std::move(other353.aad_file_unique); + supply_aad_prefix = other353.supply_aad_prefix; + __isset = other353.__isset; return *this; } void AesGcmV1::printTo(std::ostream& out) const { @@ -5342,30 +5419,30 @@ bool AesGcmCtrV1::operator==(const AesGcmCtrV1 & rhs) const return true; } -AesGcmCtrV1::AesGcmCtrV1(const AesGcmCtrV1& other350) { - aad_prefix = other350.aad_prefix; - aad_file_unique = other350.aad_file_unique; - supply_aad_prefix = other350.supply_aad_prefix; - __isset = other350.__isset; +AesGcmCtrV1::AesGcmCtrV1(const AesGcmCtrV1& other354) { + aad_prefix = other354.aad_prefix; + aad_file_unique = other354.aad_file_unique; + supply_aad_prefix = other354.supply_aad_prefix; + __isset = other354.__isset; } -AesGcmCtrV1::AesGcmCtrV1(AesGcmCtrV1&& other351) noexcept { - aad_prefix = std::move(other351.aad_prefix); - aad_file_unique = std::move(other351.aad_file_unique); - supply_aad_prefix = other351.supply_aad_prefix; - __isset = other351.__isset; +AesGcmCtrV1::AesGcmCtrV1(AesGcmCtrV1&& other355) noexcept { + aad_prefix = std::move(other355.aad_prefix); + aad_file_unique = std::move(other355.aad_file_unique); + supply_aad_prefix = other355.supply_aad_prefix; + __isset = other355.__isset; } -AesGcmCtrV1& AesGcmCtrV1::operator=(const AesGcmCtrV1& other352) { - aad_prefix = other352.aad_prefix; - aad_file_unique = other352.aad_file_unique; - supply_aad_prefix = other352.supply_aad_prefix; - __isset = other352.__isset; +AesGcmCtrV1& AesGcmCtrV1::operator=(const AesGcmCtrV1& other356) { + aad_prefix = other356.aad_prefix; + aad_file_unique = other356.aad_file_unique; + supply_aad_prefix = other356.supply_aad_prefix; + __isset = other356.__isset; return *this; } -AesGcmCtrV1& AesGcmCtrV1::operator=(AesGcmCtrV1&& other353) noexcept { - aad_prefix = std::move(other353.aad_prefix); - aad_file_unique = std::move(other353.aad_file_unique); - supply_aad_prefix = other353.supply_aad_prefix; - __isset = other353.__isset; +AesGcmCtrV1& AesGcmCtrV1::operator=(AesGcmCtrV1&& other357) noexcept { + aad_prefix = std::move(other357.aad_prefix); + aad_file_unique = std::move(other357.aad_file_unique); + supply_aad_prefix = other357.supply_aad_prefix; + __isset = other357.__isset; return *this; } void AesGcmCtrV1::printTo(std::ostream& out) const { @@ -5420,26 +5497,26 @@ bool EncryptionAlgorithm::operator==(const EncryptionAlgorithm & rhs) const return true; } -EncryptionAlgorithm::EncryptionAlgorithm(const EncryptionAlgorithm& other354) { - AES_GCM_V1 = other354.AES_GCM_V1; - AES_GCM_CTR_V1 = other354.AES_GCM_CTR_V1; - __isset = other354.__isset; +EncryptionAlgorithm::EncryptionAlgorithm(const EncryptionAlgorithm& other358) { + AES_GCM_V1 = other358.AES_GCM_V1; + AES_GCM_CTR_V1 = other358.AES_GCM_CTR_V1; + __isset = other358.__isset; } -EncryptionAlgorithm::EncryptionAlgorithm(EncryptionAlgorithm&& other355) noexcept { - AES_GCM_V1 = std::move(other355.AES_GCM_V1); - AES_GCM_CTR_V1 = std::move(other355.AES_GCM_CTR_V1); - __isset = other355.__isset; +EncryptionAlgorithm::EncryptionAlgorithm(EncryptionAlgorithm&& other359) noexcept { + AES_GCM_V1 = std::move(other359.AES_GCM_V1); + AES_GCM_CTR_V1 = std::move(other359.AES_GCM_CTR_V1); + __isset = other359.__isset; } -EncryptionAlgorithm& EncryptionAlgorithm::operator=(const EncryptionAlgorithm& other356) { - AES_GCM_V1 = other356.AES_GCM_V1; - AES_GCM_CTR_V1 = other356.AES_GCM_CTR_V1; - __isset = other356.__isset; +EncryptionAlgorithm& EncryptionAlgorithm::operator=(const EncryptionAlgorithm& other360) { + AES_GCM_V1 = other360.AES_GCM_V1; + AES_GCM_CTR_V1 = other360.AES_GCM_CTR_V1; + __isset = other360.__isset; return *this; } -EncryptionAlgorithm& EncryptionAlgorithm::operator=(EncryptionAlgorithm&& other357) noexcept { - AES_GCM_V1 = std::move(other357.AES_GCM_V1); - AES_GCM_CTR_V1 = std::move(other357.AES_GCM_CTR_V1); - __isset = other357.__isset; +EncryptionAlgorithm& EncryptionAlgorithm::operator=(EncryptionAlgorithm&& other361) noexcept { + AES_GCM_V1 = std::move(other361.AES_GCM_V1); + AES_GCM_CTR_V1 = std::move(other361.AES_GCM_CTR_V1); + __isset = other361.__isset; return *this; } void EncryptionAlgorithm::printTo(std::ostream& out) const { @@ -5555,54 +5632,54 @@ bool FileMetaData::operator==(const FileMetaData & rhs) const return true; } -FileMetaData::FileMetaData(const FileMetaData& other382) { - version = other382.version; - schema = other382.schema; - num_rows = other382.num_rows; - row_groups = other382.row_groups; - key_value_metadata = other382.key_value_metadata; - created_by = other382.created_by; - column_orders = other382.column_orders; - encryption_algorithm = other382.encryption_algorithm; - footer_signing_key_metadata = other382.footer_signing_key_metadata; - __isset = other382.__isset; -} -FileMetaData::FileMetaData(FileMetaData&& other383) noexcept { - version = other383.version; - schema = std::move(other383.schema); - num_rows = other383.num_rows; - row_groups = std::move(other383.row_groups); - key_value_metadata = std::move(other383.key_value_metadata); - created_by = std::move(other383.created_by); - column_orders = std::move(other383.column_orders); - encryption_algorithm = std::move(other383.encryption_algorithm); - footer_signing_key_metadata = std::move(other383.footer_signing_key_metadata); - __isset = other383.__isset; -} -FileMetaData& FileMetaData::operator=(const FileMetaData& other384) { - version = other384.version; - schema = other384.schema; - num_rows = other384.num_rows; - row_groups = other384.row_groups; - key_value_metadata = other384.key_value_metadata; - created_by = other384.created_by; - column_orders = other384.column_orders; - encryption_algorithm = other384.encryption_algorithm; - footer_signing_key_metadata = other384.footer_signing_key_metadata; - __isset = other384.__isset; +FileMetaData::FileMetaData(const FileMetaData& other386) { + version = other386.version; + schema = other386.schema; + num_rows = other386.num_rows; + row_groups = other386.row_groups; + key_value_metadata = other386.key_value_metadata; + created_by = other386.created_by; + column_orders = other386.column_orders; + encryption_algorithm = other386.encryption_algorithm; + footer_signing_key_metadata = other386.footer_signing_key_metadata; + __isset = other386.__isset; +} +FileMetaData::FileMetaData(FileMetaData&& other387) noexcept { + version = other387.version; + schema = std::move(other387.schema); + num_rows = other387.num_rows; + row_groups = std::move(other387.row_groups); + key_value_metadata = std::move(other387.key_value_metadata); + created_by = std::move(other387.created_by); + column_orders = std::move(other387.column_orders); + encryption_algorithm = std::move(other387.encryption_algorithm); + footer_signing_key_metadata = std::move(other387.footer_signing_key_metadata); + __isset = other387.__isset; +} +FileMetaData& FileMetaData::operator=(const FileMetaData& other388) { + version = other388.version; + schema = other388.schema; + num_rows = other388.num_rows; + row_groups = other388.row_groups; + key_value_metadata = other388.key_value_metadata; + created_by = other388.created_by; + column_orders = other388.column_orders; + encryption_algorithm = other388.encryption_algorithm; + footer_signing_key_metadata = other388.footer_signing_key_metadata; + __isset = other388.__isset; return *this; } -FileMetaData& FileMetaData::operator=(FileMetaData&& other385) noexcept { - version = other385.version; - schema = std::move(other385.schema); - num_rows = other385.num_rows; - row_groups = std::move(other385.row_groups); - key_value_metadata = std::move(other385.key_value_metadata); - created_by = std::move(other385.created_by); - column_orders = std::move(other385.column_orders); - encryption_algorithm = std::move(other385.encryption_algorithm); - footer_signing_key_metadata = std::move(other385.footer_signing_key_metadata); - __isset = other385.__isset; +FileMetaData& FileMetaData::operator=(FileMetaData&& other389) noexcept { + version = other389.version; + schema = std::move(other389.schema); + num_rows = other389.num_rows; + row_groups = std::move(other389.row_groups); + key_value_metadata = std::move(other389.key_value_metadata); + created_by = std::move(other389.created_by); + column_orders = std::move(other389.column_orders); + encryption_algorithm = std::move(other389.encryption_algorithm); + footer_signing_key_metadata = std::move(other389.footer_signing_key_metadata); + __isset = other389.__isset; return *this; } void FileMetaData::printTo(std::ostream& out) const { @@ -5661,26 +5738,26 @@ bool FileCryptoMetaData::operator==(const FileCryptoMetaData & rhs) const return true; } -FileCryptoMetaData::FileCryptoMetaData(const FileCryptoMetaData& other386) { - encryption_algorithm = other386.encryption_algorithm; - key_metadata = other386.key_metadata; - __isset = other386.__isset; +FileCryptoMetaData::FileCryptoMetaData(const FileCryptoMetaData& other390) { + encryption_algorithm = other390.encryption_algorithm; + key_metadata = other390.key_metadata; + __isset = other390.__isset; } -FileCryptoMetaData::FileCryptoMetaData(FileCryptoMetaData&& other387) noexcept { - encryption_algorithm = std::move(other387.encryption_algorithm); - key_metadata = std::move(other387.key_metadata); - __isset = other387.__isset; +FileCryptoMetaData::FileCryptoMetaData(FileCryptoMetaData&& other391) noexcept { + encryption_algorithm = std::move(other391.encryption_algorithm); + key_metadata = std::move(other391.key_metadata); + __isset = other391.__isset; } -FileCryptoMetaData& FileCryptoMetaData::operator=(const FileCryptoMetaData& other388) { - encryption_algorithm = other388.encryption_algorithm; - key_metadata = other388.key_metadata; - __isset = other388.__isset; +FileCryptoMetaData& FileCryptoMetaData::operator=(const FileCryptoMetaData& other392) { + encryption_algorithm = other392.encryption_algorithm; + key_metadata = other392.key_metadata; + __isset = other392.__isset; return *this; } -FileCryptoMetaData& FileCryptoMetaData::operator=(FileCryptoMetaData&& other389) noexcept { - encryption_algorithm = std::move(other389.encryption_algorithm); - key_metadata = std::move(other389.key_metadata); - __isset = other389.__isset; +FileCryptoMetaData& FileCryptoMetaData::operator=(FileCryptoMetaData&& other393) noexcept { + encryption_algorithm = std::move(other393.encryption_algorithm); + key_metadata = std::move(other393.key_metadata); + __isset = other393.__isset; return *this; } void FileCryptoMetaData::printTo(std::ostream& out) const { diff --git a/cpp/src/generated/parquet_types.h b/cpp/src/generated/parquet_types.h index 7dc3ccc2de2c..7072da89c2cb 100644 --- a/cpp/src/generated/parquet_types.h +++ b/cpp/src/generated/parquet_types.h @@ -202,7 +202,8 @@ struct FieldRepetitionType { /** * The field is repeated and can contain 0 or more values */ - REPEATED = 2 + REPEATED = 2, + VECTOR = 3 }; }; @@ -418,6 +419,8 @@ class GeometryType; class GeographyType; +class VectorType; + class LogicalType; class SchemaElement; @@ -1633,8 +1636,39 @@ void swap(GeographyType &a, GeographyType &b) noexcept; std::ostream& operator<<(std::ostream& out, const GeographyType& obj); + +class VectorType { + public: + + VectorType(const VectorType&) noexcept; + VectorType(VectorType&&) noexcept; + VectorType& operator=(const VectorType&) noexcept; + VectorType& operator=(VectorType&&) noexcept; + VectorType() noexcept; + + ~VectorType() noexcept; + + bool operator == (const VectorType & /* rhs */) const; + bool operator != (const VectorType &rhs) const { + return !(*this == rhs); + } + + bool operator < (const VectorType & ) const; + + template + uint32_t read(Protocol_* iprot); + template + uint32_t write(Protocol_* oprot) const; + + void printTo(std::ostream& out) const; +}; + +void swap(VectorType &a, VectorType &b) noexcept; + +std::ostream& operator<<(std::ostream& out, const VectorType& obj); + typedef struct _LogicalType__isset { - _LogicalType__isset() : STRING(false), MAP(false), LIST(false), ENUM(false), DECIMAL(false), DATE(false), TIME(false), TIMESTAMP(false), INTEGER(false), UNKNOWN(false), JSON(false), BSON(false), UUID(false), FLOAT16(false), VARIANT(false), GEOMETRY(false), GEOGRAPHY(false) {} + _LogicalType__isset() : STRING(false), MAP(false), LIST(false), ENUM(false), DECIMAL(false), DATE(false), TIME(false), TIMESTAMP(false), INTEGER(false), UNKNOWN(false), JSON(false), BSON(false), UUID(false), FLOAT16(false), VARIANT(false), GEOMETRY(false), GEOGRAPHY(false), VECTOR(false) {} bool STRING :1; bool MAP :1; bool LIST :1; @@ -1652,6 +1686,7 @@ typedef struct _LogicalType__isset { bool VARIANT :1; bool GEOMETRY :1; bool GEOGRAPHY :1; + bool VECTOR :1; } _LogicalType__isset; /** @@ -1688,6 +1723,7 @@ class LogicalType { VariantType VARIANT; GeometryType GEOMETRY; GeographyType GEOGRAPHY; + VectorType VECTOR; _LogicalType__isset __isset; @@ -1725,6 +1761,8 @@ class LogicalType { void __set_GEOGRAPHY(const GeographyType& val); + void __set_VECTOR(const VectorType& val); + bool operator == (const LogicalType & rhs) const; bool operator != (const LogicalType &rhs) const { return !(*this == rhs); @@ -1745,7 +1783,7 @@ void swap(LogicalType &a, LogicalType &b) noexcept; std::ostream& operator<<(std::ostream& out, const LogicalType& obj); typedef struct _SchemaElement__isset { - _SchemaElement__isset() : type(false), type_length(false), repetition_type(false), num_children(false), converted_type(false), scale(false), precision(false), field_id(false), logicalType(false) {} + _SchemaElement__isset() : type(false), type_length(false), repetition_type(false), num_children(false), converted_type(false), scale(false), precision(false), field_id(false), logicalType(false), vector_length(false) {} bool type :1; bool type_length :1; bool repetition_type :1; @@ -1755,6 +1793,7 @@ typedef struct _SchemaElement__isset { bool precision :1; bool field_id :1; bool logicalType :1; + bool vector_length :1; } _SchemaElement__isset; /** @@ -1833,6 +1872,7 @@ class SchemaElement { * for some logical types to ensure forward-compatibility in format v1. */ LogicalType logicalType; + int32_t vector_length; _SchemaElement__isset __isset; @@ -1856,6 +1896,8 @@ class SchemaElement { void __set_logicalType(const LogicalType& val); + void __set_vector_length(const int32_t val); + bool operator == (const SchemaElement & rhs) const; bool operator != (const SchemaElement &rhs) const { return !(*this == rhs); diff --git a/cpp/src/generated/parquet_types.tcc b/cpp/src/generated/parquet_types.tcc index 01559f897372..46dab3d15a4b 100644 --- a/cpp/src/generated/parquet_types.tcc +++ b/cpp/src/generated/parquet_types.tcc @@ -1638,6 +1638,46 @@ uint32_t GeographyType::write(Protocol_* oprot) const { return xfer; } +template +uint32_t VectorType::read(Protocol_* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + xfer += iprot->skip(ftype); + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +template +uint32_t VectorType::write(Protocol_* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("VectorType"); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + template uint32_t LogicalType::read(Protocol_* iprot) { @@ -1796,6 +1836,14 @@ uint32_t LogicalType::read(Protocol_* iprot) { xfer += iprot->skip(ftype); } break; + case 19: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->VECTOR.read(iprot); + this->__isset.VECTOR = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -1899,6 +1947,11 @@ uint32_t LogicalType::write(Protocol_* oprot) const { xfer += this->GEOGRAPHY.write(oprot); xfer += oprot->writeFieldEnd(); } + if (this->__isset.VECTOR) { + xfer += oprot->writeFieldBegin("VECTOR", ::apache::thrift::protocol::T_STRUCT, 19); + xfer += this->VECTOR.write(oprot); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -1929,9 +1982,9 @@ uint32_t SchemaElement::read(Protocol_* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast123; - xfer += iprot->readI32(ecast123); - this->type = static_cast(ecast123); + int32_t ecast127; + xfer += iprot->readI32(ecast127); + this->type = static_cast(ecast127); this->__isset.type = true; } else { xfer += iprot->skip(ftype); @@ -1947,9 +2000,9 @@ uint32_t SchemaElement::read(Protocol_* iprot) { break; case 3: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast124; - xfer += iprot->readI32(ecast124); - this->repetition_type = static_cast(ecast124); + int32_t ecast128; + xfer += iprot->readI32(ecast128); + this->repetition_type = static_cast(ecast128); this->__isset.repetition_type = true; } else { xfer += iprot->skip(ftype); @@ -1973,9 +2026,9 @@ uint32_t SchemaElement::read(Protocol_* iprot) { break; case 6: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast125; - xfer += iprot->readI32(ecast125); - this->converted_type = static_cast(ecast125); + int32_t ecast129; + xfer += iprot->readI32(ecast129); + this->converted_type = static_cast(ecast129); this->__isset.converted_type = true; } else { xfer += iprot->skip(ftype); @@ -2013,6 +2066,14 @@ uint32_t SchemaElement::read(Protocol_* iprot) { xfer += iprot->skip(ftype); } break; + case 11: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->vector_length); + this->__isset.vector_length = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -2082,6 +2143,11 @@ uint32_t SchemaElement::write(Protocol_* oprot) const { xfer += this->logicalType.write(oprot); xfer += oprot->writeFieldEnd(); } + if (this->__isset.vector_length) { + xfer += oprot->writeFieldBegin("vector_length", ::apache::thrift::protocol::T_I32, 11); + xfer += oprot->writeI32(this->vector_length); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -2123,9 +2189,9 @@ uint32_t DataPageHeader::read(Protocol_* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast130; - xfer += iprot->readI32(ecast130); - this->encoding = static_cast(ecast130); + int32_t ecast134; + xfer += iprot->readI32(ecast134); + this->encoding = static_cast(ecast134); isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2133,9 +2199,9 @@ uint32_t DataPageHeader::read(Protocol_* iprot) { break; case 3: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast131; - xfer += iprot->readI32(ecast131); - this->definition_level_encoding = static_cast(ecast131); + int32_t ecast135; + xfer += iprot->readI32(ecast135); + this->definition_level_encoding = static_cast(ecast135); isset_definition_level_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2143,9 +2209,9 @@ uint32_t DataPageHeader::read(Protocol_* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast132; - xfer += iprot->readI32(ecast132); - this->repetition_level_encoding = static_cast(ecast132); + int32_t ecast136; + xfer += iprot->readI32(ecast136); + this->repetition_level_encoding = static_cast(ecast136); isset_repetition_level_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2285,9 +2351,9 @@ uint32_t DictionaryPageHeader::read(Protocol_* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast141; - xfer += iprot->readI32(ecast141); - this->encoding = static_cast(ecast141); + int32_t ecast145; + xfer += iprot->readI32(ecast145); + this->encoding = static_cast(ecast145); isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2395,9 +2461,9 @@ uint32_t DataPageHeaderV2::read(Protocol_* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast146; - xfer += iprot->readI32(ecast146); - this->encoding = static_cast(ecast146); + int32_t ecast150; + xfer += iprot->readI32(ecast150); + this->encoding = static_cast(ecast150); isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2930,9 +2996,9 @@ uint32_t PageHeader::read(Protocol_* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast179; - xfer += iprot->readI32(ecast179); - this->type = static_cast(ecast179); + int32_t ecast183; + xfer += iprot->readI32(ecast183); + this->type = static_cast(ecast183); isset_type = true; } else { xfer += iprot->skip(ftype); @@ -3250,9 +3316,9 @@ uint32_t PageEncodingStats::read(Protocol_* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast192; - xfer += iprot->readI32(ecast192); - this->page_type = static_cast(ecast192); + int32_t ecast196; + xfer += iprot->readI32(ecast196); + this->page_type = static_cast(ecast196); isset_page_type = true; } else { xfer += iprot->skip(ftype); @@ -3260,9 +3326,9 @@ uint32_t PageEncodingStats::read(Protocol_* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast193; - xfer += iprot->readI32(ecast193); - this->encoding = static_cast(ecast193); + int32_t ecast197; + xfer += iprot->readI32(ecast197); + this->encoding = static_cast(ecast197); isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -3349,9 +3415,9 @@ uint32_t ColumnMetaData::read(Protocol_* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast198; - xfer += iprot->readI32(ecast198); - this->type = static_cast(ecast198); + int32_t ecast202; + xfer += iprot->readI32(ecast202); + this->type = static_cast(ecast202); isset_type = true; } else { xfer += iprot->skip(ftype); @@ -3361,16 +3427,16 @@ uint32_t ColumnMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->encodings.clear(); - uint32_t _size199; - ::apache::thrift::protocol::TType _etype202; - xfer += iprot->readListBegin(_etype202, _size199); - this->encodings.resize(_size199); - uint32_t _i203; - for (_i203 = 0; _i203 < _size199; ++_i203) + uint32_t _size203; + ::apache::thrift::protocol::TType _etype206; + xfer += iprot->readListBegin(_etype206, _size203); + this->encodings.resize(_size203); + uint32_t _i207; + for (_i207 = 0; _i207 < _size203; ++_i207) { - int32_t ecast204; - xfer += iprot->readI32(ecast204); - this->encodings[_i203] = static_cast(ecast204); + int32_t ecast208; + xfer += iprot->readI32(ecast208); + this->encodings[_i207] = static_cast(ecast208); } xfer += iprot->readListEnd(); } @@ -3383,14 +3449,14 @@ uint32_t ColumnMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->path_in_schema.clear(); - uint32_t _size205; - ::apache::thrift::protocol::TType _etype208; - xfer += iprot->readListBegin(_etype208, _size205); - this->path_in_schema.resize(_size205); - uint32_t _i209; - for (_i209 = 0; _i209 < _size205; ++_i209) + uint32_t _size209; + ::apache::thrift::protocol::TType _etype212; + xfer += iprot->readListBegin(_etype212, _size209); + this->path_in_schema.resize(_size209); + uint32_t _i213; + for (_i213 = 0; _i213 < _size209; ++_i213) { - xfer += iprot->readString(this->path_in_schema[_i209]); + xfer += iprot->readString(this->path_in_schema[_i213]); } xfer += iprot->readListEnd(); } @@ -3401,9 +3467,9 @@ uint32_t ColumnMetaData::read(Protocol_* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast210; - xfer += iprot->readI32(ecast210); - this->codec = static_cast(ecast210); + int32_t ecast214; + xfer += iprot->readI32(ecast214); + this->codec = static_cast(ecast214); isset_codec = true; } else { xfer += iprot->skip(ftype); @@ -3437,14 +3503,14 @@ uint32_t ColumnMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->key_value_metadata.clear(); - uint32_t _size211; - ::apache::thrift::protocol::TType _etype214; - xfer += iprot->readListBegin(_etype214, _size211); - this->key_value_metadata.resize(_size211); - uint32_t _i215; - for (_i215 = 0; _i215 < _size211; ++_i215) + uint32_t _size215; + ::apache::thrift::protocol::TType _etype218; + xfer += iprot->readListBegin(_etype218, _size215); + this->key_value_metadata.resize(_size215); + uint32_t _i219; + for (_i219 = 0; _i219 < _size215; ++_i219) { - xfer += this->key_value_metadata[_i215].read(iprot); + xfer += this->key_value_metadata[_i219].read(iprot); } xfer += iprot->readListEnd(); } @@ -3489,14 +3555,14 @@ uint32_t ColumnMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->encoding_stats.clear(); - uint32_t _size216; - ::apache::thrift::protocol::TType _etype219; - xfer += iprot->readListBegin(_etype219, _size216); - this->encoding_stats.resize(_size216); - uint32_t _i220; - for (_i220 = 0; _i220 < _size216; ++_i220) + uint32_t _size220; + ::apache::thrift::protocol::TType _etype223; + xfer += iprot->readListBegin(_etype223, _size220); + this->encoding_stats.resize(_size220); + uint32_t _i224; + for (_i224 = 0; _i224 < _size220; ++_i224) { - xfer += this->encoding_stats[_i220].read(iprot); + xfer += this->encoding_stats[_i224].read(iprot); } xfer += iprot->readListEnd(); } @@ -3578,10 +3644,10 @@ uint32_t ColumnMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("encodings", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I32, static_cast(this->encodings.size())); - std::vector ::const_iterator _iter221; - for (_iter221 = this->encodings.begin(); _iter221 != this->encodings.end(); ++_iter221) + std::vector ::const_iterator _iter225; + for (_iter225 = this->encodings.begin(); _iter225 != this->encodings.end(); ++_iter225) { - xfer += oprot->writeI32(static_cast((*_iter221))); + xfer += oprot->writeI32(static_cast((*_iter225))); } xfer += oprot->writeListEnd(); } @@ -3590,10 +3656,10 @@ uint32_t ColumnMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("path_in_schema", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->path_in_schema.size())); - std::vector ::const_iterator _iter222; - for (_iter222 = this->path_in_schema.begin(); _iter222 != this->path_in_schema.end(); ++_iter222) + std::vector ::const_iterator _iter226; + for (_iter226 = this->path_in_schema.begin(); _iter226 != this->path_in_schema.end(); ++_iter226) { - xfer += oprot->writeString((*_iter222)); + xfer += oprot->writeString((*_iter226)); } xfer += oprot->writeListEnd(); } @@ -3619,10 +3685,10 @@ uint32_t ColumnMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("key_value_metadata", ::apache::thrift::protocol::T_LIST, 8); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->key_value_metadata.size())); - std::vector ::const_iterator _iter223; - for (_iter223 = this->key_value_metadata.begin(); _iter223 != this->key_value_metadata.end(); ++_iter223) + std::vector ::const_iterator _iter227; + for (_iter227 = this->key_value_metadata.begin(); _iter227 != this->key_value_metadata.end(); ++_iter227) { - xfer += (*_iter223).write(oprot); + xfer += (*_iter227).write(oprot); } xfer += oprot->writeListEnd(); } @@ -3651,10 +3717,10 @@ uint32_t ColumnMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("encoding_stats", ::apache::thrift::protocol::T_LIST, 13); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->encoding_stats.size())); - std::vector ::const_iterator _iter224; - for (_iter224 = this->encoding_stats.begin(); _iter224 != this->encoding_stats.end(); ++_iter224) + std::vector ::const_iterator _iter228; + for (_iter228 = this->encoding_stats.begin(); _iter228 != this->encoding_stats.end(); ++_iter228) { - xfer += (*_iter224).write(oprot); + xfer += (*_iter228).write(oprot); } xfer += oprot->writeListEnd(); } @@ -3752,14 +3818,14 @@ uint32_t EncryptionWithColumnKey::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->path_in_schema.clear(); - uint32_t _size233; - ::apache::thrift::protocol::TType _etype236; - xfer += iprot->readListBegin(_etype236, _size233); - this->path_in_schema.resize(_size233); - uint32_t _i237; - for (_i237 = 0; _i237 < _size233; ++_i237) + uint32_t _size237; + ::apache::thrift::protocol::TType _etype240; + xfer += iprot->readListBegin(_etype240, _size237); + this->path_in_schema.resize(_size237); + uint32_t _i241; + for (_i241 = 0; _i241 < _size237; ++_i241) { - xfer += iprot->readString(this->path_in_schema[_i237]); + xfer += iprot->readString(this->path_in_schema[_i241]); } xfer += iprot->readListEnd(); } @@ -3799,10 +3865,10 @@ uint32_t EncryptionWithColumnKey::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("path_in_schema", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->path_in_schema.size())); - std::vector ::const_iterator _iter238; - for (_iter238 = this->path_in_schema.begin(); _iter238 != this->path_in_schema.end(); ++_iter238) + std::vector ::const_iterator _iter242; + for (_iter242 = this->path_in_schema.begin(); _iter242 != this->path_in_schema.end(); ++_iter242) { - xfer += oprot->writeString((*_iter238)); + xfer += oprot->writeString((*_iter242)); } xfer += oprot->writeListEnd(); } @@ -4082,14 +4148,14 @@ uint32_t RowGroup::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->columns.clear(); - uint32_t _size251; - ::apache::thrift::protocol::TType _etype254; - xfer += iprot->readListBegin(_etype254, _size251); - this->columns.resize(_size251); - uint32_t _i255; - for (_i255 = 0; _i255 < _size251; ++_i255) + uint32_t _size255; + ::apache::thrift::protocol::TType _etype258; + xfer += iprot->readListBegin(_etype258, _size255); + this->columns.resize(_size255); + uint32_t _i259; + for (_i259 = 0; _i259 < _size255; ++_i259) { - xfer += this->columns[_i255].read(iprot); + xfer += this->columns[_i259].read(iprot); } xfer += iprot->readListEnd(); } @@ -4118,14 +4184,14 @@ uint32_t RowGroup::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->sorting_columns.clear(); - uint32_t _size256; - ::apache::thrift::protocol::TType _etype259; - xfer += iprot->readListBegin(_etype259, _size256); - this->sorting_columns.resize(_size256); - uint32_t _i260; - for (_i260 = 0; _i260 < _size256; ++_i260) + uint32_t _size260; + ::apache::thrift::protocol::TType _etype263; + xfer += iprot->readListBegin(_etype263, _size260); + this->sorting_columns.resize(_size260); + uint32_t _i264; + for (_i264 = 0; _i264 < _size260; ++_i264) { - xfer += this->sorting_columns[_i260].read(iprot); + xfer += this->sorting_columns[_i264].read(iprot); } xfer += iprot->readListEnd(); } @@ -4185,10 +4251,10 @@ uint32_t RowGroup::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("columns", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->columns.size())); - std::vector ::const_iterator _iter261; - for (_iter261 = this->columns.begin(); _iter261 != this->columns.end(); ++_iter261) + std::vector ::const_iterator _iter265; + for (_iter265 = this->columns.begin(); _iter265 != this->columns.end(); ++_iter265) { - xfer += (*_iter261).write(oprot); + xfer += (*_iter265).write(oprot); } xfer += oprot->writeListEnd(); } @@ -4206,10 +4272,10 @@ uint32_t RowGroup::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("sorting_columns", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->sorting_columns.size())); - std::vector ::const_iterator _iter262; - for (_iter262 = this->sorting_columns.begin(); _iter262 != this->sorting_columns.end(); ++_iter262) + std::vector ::const_iterator _iter266; + for (_iter266 = this->sorting_columns.begin(); _iter266 != this->sorting_columns.end(); ++_iter266) { - xfer += (*_iter262).write(oprot); + xfer += (*_iter266).write(oprot); } xfer += oprot->writeListEnd(); } @@ -4503,14 +4569,14 @@ uint32_t OffsetIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->page_locations.clear(); - uint32_t _size283; - ::apache::thrift::protocol::TType _etype286; - xfer += iprot->readListBegin(_etype286, _size283); - this->page_locations.resize(_size283); - uint32_t _i287; - for (_i287 = 0; _i287 < _size283; ++_i287) + uint32_t _size287; + ::apache::thrift::protocol::TType _etype290; + xfer += iprot->readListBegin(_etype290, _size287); + this->page_locations.resize(_size287); + uint32_t _i291; + for (_i291 = 0; _i291 < _size287; ++_i291) { - xfer += this->page_locations[_i287].read(iprot); + xfer += this->page_locations[_i291].read(iprot); } xfer += iprot->readListEnd(); } @@ -4523,14 +4589,14 @@ uint32_t OffsetIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->unencoded_byte_array_data_bytes.clear(); - uint32_t _size288; - ::apache::thrift::protocol::TType _etype291; - xfer += iprot->readListBegin(_etype291, _size288); - this->unencoded_byte_array_data_bytes.resize(_size288); - uint32_t _i292; - for (_i292 = 0; _i292 < _size288; ++_i292) + uint32_t _size292; + ::apache::thrift::protocol::TType _etype295; + xfer += iprot->readListBegin(_etype295, _size292); + this->unencoded_byte_array_data_bytes.resize(_size292); + uint32_t _i296; + for (_i296 = 0; _i296 < _size292; ++_i296) { - xfer += iprot->readI64(this->unencoded_byte_array_data_bytes[_i292]); + xfer += iprot->readI64(this->unencoded_byte_array_data_bytes[_i296]); } xfer += iprot->readListEnd(); } @@ -4562,10 +4628,10 @@ uint32_t OffsetIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("page_locations", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->page_locations.size())); - std::vector ::const_iterator _iter293; - for (_iter293 = this->page_locations.begin(); _iter293 != this->page_locations.end(); ++_iter293) + std::vector ::const_iterator _iter297; + for (_iter297 = this->page_locations.begin(); _iter297 != this->page_locations.end(); ++_iter297) { - xfer += (*_iter293).write(oprot); + xfer += (*_iter297).write(oprot); } xfer += oprot->writeListEnd(); } @@ -4575,10 +4641,10 @@ uint32_t OffsetIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("unencoded_byte_array_data_bytes", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->unencoded_byte_array_data_bytes.size())); - std::vector ::const_iterator _iter294; - for (_iter294 = this->unencoded_byte_array_data_bytes.begin(); _iter294 != this->unencoded_byte_array_data_bytes.end(); ++_iter294) + std::vector ::const_iterator _iter298; + for (_iter298 = this->unencoded_byte_array_data_bytes.begin(); _iter298 != this->unencoded_byte_array_data_bytes.end(); ++_iter298) { - xfer += oprot->writeI64((*_iter294)); + xfer += oprot->writeI64((*_iter298)); } xfer += oprot->writeListEnd(); } @@ -4619,14 +4685,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->null_pages.clear(); - uint32_t _size299; - ::apache::thrift::protocol::TType _etype302; - xfer += iprot->readListBegin(_etype302, _size299); - this->null_pages.resize(_size299); - uint32_t _i303; - for (_i303 = 0; _i303 < _size299; ++_i303) + uint32_t _size303; + ::apache::thrift::protocol::TType _etype306; + xfer += iprot->readListBegin(_etype306, _size303); + this->null_pages.resize(_size303); + uint32_t _i307; + for (_i307 = 0; _i307 < _size303; ++_i307) { - xfer += iprot->readBool(this->null_pages[_i303]); + xfer += iprot->readBool(this->null_pages[_i307]); } xfer += iprot->readListEnd(); } @@ -4639,14 +4705,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->min_values.clear(); - uint32_t _size304; - ::apache::thrift::protocol::TType _etype307; - xfer += iprot->readListBegin(_etype307, _size304); - this->min_values.resize(_size304); - uint32_t _i308; - for (_i308 = 0; _i308 < _size304; ++_i308) + uint32_t _size308; + ::apache::thrift::protocol::TType _etype311; + xfer += iprot->readListBegin(_etype311, _size308); + this->min_values.resize(_size308); + uint32_t _i312; + for (_i312 = 0; _i312 < _size308; ++_i312) { - xfer += iprot->readBinary(this->min_values[_i308]); + xfer += iprot->readBinary(this->min_values[_i312]); } xfer += iprot->readListEnd(); } @@ -4659,14 +4725,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->max_values.clear(); - uint32_t _size309; - ::apache::thrift::protocol::TType _etype312; - xfer += iprot->readListBegin(_etype312, _size309); - this->max_values.resize(_size309); - uint32_t _i313; - for (_i313 = 0; _i313 < _size309; ++_i313) + uint32_t _size313; + ::apache::thrift::protocol::TType _etype316; + xfer += iprot->readListBegin(_etype316, _size313); + this->max_values.resize(_size313); + uint32_t _i317; + for (_i317 = 0; _i317 < _size313; ++_i317) { - xfer += iprot->readBinary(this->max_values[_i313]); + xfer += iprot->readBinary(this->max_values[_i317]); } xfer += iprot->readListEnd(); } @@ -4677,9 +4743,9 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast314; - xfer += iprot->readI32(ecast314); - this->boundary_order = static_cast(ecast314); + int32_t ecast318; + xfer += iprot->readI32(ecast318); + this->boundary_order = static_cast(ecast318); isset_boundary_order = true; } else { xfer += iprot->skip(ftype); @@ -4689,14 +4755,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->null_counts.clear(); - uint32_t _size315; - ::apache::thrift::protocol::TType _etype318; - xfer += iprot->readListBegin(_etype318, _size315); - this->null_counts.resize(_size315); - uint32_t _i319; - for (_i319 = 0; _i319 < _size315; ++_i319) + uint32_t _size319; + ::apache::thrift::protocol::TType _etype322; + xfer += iprot->readListBegin(_etype322, _size319); + this->null_counts.resize(_size319); + uint32_t _i323; + for (_i323 = 0; _i323 < _size319; ++_i323) { - xfer += iprot->readI64(this->null_counts[_i319]); + xfer += iprot->readI64(this->null_counts[_i323]); } xfer += iprot->readListEnd(); } @@ -4709,14 +4775,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->repetition_level_histograms.clear(); - uint32_t _size320; - ::apache::thrift::protocol::TType _etype323; - xfer += iprot->readListBegin(_etype323, _size320); - this->repetition_level_histograms.resize(_size320); - uint32_t _i324; - for (_i324 = 0; _i324 < _size320; ++_i324) + uint32_t _size324; + ::apache::thrift::protocol::TType _etype327; + xfer += iprot->readListBegin(_etype327, _size324); + this->repetition_level_histograms.resize(_size324); + uint32_t _i328; + for (_i328 = 0; _i328 < _size324; ++_i328) { - xfer += iprot->readI64(this->repetition_level_histograms[_i324]); + xfer += iprot->readI64(this->repetition_level_histograms[_i328]); } xfer += iprot->readListEnd(); } @@ -4729,14 +4795,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->definition_level_histograms.clear(); - uint32_t _size325; - ::apache::thrift::protocol::TType _etype328; - xfer += iprot->readListBegin(_etype328, _size325); - this->definition_level_histograms.resize(_size325); - uint32_t _i329; - for (_i329 = 0; _i329 < _size325; ++_i329) + uint32_t _size329; + ::apache::thrift::protocol::TType _etype332; + xfer += iprot->readListBegin(_etype332, _size329); + this->definition_level_histograms.resize(_size329); + uint32_t _i333; + for (_i333 = 0; _i333 < _size329; ++_i333) { - xfer += iprot->readI64(this->definition_level_histograms[_i329]); + xfer += iprot->readI64(this->definition_level_histograms[_i333]); } xfer += iprot->readListEnd(); } @@ -4749,14 +4815,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->nan_counts.clear(); - uint32_t _size330; - ::apache::thrift::protocol::TType _etype333; - xfer += iprot->readListBegin(_etype333, _size330); - this->nan_counts.resize(_size330); - uint32_t _i334; - for (_i334 = 0; _i334 < _size330; ++_i334) + uint32_t _size334; + ::apache::thrift::protocol::TType _etype337; + xfer += iprot->readListBegin(_etype337, _size334); + this->nan_counts.resize(_size334); + uint32_t _i338; + for (_i338 = 0; _i338 < _size334; ++_i338) { - xfer += iprot->readI64(this->nan_counts[_i334]); + xfer += iprot->readI64(this->nan_counts[_i338]); } xfer += iprot->readListEnd(); } @@ -4794,10 +4860,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("null_pages", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_BOOL, static_cast(this->null_pages.size())); - std::vector ::const_iterator _iter335; - for (_iter335 = this->null_pages.begin(); _iter335 != this->null_pages.end(); ++_iter335) + std::vector ::const_iterator _iter339; + for (_iter339 = this->null_pages.begin(); _iter339 != this->null_pages.end(); ++_iter339) { - xfer += oprot->writeBool((*_iter335)); + xfer += oprot->writeBool((*_iter339)); } xfer += oprot->writeListEnd(); } @@ -4806,10 +4872,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("min_values", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->min_values.size())); - std::vector ::const_iterator _iter336; - for (_iter336 = this->min_values.begin(); _iter336 != this->min_values.end(); ++_iter336) + std::vector ::const_iterator _iter340; + for (_iter340 = this->min_values.begin(); _iter340 != this->min_values.end(); ++_iter340) { - xfer += oprot->writeBinary((*_iter336)); + xfer += oprot->writeBinary((*_iter340)); } xfer += oprot->writeListEnd(); } @@ -4818,10 +4884,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("max_values", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->max_values.size())); - std::vector ::const_iterator _iter337; - for (_iter337 = this->max_values.begin(); _iter337 != this->max_values.end(); ++_iter337) + std::vector ::const_iterator _iter341; + for (_iter341 = this->max_values.begin(); _iter341 != this->max_values.end(); ++_iter341) { - xfer += oprot->writeBinary((*_iter337)); + xfer += oprot->writeBinary((*_iter341)); } xfer += oprot->writeListEnd(); } @@ -4835,10 +4901,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("null_counts", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->null_counts.size())); - std::vector ::const_iterator _iter338; - for (_iter338 = this->null_counts.begin(); _iter338 != this->null_counts.end(); ++_iter338) + std::vector ::const_iterator _iter342; + for (_iter342 = this->null_counts.begin(); _iter342 != this->null_counts.end(); ++_iter342) { - xfer += oprot->writeI64((*_iter338)); + xfer += oprot->writeI64((*_iter342)); } xfer += oprot->writeListEnd(); } @@ -4848,10 +4914,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("repetition_level_histograms", ::apache::thrift::protocol::T_LIST, 6); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->repetition_level_histograms.size())); - std::vector ::const_iterator _iter339; - for (_iter339 = this->repetition_level_histograms.begin(); _iter339 != this->repetition_level_histograms.end(); ++_iter339) + std::vector ::const_iterator _iter343; + for (_iter343 = this->repetition_level_histograms.begin(); _iter343 != this->repetition_level_histograms.end(); ++_iter343) { - xfer += oprot->writeI64((*_iter339)); + xfer += oprot->writeI64((*_iter343)); } xfer += oprot->writeListEnd(); } @@ -4861,10 +4927,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("definition_level_histograms", ::apache::thrift::protocol::T_LIST, 7); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->definition_level_histograms.size())); - std::vector ::const_iterator _iter340; - for (_iter340 = this->definition_level_histograms.begin(); _iter340 != this->definition_level_histograms.end(); ++_iter340) + std::vector ::const_iterator _iter344; + for (_iter344 = this->definition_level_histograms.begin(); _iter344 != this->definition_level_histograms.end(); ++_iter344) { - xfer += oprot->writeI64((*_iter340)); + xfer += oprot->writeI64((*_iter344)); } xfer += oprot->writeListEnd(); } @@ -4874,10 +4940,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("nan_counts", ::apache::thrift::protocol::T_LIST, 8); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->nan_counts.size())); - std::vector ::const_iterator _iter341; - for (_iter341 = this->nan_counts.begin(); _iter341 != this->nan_counts.end(); ++_iter341) + std::vector ::const_iterator _iter345; + for (_iter345 = this->nan_counts.begin(); _iter345 != this->nan_counts.end(); ++_iter345) { - xfer += oprot->writeI64((*_iter341)); + xfer += oprot->writeI64((*_iter345)); } xfer += oprot->writeListEnd(); } @@ -5165,14 +5231,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->schema.clear(); - uint32_t _size358; - ::apache::thrift::protocol::TType _etype361; - xfer += iprot->readListBegin(_etype361, _size358); - this->schema.resize(_size358); - uint32_t _i362; - for (_i362 = 0; _i362 < _size358; ++_i362) + uint32_t _size362; + ::apache::thrift::protocol::TType _etype365; + xfer += iprot->readListBegin(_etype365, _size362); + this->schema.resize(_size362); + uint32_t _i366; + for (_i366 = 0; _i366 < _size362; ++_i366) { - xfer += this->schema[_i362].read(iprot); + xfer += this->schema[_i366].read(iprot); } xfer += iprot->readListEnd(); } @@ -5193,14 +5259,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->row_groups.clear(); - uint32_t _size363; - ::apache::thrift::protocol::TType _etype366; - xfer += iprot->readListBegin(_etype366, _size363); - this->row_groups.resize(_size363); - uint32_t _i367; - for (_i367 = 0; _i367 < _size363; ++_i367) + uint32_t _size367; + ::apache::thrift::protocol::TType _etype370; + xfer += iprot->readListBegin(_etype370, _size367); + this->row_groups.resize(_size367); + uint32_t _i371; + for (_i371 = 0; _i371 < _size367; ++_i371) { - xfer += this->row_groups[_i367].read(iprot); + xfer += this->row_groups[_i371].read(iprot); } xfer += iprot->readListEnd(); } @@ -5213,14 +5279,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->key_value_metadata.clear(); - uint32_t _size368; - ::apache::thrift::protocol::TType _etype371; - xfer += iprot->readListBegin(_etype371, _size368); - this->key_value_metadata.resize(_size368); - uint32_t _i372; - for (_i372 = 0; _i372 < _size368; ++_i372) + uint32_t _size372; + ::apache::thrift::protocol::TType _etype375; + xfer += iprot->readListBegin(_etype375, _size372); + this->key_value_metadata.resize(_size372); + uint32_t _i376; + for (_i376 = 0; _i376 < _size372; ++_i376) { - xfer += this->key_value_metadata[_i372].read(iprot); + xfer += this->key_value_metadata[_i376].read(iprot); } xfer += iprot->readListEnd(); } @@ -5241,14 +5307,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->column_orders.clear(); - uint32_t _size373; - ::apache::thrift::protocol::TType _etype376; - xfer += iprot->readListBegin(_etype376, _size373); - this->column_orders.resize(_size373); - uint32_t _i377; - for (_i377 = 0; _i377 < _size373; ++_i377) + uint32_t _size377; + ::apache::thrift::protocol::TType _etype380; + xfer += iprot->readListBegin(_etype380, _size377); + this->column_orders.resize(_size377); + uint32_t _i381; + for (_i381 = 0; _i381 < _size377; ++_i381) { - xfer += this->column_orders[_i377].read(iprot); + xfer += this->column_orders[_i381].read(iprot); } xfer += iprot->readListEnd(); } @@ -5306,10 +5372,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("schema", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->schema.size())); - std::vector ::const_iterator _iter378; - for (_iter378 = this->schema.begin(); _iter378 != this->schema.end(); ++_iter378) + std::vector ::const_iterator _iter382; + for (_iter382 = this->schema.begin(); _iter382 != this->schema.end(); ++_iter382) { - xfer += (*_iter378).write(oprot); + xfer += (*_iter382).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5322,10 +5388,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("row_groups", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->row_groups.size())); - std::vector ::const_iterator _iter379; - for (_iter379 = this->row_groups.begin(); _iter379 != this->row_groups.end(); ++_iter379) + std::vector ::const_iterator _iter383; + for (_iter383 = this->row_groups.begin(); _iter383 != this->row_groups.end(); ++_iter383) { - xfer += (*_iter379).write(oprot); + xfer += (*_iter383).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5335,10 +5401,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("key_value_metadata", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->key_value_metadata.size())); - std::vector ::const_iterator _iter380; - for (_iter380 = this->key_value_metadata.begin(); _iter380 != this->key_value_metadata.end(); ++_iter380) + std::vector ::const_iterator _iter384; + for (_iter384 = this->key_value_metadata.begin(); _iter384 != this->key_value_metadata.end(); ++_iter384) { - xfer += (*_iter380).write(oprot); + xfer += (*_iter384).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5353,10 +5419,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("column_orders", ::apache::thrift::protocol::T_LIST, 7); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->column_orders.size())); - std::vector ::const_iterator _iter381; - for (_iter381 = this->column_orders.begin(); _iter381 != this->column_orders.end(); ++_iter381) + std::vector ::const_iterator _iter385; + for (_iter385 = this->column_orders.begin(); _iter385 != this->column_orders.end(); ++_iter385) { - xfer += (*_iter381).write(oprot); + xfer += (*_iter385).write(oprot); } xfer += oprot->writeListEnd(); } diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc index 8735aea731ce..a77936be0764 100644 --- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc +++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc @@ -24,10 +24,13 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#include #include #include #include #include +#include +#include #include #include #include @@ -66,6 +69,8 @@ #include "parquet/api/reader.h" #include "parquet/api/writer.h" +#include "parquet/bloom_filter.h" +#include "parquet/bloom_filter_reader.h" #include "parquet/arrow/fuzz_internal.h" #include "parquet/arrow/reader.h" @@ -75,6 +80,7 @@ #include "parquet/arrow/writer.h" #include "parquet/column_writer.h" #include "parquet/file_writer.h" +#include "parquet/page_index.h" #include "parquet/properties.h" #include "parquet/test_util.h" #include "parquet/types.h" @@ -423,6 +429,19 @@ void WriteTableToBuffer(const std::shared_ptr& table, int64_t row_group_s *out, WriteTableToBuffer(table, row_group_size, write_props, arrow_properties)); } +std::shared_ptr VectorWriterProperties() { + auto builder = WriterProperties::Builder(); + return builder.enable_write_page_index()->encoding(Encoding::PLAIN)->build(); +} + +std::shared_ptr VectorByteStreamSplitWriterProperties() { + auto builder = WriterProperties::Builder(); + return builder.disable_dictionary() + ->enable_write_page_index() + ->encoding(Encoding::BYTE_STREAM_SPLIT) + ->build(); +} + void DoRoundtrip(const std::shared_ptr
& table, int64_t row_group_size, std::shared_ptr
* out, const std::shared_ptr<::parquet::WriterProperties>& writer_properties = @@ -444,6 +463,62 @@ void DoRoundtrip(const std::shared_ptr
& table, int64_t row_group_size, ASSERT_OK_AND_ASSIGN(*out, reader->ReadTable()); } +std::shared_ptr<::arrow::DataType> MakeVectorFixedSizeListType( + const std::shared_ptr<::arrow::DataType>& item_type, bool element_nullable = false) { + return ::arrow::fixed_size_list(::arrow::field("item", item_type, element_nullable), + /*size=*/3); +} + +std::shared_ptr<::arrow::DataType> VectorFixedSizeListType( + bool element_nullable = false) { + return MakeVectorFixedSizeListType(::arrow::int16(), element_nullable); +} + +std::shared_ptr<::arrow::DataType> VectorFloatFixedSizeListType( + bool element_nullable = false) { + return MakeVectorFixedSizeListType(::arrow::float32(), element_nullable); +} + +std::shared_ptr
MakeVectorFixedSizeListTable( + const std::vector>& chunks, bool nullable = true, + bool element_nullable = false) { + auto type = VectorFixedSizeListType(element_nullable); + auto field = ::arrow::field("root", type, nullable); + auto column = std::make_shared(chunks, type); + return ::arrow::Table::Make(::arrow::schema({field}), {column}); +} + +std::shared_ptr
MakeVectorFixedSizeListTable(std::string_view json, + bool nullable = true, + bool element_nullable = false) { + auto type = VectorFixedSizeListType(element_nullable); + return MakeVectorFixedSizeListTable({::arrow::ArrayFromJSON(type, std::string(json))}, + nullable, element_nullable); +} + +std::shared_ptr
MakeVectorFixedSizeListTable( + const std::shared_ptr<::arrow::DataType>& item_type, std::string_view json, + bool nullable = true, bool element_nullable = false) { + auto type = MakeVectorFixedSizeListType(item_type, element_nullable); + auto field = ::arrow::field("root", type, nullable); + auto array = ::arrow::ArrayFromJSON(type, std::string(json)); + return ::arrow::Table::Make(::arrow::schema({field}), {array}); +} + +void CheckVectorFixedSizeListRoundtrip( + const std::shared_ptr
& table, int64_t row_group_size, + const ArrowReaderProperties& arrow_reader_properties = + default_arrow_reader_properties()) { + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE(DoRoundtrip(table, row_group_size, &result, + VectorWriterProperties(), builder.build(), + arrow_reader_properties)); + ::arrow::AssertSchemaEqual(*table->schema(), *result->schema(), false); + ::arrow::AssertTablesEqual(*table, *result, false); +} + void CheckConfiguredRoundtrip( const std::shared_ptr
& input_table, const std::shared_ptr
& expected_table = nullptr, @@ -553,6 +628,18 @@ void CheckSimpleRoundtrip(const std::shared_ptr
& table, int64_t row_group ::arrow::AssertTablesEqual(*table, *result, false); } +// Roundtrip for tables whose FixedSizeList columns fall back to the standard +// LIST encoding even though vector encoding is enabled. store_schema() is +// needed to restore the FixedSizeList type from the LIST encoding on read. +void CheckVectorFallsBackToListRoundtrip(const std::shared_ptr
& table, + int64_t row_group_size) { + auto arrow_writer_properties = ArrowWriterProperties::Builder() + .store_schema() + ->enable_experimental_vector_encoding() + ->build(); + CheckSimpleRoundtrip(table, row_group_size, arrow_writer_properties); +} + static std::shared_ptr MakeSimpleSchema(const DataType& type, Repetition::type repetition) { int32_t byte_width = -1; @@ -2789,6 +2876,32 @@ TEST(TestArrowReadWrite, ScanContents) { ASSERT_EQ(num_rows, num_rows_returned); } +TEST(TestArrowReadWrite, ScanContentsVector) { + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 3); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("id", ::arrow::int32(), /*nullable=*/false), + ::arrow::field("embedding", vector_type, /*nullable=*/true)}), + {::arrow::ArrayFromJSON(::arrow::int32(), "[0, 1, 2, 3]"), + ::arrow::ArrayFromJSON(vector_type, + R"([[1, 2, 3], null, [7, 8, 9], [10, 11, 12]])")}); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, + VectorWriterProperties(), builder.build())); + ASSERT_OK_AND_ASSIGN(auto reader, OpenFile(std::make_shared(buffer), + ::arrow::default_memory_pool())); + + int64_t num_rows_returned = 0; + ASSERT_OK_NO_THROW(reader->ScanContents({}, 2, &num_rows_returned)); + ASSERT_EQ(table->num_rows(), num_rows_returned); + + ASSERT_OK_NO_THROW(reader->ScanContents({1}, 2, &num_rows_returned)); + ASSERT_EQ(table->num_rows(), num_rows_returned); +} + TEST(TestArrowReadWrite, ReadColumnSubset) { const int num_columns = 20; const int num_rows = 1000; @@ -3374,6 +3487,78 @@ TEST(ArrowReadWrite, LargeListView) { ::arrow::large_list(::arrow::int32()), reader_props); } +TEST(ArrowReadWrite, ListViewOfVectorsRoundTrip) { + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto list_view_type = + ::arrow::list_view(::arrow::field("item", vector_type, /*nullable=*/false)); + auto values = ::arrow::ArrayFromJSON( + vector_type, R"([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12]])"); + auto offsets = ::arrow::ArrayFromJSON(::arrow::int32(), "[0, 2, 0, 4, 1]"); + auto sizes = ::arrow::ArrayFromJSON(::arrow::int32(), "[2, 0, null, 2, 1]"); + ASSERT_OK_AND_ASSIGN(auto array, ::arrow::ListViewArray::FromArrays( + list_view_type, *offsets, *sizes, *values, + ::arrow::default_memory_pool())); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("root", list_view_type, /*nullable=*/true)}), + {array}); + + ArrowWriterProperties::Builder builder; + builder.store_schema()->enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, + VectorWriterProperties(), builder.build())); + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + ASSERT_TRUE(parquet_reader->metadata()->schema()->Column(0)->in_vector_column()); + + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE(DoRoundtrip(table, /*row_group_size=*/2, &result, + VectorWriterProperties(), builder.build())); + ::arrow::AssertTablesEqual(*table, *result, /*same_chunk_layout=*/false); +} + +TEST(ArrowReadWrite, ListViewOfVectorsUnderNullableStructRoundTrip) { + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto list_view_type = + ::arrow::list_view(::arrow::field("item", vector_type, /*nullable=*/false)); + auto values = ::arrow::ArrayFromJSON(vector_type, R"([[1, 2], [3, 4], [5, 6]])"); + auto offsets = ::arrow::ArrayFromJSON(::arrow::int32(), "[0, 0, 0, 2]"); + auto sizes = ::arrow::ArrayFromJSON(::arrow::int32(), "[2, 0, 0, 1]"); + ASSERT_OK_AND_ASSIGN( + auto list_view_array, + ::arrow::ListViewArray::FromArrays(list_view_type, *offsets, *sizes, *values, + ::arrow::default_memory_pool())); + auto list_view_field = ::arrow::field("vectors", list_view_type, /*nullable=*/false); + + ::arrow::TypedBufferBuilder bitmap_builder; + ASSERT_OK(bitmap_builder.Append(true)); + ASSERT_OK(bitmap_builder.Append(false)); + ASSERT_OK(bitmap_builder.Append(true)); + ASSERT_OK(bitmap_builder.Append(true)); + ASSERT_OK_AND_ASSIGN(auto null_bitmap, bitmap_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto struct_array, + ::arrow::StructArray::Make({list_view_array}, {list_view_field}, null_bitmap)); + auto struct_type = ::arrow::struct_({list_view_field}); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("root", struct_type, /*nullable=*/true)}), + {struct_array}); + + ArrowWriterProperties::Builder builder; + builder.store_schema()->enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, + VectorWriterProperties(), builder.build())); + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + ASSERT_TRUE(parquet_reader->metadata()->schema()->Column(0)->in_vector_column()); + + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE(DoRoundtrip(table, /*row_group_size=*/2, &result, + VectorWriterProperties(), builder.build())); + ::arrow::AssertTablesEqual(*table, *result, /*same_chunk_layout=*/false); +} + TEST(ArrowReadWrite, EmptyListView) { auto type = ::arrow::list_view(::arrow::int32()); auto array = ArrayFromJSON(type, "[]"); @@ -3396,22 +3581,1031 @@ TEST(ArrowReadWrite, EmptyListView) { } TEST(ArrowReadWrite, FixedSizeList) { - using ::arrow::field; - using ::arrow::fixed_size_list; - using ::arrow::struct_; - - auto type = fixed_size_list(::arrow::int16(), /*size=*/3); - - const char* json = R"([ + auto table = MakeVectorFixedSizeListTable(R"([ [1, 2, 3], [4, 5, 6], - [7, 8, 9]])"; - auto array = ::arrow::ArrayFromJSON(type, json); - auto table = ::arrow::Table::Make(::arrow::schema({field("root", type)}), {array}); + [7, 8, 9]])", + /*nullable=*/true, + /*element_nullable=*/true); auto props_store_schema = ArrowWriterProperties::Builder().store_schema()->build(); CheckSimpleRoundtrip(table, 2, props_store_schema); } +TEST(ArrowWriteOnly, FixedSizeListVectorSchemaRequired) { + auto table = MakeVectorFixedSizeListTable(R"([ + [1, 2, 3], + [4, 5, 6], + [7, 8, 9]])", + /*nullable=*/false); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, + VectorWriterProperties(), builder.build())); + + auto reader = ParquetFileReader::Open(std::make_shared(buffer)); + const auto* schema = reader->metadata()->schema(); + const auto* root = schema->group_node()->field(0).get(); + ASSERT_TRUE(root->is_group()); + const auto* root_group = static_cast(root); + ASSERT_EQ(root_group->repetition(), Repetition::REQUIRED); + ASSERT_TRUE(root_group->logical_type() != nullptr && + root_group->logical_type()->is_vector()); + ASSERT_EQ(root_group->field_count(), 1); + const auto* vector_node = root_group->field(0).get(); + ASSERT_TRUE(vector_node->is_vector()); + ASSERT_TRUE(vector_node->is_group()); + ASSERT_EQ(vector_node->vector_length(), 3); + const auto* vector_group = static_cast(vector_node); + ASSERT_EQ(vector_group->field_count(), 1); + ASSERT_TRUE(vector_group->field(0)->is_primitive()); + ASSERT_TRUE(vector_group->field(0)->is_required()); + ASSERT_EQ(reader->metadata()->RowGroup(0)->ColumnChunk(0)->num_values(), 6); + ASSERT_EQ(reader->metadata()->RowGroup(1)->ColumnChunk(0)->num_values(), 3); +} + +TEST(ArrowWriteOnly, FixedSizeListVectorSchemaNullable) { + auto table = MakeVectorFixedSizeListTable(R"([ + [1, 2, 3], + null, + [7, 8, 9]])"); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/3, + VectorWriterProperties(), builder.build())); + + auto reader = ParquetFileReader::Open(std::make_shared(buffer)); + const auto* schema = reader->metadata()->schema(); + const auto* root = schema->group_node()->field(0).get(); + ASSERT_TRUE(root->is_group()); + const auto* root_group = static_cast(root); + ASSERT_EQ(root_group->repetition(), Repetition::OPTIONAL); + ASSERT_TRUE(root_group->logical_type() != nullptr && + root_group->logical_type()->is_vector()); + ASSERT_EQ(root_group->field_count(), 1); + const auto* vector_node = root_group->field(0).get(); + ASSERT_TRUE(vector_node->is_vector()); + ASSERT_TRUE(vector_node->is_group()); + ASSERT_EQ(vector_node->vector_length(), 3); + const auto* vector_group = static_cast(vector_node); + ASSERT_EQ(vector_group->field_count(), 1); + ASSERT_TRUE(vector_group->field(0)->is_primitive()); + ASSERT_TRUE(vector_group->field(0)->is_required()); + // Option B uses fixed-stride nullable parent layout: every parent row, + // including a null vector row, contributes vector_length child slots. + ASSERT_EQ(reader->metadata()->RowGroup(0)->ColumnChunk(0)->num_values(), 9); +} + +TEST(ArrowWriteOnly, FixedSizeListVectorNullRowsEmitFixedWidthNullSlots) { + auto table = MakeVectorFixedSizeListTable(R"([ + null, + null, + null])"); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/3, + VectorWriterProperties(), builder.build())); + + auto reader = ParquetFileReader::Open(std::make_shared(buffer)); + ASSERT_EQ(reader->metadata()->RowGroup(0)->num_rows(), 3); + ASSERT_EQ(reader->metadata()->RowGroup(0)->ColumnChunk(0)->num_values(), 9); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableRoundTripNullPatterns) { + const std::vector cases = { + R"([[1, 2, 3], [4, 5, 6], [7, 8, 9]])", + R"([[1, 2, 3], null, [7, 8, 9]])", + R"([null, [4, 5, 6], [7, 8, 9]])", + R"([[1, 2, 3], [4, 5, 6], null])", + R"([[1, 2, 3], null, null, [10, 11, 12], null])", + R"([null, null, null])", + R"([])"}; + + for (const auto& json : cases) { + SCOPED_TRACE(json); + auto table = MakeVectorFixedSizeListTable(json); + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip( + table, std::max(1, table->num_rows()))); + } +} + +TEST(ArrowReadWrite, FixedSizeListVectorRestoresStoredChildMetadata) { + auto child_metadata = ::arrow::key_value_metadata({{"child_key", "child_value"}}); + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("item", ::arrow::int32(), /*nullable=*/false, child_metadata), 2); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("root", vector_type, /*nullable=*/true)}), + {::arrow::ArrayFromJSON(vector_type, R"([[1, 2], null, [5, 6]])")}); + + ArrowWriterProperties::Builder builder; + builder.store_schema()->enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/3, + VectorWriterProperties(), builder.build())); + + std::unique_ptr reader; + FileReaderBuilder reader_builder; + ASSERT_OK_NO_THROW(reader_builder.Open(std::make_shared(buffer))); + ASSERT_OK(reader_builder.Build(&reader)); + ASSERT_OK_AND_ASSIGN(auto result, reader->ReadTable()); + + const auto& result_type = checked_cast( + *result->schema()->field(0)->type()); + ASSERT_NE(nullptr, result_type.value_field()->metadata()); + ASSERT_TRUE(result_type.value_field()->metadata()->Equals(*child_metadata)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorStructOfVectorsRoundTrip) { + auto ids_type = ::arrow::fixed_size_list( + ::arrow::field("value", ::arrow::int32(), /*nullable=*/true), 2); + auto scores_type = ::arrow::fixed_size_list(::arrow::float32(), 3); + auto type = ::arrow::struct_({::arrow::field("ids", ids_type, /*nullable=*/true), + ::arrow::field("scores", scores_type, + /*nullable=*/false)}); + auto array = ::arrow::ArrayFromJSON(type, R"([ + {"ids": [10, null], "scores": [0.1, 0.2, 0.3]}, + {"ids": null, "scores": [1.1, 1.2, 1.3]}, + {"ids": [30, 40], "scores": [2.1, 2.2, 2.3]}])"); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("root", type, /*nullable=*/false)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableStructAboveVectorFallsBackToList) { + // The writer encodes vectors below nullable groups as LIST so it does not + // have to expand one null-struct definition level to vector_length leaf slots. + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto struct_type = + ::arrow::struct_({::arrow::field("v", vector_type, /*nullable=*/false)}); + // No null struct rows: writing those through the LIST encoding is an + // upstream NotImplemented ("Lists with non-zero length null components") + // independent of VECTOR. + auto array = ::arrow::ArrayFromJSON(struct_type, R"([ + {"v": [1, 2]}, + {"v": [3, 4]}, + {"v": [5, 6]}])"); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("root", struct_type, /*nullable=*/true)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFallsBackToListRoundtrip(table, + /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableStructUnderListFallsBackToList) { + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::float32(), /*nullable=*/false), 3); + auto struct_type = + ::arrow::struct_({::arrow::field("embedding", vector_type, /*nullable=*/false)}); + auto type = ::arrow::list(::arrow::field("item", struct_type, /*nullable=*/true)); + // No null struct items: writing those through the LIST encoding is an + // upstream NotImplemented ("Lists with non-zero length null components") + // independent of VECTOR. + auto array = ::arrow::ArrayFromJSON(type, R"([ + [{"embedding": [1.0, 2.0, 3.0]}, {"embedding": [4.0, 5.0, 6.0]}], + [{"embedding": [7.0, 8.0, 9.0]}]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFallsBackToListRoundtrip(table, + /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorStructWithListFallsBackToListRoundTrip) { + // The struct element contains a variable-size list, so the column cannot use + // VECTOR encoding and must fall back to the standard LIST encoding. The + // schema conversion and the write-path level generation must agree on that + // fallback, otherwise the writer emits VECTOR-style levels for a LIST + // schema. + auto items_type = ::arrow::list(::arrow::field("item", ::arrow::int32(), + /*nullable=*/false)); + auto struct_type = + ::arrow::struct_({::arrow::field("items", items_type, /*nullable=*/false)}); + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", struct_type, /*nullable=*/false), 2); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [{"items": [1, 2]}, {"items": [3]}], + [{"items": []}, {"items": [4, 5, 6]}]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFallsBackToListRoundtrip(table, + /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorFloat16RoundTrip) { + // The element's logical type (FLOAT16 on FIXED_LEN_BYTE_ARRAY) survives the + // round trip without a stored Arrow schema. + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::float16(), /*nullable=*/false), 3); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [1.5, 2.5, 3.5], + null, + [4.5, 5.5, 6.5]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorDecimalRoundTrip) { + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::decimal128(10, 2), /*nullable=*/false), 2); + auto array = ::arrow::ArrayFromJSON(type, R"([ + ["123.45", "-67.89"], + null, + ["0.01", "99999999.99"]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorRandomNullPatternsRoundTrip) { + // Randomized null patterns across several row groups, read both as a table + // and through the record batch reader with a small batch size. + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 3); + auto type = ::arrow::list(::arrow::field("item", vector_type, /*nullable=*/true)); + std::stringstream json; + json << "["; + unsigned seed = 42; + auto rnd = [&seed]() { + seed = seed * 1103515245 + 12345; + return (seed >> 16) % 8; + }; + int value = 0; + for (int row = 0; row < 200; ++row) { + if (row != 0) json << ","; + unsigned r = rnd(); + if (r == 0) { + json << "null"; + } else if (r == 1) { + json << "[]"; + } else { + json << "["; + int items = r - 1; + for (int i = 0; i < items; ++i) { + if (i != 0) json << ","; + if (rnd() == 2) { + json << "null"; + } else { + json << "[" << value << "," << (value + 1) << "," << (value + 2) << "]"; + value += 3; + } + } + json << "]"; + } + } + json << "]"; + auto array = ::arrow::ArrayFromJSON(type, json.str()); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE( + CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/17)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/17, + VectorWriterProperties(), builder.build())); + ArrowReaderProperties props; + props.set_batch_size(7); + std::unique_ptr reader; + FileReaderBuilder reader_builder; + ASSERT_OK_NO_THROW(reader_builder.Open(std::make_shared(buffer))); + ASSERT_OK(reader_builder.properties(props)->Build(&reader)); + std::unique_ptr<::arrow::RecordBatchReader> batch_reader; + ASSERT_OK_AND_ASSIGN(batch_reader, reader->GetRecordBatchReader()); + ASSERT_OK_AND_ASSIGN(auto result, Table::FromRecordBatchReader(batch_reader.get())); + ::arrow::AssertTablesEqual(*table, *result->CombineChunks().ValueOrDie(), false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNestedRandomNullPatternsRoundTrip) { + auto inner_type = ::arrow::fixed_size_list( + ::arrow::field("value", ::arrow::int32(), /*nullable=*/false), 2); + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", inner_type, /*nullable=*/false), 2); + auto type = ::arrow::list(::arrow::field("item", vector_type, /*nullable=*/true)); + std::stringstream json; + json << "["; + unsigned seed = 1234; + auto rnd = [&seed]() { + seed = seed * 1103515245 + 12345; + return (seed >> 16) % 8; + }; + int value = 0; + for (int row = 0; row < 150; ++row) { + if (row != 0) json << ","; + unsigned r = rnd(); + if (r == 0) { + json << "null"; + } else if (r == 1) { + json << "[]"; + } else { + json << "["; + int items = r - 1; + for (int i = 0; i < items; ++i) { + if (i != 0) json << ","; + if (rnd() == 2) { + json << "null"; + } else { + json << "[[" << value << "," << (value + 1) << "],[" << (value + 2) << "," + << (value + 3) << "]]"; + value += 4; + } + } + json << "]"; + } + } + json << "]"; + auto array = ::arrow::ArrayFromJSON(type, json.str()); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE( + CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/13)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/13, + VectorWriterProperties(), builder.build())); + ArrowReaderProperties props; + props.set_batch_size(5); + std::unique_ptr reader; + FileReaderBuilder reader_builder; + ASSERT_OK_NO_THROW(reader_builder.Open(std::make_shared(buffer))); + ASSERT_OK(reader_builder.properties(props)->Build(&reader)); + std::unique_ptr<::arrow::RecordBatchReader> batch_reader; + ASSERT_OK_AND_ASSIGN(batch_reader, reader->GetRecordBatchReader()); + ASSERT_OK_AND_ASSIGN(auto result, Table::FromRecordBatchReader(batch_reader.get())); + ::arrow::AssertTablesEqual(*table, *result->CombineChunks().ValueOrDie(), false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorCdcAndByteStreamSplitUnderList) { + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::float32(), /*nullable=*/false), 3); + auto type = ::arrow::list(::arrow::field("item", vector_type, /*nullable=*/false)); + std::stringstream json; + json << "["; + for (int row = 0; row < 50; ++row) { + if (row != 0) json << ","; + if (row % 7 == 0) { + json << "[]"; + } else { + json << "[[" << row << ".0," << row << ".5," << row << ".25]]"; + } + } + json << "]"; + auto array = ::arrow::ArrayFromJSON(type, json.str()); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + auto cdc_props = WriterProperties::Builder() + .enable_content_defined_chunking() + ->enable_write_page_index() + ->build(); + ArrowWriterProperties::Builder cdc_arrow_builder; + cdc_arrow_builder.enable_experimental_vector_encoding(); + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE( + DoRoundtrip(table, 13, &result, cdc_props, cdc_arrow_builder.build())); + ::arrow::AssertTablesEqual(*table, *result, false); + + auto bss_props = WriterProperties::Builder() + .disable_dictionary() + ->encoding(Encoding::BYTE_STREAM_SPLIT) + ->build(); + ArrowWriterProperties::Builder bss_arrow_builder; + bss_arrow_builder.enable_experimental_vector_encoding(); + ASSERT_NO_FATAL_FAILURE( + DoRoundtrip(table, 13, &result, bss_props, bss_arrow_builder.build())); + ::arrow::AssertTablesEqual(*table, *result, false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorListOfNullableVectorsRoundTrip) { + // Null and present vector items interleaved within and across lists. + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto type = ::arrow::list(::arrow::field("item", vector_type, /*nullable=*/true)); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [[1, 2], null, [3, 4]], + [null], + [], + null, + [[5, 6], [7, 8], null, [9, 10]]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, + /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorListOfOnlyNullVectorsRoundTrip) { + // A row group containing only null VECTOR items still has physical null + // slots but no visited value ranges. + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto type = ::arrow::list(::arrow::field("item", vector_type, /*nullable=*/true)); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [null], + [null, null], + [], + null, + [[1, 2]]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, + /*row_group_size=*/1)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNestedVectorsUnderListRoundTrip) { + auto inner = ::arrow::fixed_size_list( + ::arrow::field("value", ::arrow::int32(), /*nullable=*/false), 2); + auto outer = + ::arrow::fixed_size_list(::arrow::field("element", inner, /*nullable=*/false), 2); + auto type = ::arrow::list(::arrow::field("item", outer, /*nullable=*/true)); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [[[1, 2], [3, 4]], null], + [], + null, + [[[5, 6], [7, 8]]]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, + /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorListOfVectorsRoundTrip) { + // FixedSizeList values below a repeated field can use VECTOR encoding; + // empty and null list entries must roundtrip. + auto vector_type = ::arrow::fixed_size_list(::arrow::int32(), 2); + auto type = ::arrow::list(::arrow::field("item", vector_type, /*nullable=*/false)); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [[1, 2], [3, 4]], + [], + null, + [[5, 6]], + [[7, 8], [9, 10], [11, 12]]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, + /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorListOfStructsWithVectorsRoundTrip) { + auto vector_type = ::arrow::fixed_size_list(::arrow::float32(), 3); + auto struct_type = + ::arrow::struct_({::arrow::field("embedding", vector_type, /*nullable=*/false)}); + auto type = ::arrow::list(::arrow::field("item", struct_type, /*nullable=*/false)); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [{"embedding": [1.0, 2.0, 3.0]}, {"embedding": [4.0, 5.0, 6.0]}], + [], + [{"embedding": [7.0, 8.0, 9.0]}], + [{"embedding": [10.0, 11.0, 12.0]}, {"embedding": [13.0, 14.0, 15.0]}]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, + /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorMapOfVectorsRoundTrip) { + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto type = ::arrow::map(::arrow::utf8(), ::arrow::field("value", vector_type, + /*nullable=*/false)); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [["a", [1, 2]], ["b", [3, 4]]], + [], + [["c", [5, 6]]]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, + /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorStructWithNestedVectorFallsBackToList) { + // A fixed-size list nested inside a struct vector element would give the + // struct's leaves different per-row multiplicities, which the write path + // does not support; the column falls back to the LIST encoding. + auto inner = ::arrow::fixed_size_list( + ::arrow::field("value", ::arrow::int32(), /*nullable=*/false), 2); + auto struct_type = + ::arrow::struct_({::arrow::field("v", inner, /*nullable=*/false), + ::arrow::field("x", ::arrow::int32(), /*nullable=*/false)}); + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", struct_type, /*nullable=*/false), 3); + // No null rows: writing null FixedSizeList rows with nested children via + // the LIST encoding is an upstream NotImplemented independent of VECTOR. + auto array = ::arrow::ArrayFromJSON(type, R"([ + [{"v": [1, 2], "x": 1}, {"v": [3, 4], "x": 2}, {"v": [5, 6], "x": 3}], + [{"v": [7, 8], "x": 4}, {"v": [9, 10], "x": 5}, {"v": [11, 12], "x": 6}]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFallsBackToListRoundtrip(table, + /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListZeroLength) { + // Regression test: zero-length fixed-size lists used to be written with the + // definition level of a present element instead of an empty list, producing + // one garbage element per row on read. + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 0); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [], + null, + []])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + auto props_store_schema = ArrowWriterProperties::Builder().store_schema()->build(); + CheckSimpleRoundtrip(table, 3, props_store_schema); +} + +TEST(ArrowReadWrite, FixedSizeListVectorZeroLengthFallsBackToList) { + // A zero-length vector value would contribute no physical leaf slots, so + // neither row counts nor row nullability are representable in the VECTOR + // encoding; zero-length fixed-size lists use the LIST encoding instead. + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 0); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [], + null, + []])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFallsBackToListRoundtrip(table, + /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorBoolRoundTrip) { + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::boolean(), /*nullable=*/false), 3); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [true, false, true], + null, + [false, false, true]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorRecordBatchReaderSmallBatches) { + auto table = MakeVectorFixedSizeListTable(R"([ + [1, 2, 3], + null, + [7, 8, 9], + [10, 11, 12]])"); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/4, + VectorWriterProperties(), builder.build())); + + ArrowReaderProperties reader_properties; + reader_properties.set_batch_size(1); + std::unique_ptr reader; + FileReaderBuilder reader_builder; + ASSERT_OK_NO_THROW(reader_builder.Open(std::make_shared(buffer))); + ASSERT_OK(reader_builder.properties(reader_properties)->Build(&reader)); + std::unique_ptr<::arrow::RecordBatchReader> batch_reader; + ASSERT_OK_AND_ASSIGN(batch_reader, reader->GetRecordBatchReader()); + ASSERT_OK_AND_ASSIGN(auto result, Table::FromRecordBatchReader(batch_reader.get())); + ::arrow::AssertTablesEqual(*table, *result, false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNestedVectorsRoundTrip) { + auto inner_type = ::arrow::fixed_size_list(::arrow::uint8(), 2); + auto type = + ::arrow::fixed_size_list(::arrow::field("item", inner_type, /*nullable=*/false), 2); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [[1, 2], [3, 4]], + null, + [[5, 6], [7, 8]]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableChunkedReadAcrossRowGroups) { + auto table = MakeVectorFixedSizeListTable(R"([ + [1, 2, 3], + null, + [7, 8, 9], + null, + [13, 14, 15]])"); + + ArrowWriterProperties::Builder writer_builder; + writer_builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN( + auto buffer, WriteTableToBuffer(table, /*row_group_size=*/2, + VectorWriterProperties(), writer_builder.build())); + + ArrowReaderProperties reader_properties; + reader_properties.set_batch_size(2); + FileReaderBuilder reader_builder; + ASSERT_OK(reader_builder.Open(std::make_shared(buffer))); + reader_builder.properties(reader_properties); + std::unique_ptr reader; + ASSERT_OK(reader_builder.Build(&reader)); + ASSERT_OK_AND_ASSIGN(auto rb_reader, reader->GetRecordBatchReader()); + ASSERT_OK_AND_ASSIGN(auto out, Table::FromRecordBatchReader(rb_reader.get())); + + ASSERT_EQ(out->column(0)->num_chunks(), 3); + ASSERT_EQ(out->column(0)->chunk(0)->length(), 2); + ASSERT_EQ(out->column(0)->chunk(1)->length(), 2); + ASSERT_EQ(out->column(0)->chunk(2)->length(), 1); + ::arrow::AssertTablesEqual(*table, *out, false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableRoundTripSlicedInput) { + auto base = ::arrow::ArrayFromJSON(VectorFixedSizeListType(), R"([ + [100, 101, 102], + [1, 2, 3], + null, + [7, 8, 9], + null, + [200, 201, 202]])"); + auto table = MakeVectorFixedSizeListTable({base->Slice(/*offset=*/1, /*length=*/4)}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableRowsAndElementsRoundTrip) { + auto table = MakeVectorFixedSizeListTable(R"([ + [1, null, 3], + null, + [null, null, null], + [7, 8, null]])", + /*nullable=*/true, + /*element_nullable=*/true); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/4)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNestedRoundTrip) { + auto inner_type = ::arrow::fixed_size_list( + ::arrow::field("item", ::arrow::int16(), /*nullable=*/true), /*list_size=*/2); + auto outer_type = ::arrow::fixed_size_list( + ::arrow::field("item", inner_type, /*nullable=*/true), /*list_size=*/3); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("root", outer_type, /*nullable=*/true)}), + {::arrow::ArrayFromJSON(outer_type, R"([ + [[1, 2], null, [5, null]], + null, + [[7, 8], [9, 10], [null, 12]]])")}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorMixedColumnsRoundTripAcrossRowGroups) { + auto vector_type = + ::arrow::fixed_size_list(::arrow::field("item", ::arrow::int16(), true), + /*list_size=*/3); + auto list_type = ::arrow::list(::arrow::field("item", ::arrow::int32(), true)); + + auto vector_chunks = std::vector>{ + ::arrow::ArrayFromJSON(vector_type, + R"([[1, null, 3], null, [7, 8, null], [10, 11, 12]])"), + ::arrow::ArrayFromJSON(vector_type, + R"([null, [16, null, 18], [19, 20, 21], [22, null, 24]])")}; + auto table = ::arrow::Table::Make( + ::arrow::schema({ + ::arrow::field("id", ::arrow::int32(), false), + ::arrow::field("embedding", vector_type, true), + ::arrow::field("label", ::arrow::utf8(), true), + ::arrow::field("tags", list_type, true), + }), + { + std::make_shared( + ::arrow::ArrayFromJSON(::arrow::int32(), "[0, 1, 2, 3, 4, 5, 6, 7]")), + std::make_shared(std::move(vector_chunks), vector_type), + std::make_shared(::arrow::ArrayFromJSON( + ::arrow::utf8(), R"(["a", null, "c", "d", null, "f", "g", "h"])")), + std::make_shared(::arrow::ArrayFromJSON( + list_type, R"([[1, 2], [], null, [3, null], [4], [5, 6], null, []])")), + }); + + ArrowWriterProperties::Builder writer_builder; + writer_builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN( + auto buffer, WriteTableToBuffer(table, /*row_group_size=*/3, + VectorWriterProperties(), writer_builder.build())); + + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + ASSERT_EQ(parquet_reader->metadata()->num_row_groups(), 3); + for (int i = 0; i < 3; ++i) { + const int64_t expected_rows = i == 2 ? 2 : 3; + EXPECT_EQ(parquet_reader->metadata()->RowGroup(i)->num_rows(), expected_rows); + EXPECT_EQ(parquet_reader->metadata()->RowGroup(i)->ColumnChunk(1)->num_values(), + expected_rows * 3); + } + + ArrowReaderProperties reader_properties; + reader_properties.set_batch_size(2); + FileReaderBuilder reader_builder; + ASSERT_OK(reader_builder.Open(std::make_shared(buffer))); + reader_builder.properties(reader_properties); + std::unique_ptr reader; + ASSERT_OK(reader_builder.Build(&reader)); + ASSERT_OK_AND_ASSIGN(auto rb_reader, reader->GetRecordBatchReader()); + ASSERT_OK_AND_ASSIGN(auto out, Table::FromRecordBatchReader(rb_reader.get())); + + ::arrow::AssertSchemaEqual(*table->schema(), *out->schema(), false); + ::arrow::AssertTablesEqual(*table, *out, false); +} + +std::shared_ptr<::arrow::DataType> VectorFixedSizeListStructType( + bool element_nullable = false, bool field_nullable = false) { + return ::arrow::fixed_size_list( + ::arrow::field( + "item", + ::arrow::struct_({::arrow::field("x", ::arrow::float32(), false), + ::arrow::field("y", ::arrow::int32(), field_nullable)}), + element_nullable), + /*size=*/2); +} + +std::shared_ptr
MakeVectorFixedSizeListStructTable(std::string_view json, + bool nullable = true, + bool element_nullable = false, + bool field_nullable = false) { + auto type = VectorFixedSizeListStructType(element_nullable, field_nullable); + auto field = ::arrow::field("root", type, nullable); + auto array = ::arrow::ArrayFromJSON(type, std::string(json)); + return ::arrow::Table::Make(::arrow::schema({field}), {array}); +} + +std::shared_ptr<::arrow::DataType> VectorFixedSizeListNestedStructType( + bool element_nullable = false) { + return ::arrow::fixed_size_list( + ::arrow::field( + "item", + ::arrow::struct_( + {::arrow::field( + "point", + ::arrow::struct_({::arrow::field("x", ::arrow::float32(), false), + ::arrow::field("y", ::arrow::int32(), true)}), + false), + ::arrow::field("z", ::arrow::int16(), true)}), + element_nullable), + /*size=*/2); +} + +std::shared_ptr
MakeVectorFixedSizeListNestedStructTable( + std::string_view json, bool nullable = true, bool element_nullable = false) { + auto type = VectorFixedSizeListNestedStructType(element_nullable); + auto field = ::arrow::field("root", type, nullable); + auto array = ::arrow::ArrayFromJSON(type, std::string(json)); + return ::arrow::Table::Make(::arrow::schema({field}), {array}); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableStructElementsRoundTrip) { + auto table = MakeVectorFixedSizeListStructTable(R"([ + [{"x": 1.0, "y": 1}, null], + null, + [null, {"x": 6.0, "y": null}], + [{"x": 7.0, "y": 7}, {"x": 8.0, "y": 8}]])", + /*nullable=*/true, + /*element_nullable=*/true, + /*field_nullable=*/true); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableNestedStructElementsRoundTrip) { + auto table = MakeVectorFixedSizeListNestedStructTable(R"([ + [{"point": {"x": 1.0, "y": 1}, "z": 10}, null], + null, + [null, {"point": {"x": 4.0, "y": 4}, "z": 40}], + [{"point": {"x": 5.0, "y": null}, "z": null}, + {"point": {"x": 6.0, "y": 6}, "z": 60}]])", + /*nullable=*/true, + /*element_nullable=*/true); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/2)); +} + +TEST(ArrowWriteOnly, FixedSizeListVectorDefaultWriterProperties) { + auto table = MakeVectorFixedSizeListTable(R"([ + [1, 2, 3], + [4, 5, 6]])", + /*nullable=*/false); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, + default_writer_properties(), builder.build())); + + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + const auto encodings = + parquet_reader->metadata()->RowGroup(0)->ColumnChunk(0)->encodings(); + EXPECT_NE(std::find(encodings.begin(), encodings.end(), Encoding::RLE_DICTIONARY), + encodings.end()); +} + +TEST(ArrowWriteOnly, FixedSizeListVectorNullableElementStatistics) { + auto table = MakeVectorFixedSizeListTable(::arrow::int32(), R"([ + [1, 2, 3], + null, + [4, null, 6]])", + /*nullable=*/true, + /*element_nullable=*/true); + + auto writer_properties = WriterProperties::Builder() + .disable_dictionary() + ->encoding(Encoding::PLAIN) + ->build(); + ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/3, writer_properties, + arrow_builder.build())); + + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + auto column = parquet_reader->metadata()->RowGroup(0)->ColumnChunk(0); + ASSERT_TRUE(column->is_stats_set()); + auto statistics = column->statistics(); + ASSERT_EQ(5, statistics->num_values()); + ASSERT_EQ(4, statistics->null_count()); + + std::shared_ptr min, max; + ASSERT_OK(StatisticsAsScalars(*statistics, &min, &max)); + ASSERT_OK_AND_ASSIGN(auto expected_min, ::arrow::MakeScalar(::arrow::int32(), 1)); + ASSERT_OK_AND_ASSIGN(auto expected_max, ::arrow::MakeScalar(::arrow::int32(), 6)); + ::arrow::AssertScalarsEqual(*expected_min, *min, /*verbose=*/true); + ::arrow::AssertScalarsEqual(*expected_max, *max, /*verbose=*/true); +} + +TEST(ArrowWriteOnly, FixedSizeListVectorPageIndex) { + auto table = MakeVectorFixedSizeListTable(::arrow::int32(), R"([ + [1, 2, 3], + [4, 5, 6], + [7, 8, 9]])", + /*nullable=*/false); + + auto writer_properties = WriterProperties::Builder() + .disable_dictionary() + ->encoding(Encoding::PLAIN) + ->enable_write_page_index() + ->build(); + ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, writer_properties, + arrow_builder.build())); + + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + auto page_index_reader = parquet_reader->GetPageIndexReader(); + ASSERT_NE(nullptr, page_index_reader); + auto row_group_index_reader = page_index_reader->RowGroup(0); + ASSERT_NE(nullptr, row_group_index_reader); + ASSERT_NE(nullptr, row_group_index_reader->GetColumnIndex(0)); + ASSERT_NE(nullptr, row_group_index_reader->GetOffsetIndex(0)); +} + +TEST(ArrowWriteOnly, FixedSizeListVectorBloomFilter) { + auto table = MakeVectorFixedSizeListTable(::arrow::int32(), R"([ + [1, 2, 3], + [4, 5, 6]])", + /*nullable=*/false); + + BloomFilterOptions bloom_filter_options{10, 0.05}; + auto writer_properties = + WriterProperties::Builder() + .disable_dictionary() + ->encoding(Encoding::PLAIN) + ->enable_bloom_filter("root.list.element", bloom_filter_options) + ->build(); + ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, writer_properties, + arrow_builder.build())); + + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + auto& bloom_filter_reader = parquet_reader->GetBloomFilterReader(); + auto row_group_bloom_filter = bloom_filter_reader.RowGroup(0); + ASSERT_NE(nullptr, row_group_bloom_filter); + auto bloom_filter = row_group_bloom_filter->GetColumnBloomFilter(0); + ASSERT_NE(nullptr, bloom_filter); + + int32_t present = 4; + EXPECT_TRUE(bloom_filter->FindHash(bloom_filter->Hash(present))); +} + +TEST(ArrowReadWrite, FixedSizeListVectorContentDefinedChunkingRoundTrip) { + std::stringstream json; + json << "["; + for (int i = 0; i < 40; ++i) { + if (i != 0) { + json << ","; + } + json << "[" << (3 * i + 1) << "," << (3 * i + 2) << "," << (3 * i + 3) << "]"; + } + json << "]"; + auto table = MakeVectorFixedSizeListTable(::arrow::int32(), json.str(), + /*nullable=*/false); + + CdcOptions cdc_options; + cdc_options.min_chunk_size = 64; + cdc_options.max_chunk_size = 128; + auto writer_properties = WriterProperties::Builder() + .enable_content_defined_chunking() + ->content_defined_chunking_options(cdc_options) + ->build(); + ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_experimental_vector_encoding(); + + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE(DoRoundtrip(table, /*row_group_size=*/40, &result, + writer_properties, arrow_builder.build())); + ::arrow::AssertSchemaEqual(*table->schema(), *result->schema(), false); + ::arrow::AssertTablesEqual(*table, *result, false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorContentDefinedChunkingNullableRoundTrip) { + auto table = MakeVectorFixedSizeListTable(::arrow::int32(), R"([ + [1, 2, 3], + null, + [4, null, 6], + [7, 8, 9]])", + /*nullable=*/true, + /*element_nullable=*/true); + + CdcOptions cdc_options; + cdc_options.min_chunk_size = 64; + cdc_options.max_chunk_size = 128; + auto writer_properties = WriterProperties::Builder() + .disable_dictionary() + ->enable_content_defined_chunking() + ->content_defined_chunking_options(cdc_options) + ->build(); + ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_experimental_vector_encoding(); + + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE(DoRoundtrip(table, /*row_group_size=*/4, &result, + writer_properties, arrow_builder.build())); + ::arrow::AssertSchemaEqual(*table->schema(), *result->schema(), false); + ::arrow::AssertTablesEqual(*table, *result, false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorByteStreamSplitRoundTrip) { + auto table = MakeVectorFixedSizeListTable(::arrow::float32(), R"([ + [1.0, 2.0, 3.0], + [4.0, 5.0, 6.0], + [7.0, 8.0, 9.0]])", + /*nullable=*/false); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN( + auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, + VectorByteStreamSplitWriterProperties(), builder.build())); + + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + const auto encodings = + parquet_reader->metadata()->RowGroup(0)->ColumnChunk(0)->encodings(); + ASSERT_NE(std::find(encodings.begin(), encodings.end(), Encoding::BYTE_STREAM_SPLIT), + encodings.end()); + + FileReaderBuilder reader_builder; + ASSERT_OK(reader_builder.Open(std::make_shared(buffer))); + std::unique_ptr reader; + ASSERT_OK(reader_builder.Build(&reader)); + std::shared_ptr
result; + ASSERT_OK(reader->ReadTable(&result)); + ::arrow::AssertSchemaEqual(*table->schema(), *result->schema(), false); + ::arrow::AssertTablesEqual(*table, *result, false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorByteStreamSplitNullableRoundTrip) { + auto table = MakeVectorFixedSizeListTable(::arrow::float32(), R"([ + [1.0, 2.0, 3.0], + null, + [7.0, 8.0, 9.0]])"); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE(DoRoundtrip( + table, 3, &result, VectorByteStreamSplitWriterProperties(), builder.build())); + ::arrow::AssertSchemaEqual(*table->schema(), *result->schema(), false); + ::arrow::AssertTablesEqual(*table, *result, false); +} + TEST(ArrowReadWrite, ListOfStructOfList2) { using ::arrow::field; using ::arrow::list; @@ -3864,6 +5058,168 @@ TEST(ArrowReadWrite, NestedRequiredOuterOptionalDecimal) { } } +TEST(TestArrowReaderAdHoc, ReadVectorUnderNullableStructAncestor) { + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::INT32); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + ConvertedType::NONE, /*field_id=*/-1, + /*vector_length=*/2); + auto embedding = + GroupNode::Make("embedding", Repetition::REQUIRED, {vector}, LogicalType::Vector()); + auto parent = GroupNode::Make("parent", Repetition::OPTIONAL, {embedding}); + auto list = GroupNode::Make("list", Repetition::REPEATED, {parent}); + auto rows = GroupNode::Make("rows", Repetition::OPTIONAL, {list}, LogicalType::List()); + auto schema = GroupNode::Make("schema", Repetition::REQUIRED, {rows}); + + auto sink = CreateOutputStream(); + auto file_writer = + ParquetFileWriter::Open(sink, checked_pointer_cast(schema)); + auto column_writer = + checked_cast(file_writer->AppendRowGroup()->NextColumn()); + + // One top-level list row: [{"embedding": [1, 2]}, null, {"embedding": [3, 4]}]. + // The null struct element is represented by a single level with def_level=2; + // each present VECTOR value contributes two physical leaf slots at def_level=3. + const std::vector def_levels = {3, 3, 2, 3, 3}; + const std::vector rep_levels = {0, 1, 1, 1, 1}; + const std::vector values = {1, 2, 3, 4}; + column_writer->WriteBatch(static_cast(def_levels.size()), def_levels.data(), + rep_levels.data(), values.data()); + column_writer->Close(); + file_writer->Close(); + + ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish()); + + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto parent_type = ::arrow::struct_({::arrow::field("embedding", vector_type, + /*nullable=*/false)}); + auto rows_type = ::arrow::list(::arrow::field("parent", parent_type, + /*nullable=*/true)); + auto expected = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("rows", rows_type, /*nullable=*/true)}), + {::arrow::ArrayFromJSON( + rows_type, R"([[{"embedding": [1, 2]}, null, {"embedding": [3, 4]}]])")}); + + FileReaderBuilder reader_builder; + ASSERT_OK(reader_builder.Open(std::make_shared(buffer))); + std::unique_ptr reader; + ASSERT_OK(reader_builder.Build(&reader)); + ASSERT_OK_AND_ASSIGN(auto result, reader->ReadTable()); + ASSERT_OK(result->ValidateFull()); + ::arrow::AssertTablesEqual(*expected, *result, /*same_chunk_layout=*/false); +} + +TEST(TestArrowReaderAdHoc, ReadVectorUnderNullableStructAncestorWithEmptyLists) { + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::INT32); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + ConvertedType::NONE, /*field_id=*/-1, + /*vector_length=*/2); + auto embedding = + GroupNode::Make("embedding", Repetition::REQUIRED, {vector}, LogicalType::Vector()); + auto parent = GroupNode::Make("parent", Repetition::OPTIONAL, {embedding}); + auto list = GroupNode::Make("list", Repetition::REPEATED, {parent}); + auto rows = GroupNode::Make("rows", Repetition::OPTIONAL, {list}, LogicalType::List()); + auto schema = GroupNode::Make("schema", Repetition::REQUIRED, {rows}); + + auto sink = CreateOutputStream(); + auto file_writer = + ParquetFileWriter::Open(sink, checked_pointer_cast(schema)); + auto column_writer = + checked_cast(file_writer->AppendRowGroup()->NextColumn()); + + // Four top-level rows: null, [], [null], [{"embedding": [1, 2]}]. + // The null and empty lists are compact absent-list markers and must not be + // expanded into FixedSizeList child rows; only the null struct item expands. + const std::vector def_levels = {0, 1, 2, 3, 3}; + const std::vector rep_levels = {0, 0, 0, 0, 1}; + const std::vector values = {1, 2}; + column_writer->WriteBatch(static_cast(def_levels.size()), def_levels.data(), + rep_levels.data(), values.data()); + column_writer->Close(); + file_writer->Close(); + + ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish()); + + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto parent_type = ::arrow::struct_({::arrow::field("embedding", vector_type, + /*nullable=*/false)}); + auto rows_type = ::arrow::list(::arrow::field("parent", parent_type, + /*nullable=*/true)); + auto expected = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("rows", rows_type, /*nullable=*/true)}), + {::arrow::ArrayFromJSON(rows_type, + R"([null, [], [null], [{"embedding": [1, 2]}]])")}); + + FileReaderBuilder reader_builder; + ASSERT_OK(reader_builder.Open(std::make_shared(buffer))); + std::unique_ptr reader; + ASSERT_OK(reader_builder.Build(&reader)); + ASSERT_OK_AND_ASSIGN(auto result, reader->ReadTable()); + ASSERT_OK(result->ValidateFull()); + ::arrow::AssertTablesEqual(*expected, *result, /*same_chunk_layout=*/false); +} + +TEST(TestArrowReaderAdHoc, ReadNestedVectorUnderNullableStructAncestor) { + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::INT32); + auto inner_vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + ConvertedType::NONE, /*field_id=*/-1, + /*vector_length=*/3); + auto inner = GroupNode::Make("element", Repetition::REQUIRED, {inner_vector}, + LogicalType::Vector()); + auto outer_vector = GroupNode::Make("list", Repetition::VECTOR, {inner}, + ConvertedType::NONE, /*field_id=*/-1, + /*vector_length=*/2); + auto embedding = GroupNode::Make("embedding", Repetition::REQUIRED, {outer_vector}, + LogicalType::Vector()); + auto parent = GroupNode::Make("parent", Repetition::OPTIONAL, {embedding}); + auto list = GroupNode::Make("list", Repetition::REPEATED, {parent}); + auto rows = GroupNode::Make("rows", Repetition::OPTIONAL, {list}, LogicalType::List()); + auto schema = GroupNode::Make("schema", Repetition::REQUIRED, {rows}); + + auto sink = CreateOutputStream(); + auto file_writer = + ParquetFileWriter::Open(sink, checked_pointer_cast(schema)); + auto column_writer = + checked_cast(file_writer->AppendRowGroup()->NextColumn()); + + // One top-level list row: + // [{"embedding": [[1, 2, 3], [4, 5, 6]]}, null, + // {"embedding": [[7, 8, 9], [10, 11, 12]]}]. + const std::vector def_levels = {3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3}; + std::vector rep_levels(def_levels.size(), 1); + rep_levels[0] = 0; + const std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; + column_writer->WriteBatch(static_cast(def_levels.size()), def_levels.data(), + rep_levels.data(), values.data()); + column_writer->Close(); + file_writer->Close(); + + ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish()); + + auto inner_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 3); + auto outer_type = ::arrow::fixed_size_list( + ::arrow::field("element", inner_type, /*nullable=*/false), 2); + auto parent_type = ::arrow::struct_({::arrow::field("embedding", outer_type, + /*nullable=*/false)}); + auto rows_type = ::arrow::list(::arrow::field("parent", parent_type, + /*nullable=*/true)); + auto expected = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("rows", rows_type, /*nullable=*/true)}), + {::arrow::ArrayFromJSON(rows_type, + R"([[{"embedding": [[1, 2, 3], [4, 5, 6]]}, null, + {"embedding": [[7, 8, 9], [10, 11, 12]]}]])")}); + + FileReaderBuilder reader_builder; + ASSERT_OK(reader_builder.Open(std::make_shared(buffer))); + std::unique_ptr reader; + ASSERT_OK(reader_builder.Build(&reader)); + ASSERT_OK_AND_ASSIGN(auto result, reader->ReadTable()); + ASSERT_OK(result->ValidateFull()); + ::arrow::AssertTablesEqual(*expected, *result, /*same_chunk_layout=*/false); +} + TEST(ArrowReadWrite, Decimal256AsInt) { using ::arrow::Decimal256; using ::arrow::field; @@ -4431,8 +5787,7 @@ TEST(TestArrowReaderAdHoc, OldDataPageV2) { GTEST_SKIP() << "ARROW_TEST_DATA not set."; } std::stringstream ss; - ss << c_root << "/" - << "parquet/ARROW-17100.parquet"; + ss << c_root << "/" << "parquet/ARROW-17100.parquet"; std::string path = ss.str(); TryReadDataFile(path); } diff --git a/cpp/src/parquet/arrow/arrow_schema_test.cc b/cpp/src/parquet/arrow/arrow_schema_test.cc index 7a7b5a336939..a3f861d9d600 100644 --- a/cpp/src/parquet/arrow/arrow_schema_test.cc +++ b/cpp/src/parquet/arrow/arrow_schema_test.cc @@ -289,6 +289,188 @@ TEST_F(TestConvertParquetSchema, ParquetAnnotatedFields) { ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(arrow_schema)); } +TEST_F(TestConvertParquetSchema, UnannotatedVectorGroupRejected) { + // The VECTOR logical type annotation is required: a plain group containing a + // VECTOR-repeated child is not silently interpreted as a vector. + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("vector", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + std::vector parquet_fields = {GroupNode::Make( + "embedding", Repetition::OPTIONAL, {vector}, /*logical_type=*/nullptr)}; + + ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); +} + +TEST_F(TestConvertParquetSchema, VectorFixedSizeListAnnotated) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + arrow_fields.push_back(::arrow::field( + "embedding", + ::arrow::fixed_size_list(::arrow::field("element", ::arrow::float32(), false), 3), + true)); + + ASSERT_OK(ConvertSchema(parquet_fields)); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(::arrow::schema(arrow_fields))); +} + +TEST_F(TestConvertParquetSchema, VectorAnnotatedGroupWithoutVectorChildRejected) { + // A group annotated with the VECTOR logical type must contain a single + // VECTOR-repeated child. + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + std::vector parquet_fields = {GroupNode::Make( + "embedding", Repetition::OPTIONAL, {element}, LogicalType::Vector())}; + + ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); +} + +TEST_F(TestConvertParquetSchema, VectorMiddleGroupNameRejected) { + // The canonical VECTOR layout requires the middle group to be named "list". + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("vector", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + std::vector parquet_fields = {GroupNode::Make( + "embedding", Repetition::OPTIONAL, {vector}, LogicalType::Vector())}; + + ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); +} + +TEST_F(TestConvertParquetSchema, VectorElementNameRejected) { + // The canonical VECTOR layout requires the element field to be named "element". + auto element = PrimitiveNode::Make("item", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + std::vector parquet_fields = {GroupNode::Make( + "embedding", Repetition::OPTIONAL, {vector}, LogicalType::Vector())}; + + ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); +} + +TEST_F(TestConvertParquetSchema, VectorMiddleGroupLogicalTypeRejected) { + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + ASSERT_THROW( + GroupNode::Make("list", Repetition::VECTOR, {element}, LogicalType::List(), -1, 3), + ParquetException); +} + +TEST_F(TestConvertParquetSchema, VectorOnPrimitiveRejected) { + // VECTOR repetition directly on a primitive is not part of the canonical + // 3-level vector structure; vectors must use a VECTOR group whose single + // child is a VECTOR-repeated group containing the element. + ASSERT_THROW(PrimitiveNode::Make("element", Repetition::VECTOR, ParquetType::FLOAT, + ConvertedType::NONE, -1, -1, -1, -1, 3), + ParquetException); +} + +TEST_F(TestConvertParquetSchema, VectorUnderNullableStructAncestor) { + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 2); + auto embedding = + GroupNode::Make("embedding", Repetition::REQUIRED, {vector}, LogicalType::Vector()); + std::vector parquet_fields = { + GroupNode::Make("outer", Repetition::OPTIONAL, {embedding})}; + + auto expected = ::arrow::schema({::arrow::field( + "outer", + ::arrow::struct_({::arrow::field( + "embedding", + ::arrow::fixed_size_list(::arrow::field("element", INT32, false), 2), false)}), + true)}); + + ASSERT_OK(ConvertSchema(parquet_fields)); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(expected)); +} + +TEST_F(TestConvertParquetSchema, VectorFixedSizeListNullableStructElement) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto x = PrimitiveNode::Make("x", Repetition::REQUIRED, ParquetType::FLOAT); + auto y = PrimitiveNode::Make("y", Repetition::OPTIONAL, ParquetType::INT32); + auto item = GroupNode::Make("element", Repetition::OPTIONAL, {x, y}); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {item}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + arrow_fields.push_back(::arrow::field( + "embedding", + ::arrow::fixed_size_list( + ::arrow::field("element", + ::arrow::struct_({::arrow::field("x", FLOAT, false), + ::arrow::field("y", INT32, true)}), + true), + 3), + true)); + + ASSERT_OK(ConvertSchema(parquet_fields)); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(::arrow::schema(arrow_fields))); +} + +TEST_F(TestConvertParquetSchema, VectorFixedSizeListNullableNestedStructElement) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto x = PrimitiveNode::Make("x", Repetition::REQUIRED, ParquetType::FLOAT); + auto y = PrimitiveNode::Make("y", Repetition::OPTIONAL, ParquetType::INT32); + auto point = GroupNode::Make("point", Repetition::REQUIRED, {x, y}); + auto z = PrimitiveNode::Make("z", Repetition::OPTIONAL, LogicalType::Int(16, true), + ParquetType::INT32); + auto item = GroupNode::Make("element", Repetition::OPTIONAL, {point, z}); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {item}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + arrow_fields.push_back(::arrow::field( + "embedding", + ::arrow::fixed_size_list( + ::arrow::field( + "element", + ::arrow::struct_( + {::arrow::field("point", + ::arrow::struct_({::arrow::field("x", FLOAT, false), + ::arrow::field("y", INT32, true)}), + false), + ::arrow::field("z", ::arrow::int16(), true)}), + true), + 3), + true)); + + ASSERT_OK(ConvertSchema(parquet_fields)); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(::arrow::schema(arrow_fields))); +} + +TEST_F(TestConvertParquetSchema, VectorFixedSizeListStructWithListRejected) { + auto list_element = + PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32); + auto list = GroupNode::Make("list", Repetition::REPEATED, {list_element}); + auto items = + GroupNode::Make("items", Repetition::REQUIRED, {list}, LogicalType::List()); + auto element = GroupNode::Make("element", Repetition::REQUIRED, {items}); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + std::vector parquet_fields = {GroupNode::Make( + "embedding", Repetition::OPTIONAL, {vector}, LogicalType::Vector())}; + + ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); +} + +TEST_F(TestConvertParquetSchema, VectorFixedSizeListStructWithRepeatedPrimitiveRejected) { + auto values = PrimitiveNode::Make("values", Repetition::REPEATED, ParquetType::INT32); + auto item = GroupNode::Make("element", Repetition::REQUIRED, {values}); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {item}, + /*logical_type=*/nullptr, -1, 3); + std::vector parquet_fields = {GroupNode::Make( + "embedding", Repetition::OPTIONAL, {vector}, LogicalType::Vector())}; + + ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); +} + TEST_F(TestConvertParquetSchema, DuplicateFieldNames) { std::vector parquet_fields; std::vector> arrow_fields; @@ -1755,7 +1937,8 @@ TEST_F(TestConvertArrowSchema, ParquetOtherLists) { AddListLikeField(::arrow::list_view(arrow_element)); // LargeListView (list-like non-null, elements nullable) AddListLikeField(::arrow::large_list_view(arrow_element)); - // FixedSizeList[10] (list-like non-null, elements nullable) + // FixedSizeList defaults to the legacy LIST encoding unless experimental VECTOR + // encoding is explicitly enabled. AddListLikeField(::arrow::fixed_size_list(arrow_element, 10)); ASSERT_OK(ConvertSchema(arrow_fields)); @@ -1763,6 +1946,272 @@ TEST_F(TestConvertArrowSchema, ParquetOtherLists) { ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); } +TEST_F(TestConvertArrowSchema, + ParquetFixedSizeListVectorUnsupportedElementFallsBackToList) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto element = PrimitiveNode::Make("element", Repetition::OPTIONAL, + ParquetType::BYTE_ARRAY, ConvertedType::UTF8); + auto list = GroupNode::Make("list", Repetition::REPEATED, {element}); + parquet_fields.push_back( + GroupNode::Make("embedding", Repetition::OPTIONAL, {list}, ConvertedType::LIST)); + + auto arrow_element = ::arrow::field("string", UTF8, true); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorZeroLengthFallsBackToList) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto list = GroupNode::Make("list", Repetition::REPEATED, {element}); + parquet_fields.push_back( + GroupNode::Make("embedding", Repetition::OPTIONAL, {list}, ConvertedType::LIST)); + + auto arrow_element = ::arrow::field("element", FLOAT, false); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 0); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, + ParquetFixedSizeListVectorOuterLengthOverflowFallsBackToList) { + auto inner_type = + ::arrow::fixed_size_list(::arrow::field("element", FLOAT, false), 46341); + auto outer_type = ::arrow::fixed_size_list( + ::arrow::field("inner", inner_type, /*nullable=*/false), 46341); + std::vector> arrow_fields = { + ::arrow::field("embedding", outer_type, /*nullable=*/true)}; + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + + ASSERT_EQ(1, result_schema_->num_columns()); + // The outer FixedSizeList falls back to LIST because the nested VECTOR product + // would overflow, while the inner FixedSizeList can still use VECTOR. + EXPECT_EQ(46341, result_schema_->Column(0)->effective_vector_length()); + const auto* field = result_schema_->group_node()->field(0).get(); + ASSERT_TRUE(field->is_group()); + EXPECT_TRUE(static_cast(field)->logical_type()->is_list()); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorUnderList) { + // FixedSizeList values below a repeated field can use VECTOR encoding. + std::vector parquet_fields; + std::vector> arrow_fields; + + auto inner_element = + PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {inner_element}, + /*logical_type=*/nullptr, -1, 2); + auto element = + GroupNode::Make("element", Repetition::REQUIRED, {vector}, LogicalType::Vector()); + auto list = GroupNode::Make("list", Repetition::REPEATED, {element}); + parquet_fields.push_back( + GroupNode::Make("embeddings", Repetition::OPTIONAL, {list}, ConvertedType::LIST)); + + auto vector_type = + ::arrow::fixed_size_list(::arrow::field("element", INT32, /*nullable=*/false), 2); + arrow_fields.push_back(::arrow::field( + "embeddings", ::arrow::list(::arrow::field("element", vector_type, false)), true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorMixedSupportedAndFallback) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + + auto string_element = PrimitiveNode::Make("element", Repetition::OPTIONAL, + ParquetType::BYTE_ARRAY, ConvertedType::UTF8); + auto string_list = GroupNode::Make("list", Repetition::REPEATED, {string_element}); + parquet_fields.push_back(GroupNode::Make("labels", Repetition::OPTIONAL, {string_list}, + ConvertedType::LIST)); + + arrow_fields.push_back(::arrow::field( + "embedding", ::arrow::fixed_size_list(::arrow::field("element", FLOAT, false), 3), + true)); + arrow_fields.push_back(::arrow::field( + "labels", ::arrow::fixed_size_list(::arrow::field("string", UTF8, true), 3), true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorNullableElement) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto item = PrimitiveNode::Make("element", Repetition::OPTIONAL, ParquetType::FLOAT, + ConvertedType::NONE, -1, -1, -1, -1, -1); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {item}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + + auto arrow_element = ::arrow::field("element", FLOAT, true); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorNullableStructElement) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto x = PrimitiveNode::Make("x", Repetition::REQUIRED, ParquetType::FLOAT, + ConvertedType::NONE, -1, -1, -1, -1, -1); + auto y = PrimitiveNode::Make("y", Repetition::OPTIONAL, ParquetType::INT32, + ConvertedType::NONE, -1, -1, -1, -1, -1); + auto item = GroupNode::Make("element", Repetition::OPTIONAL, {x, y}); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {item}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + + auto arrow_element = ::arrow::field( + "element", + ::arrow::struct_({::arrow::field("x", FLOAT, false), ::arrow::field("y", INT32)}), + true); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorNullableNestedStructElement) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto x = PrimitiveNode::Make("x", Repetition::REQUIRED, ParquetType::FLOAT, + ConvertedType::NONE, -1, -1, -1, -1, -1); + auto y = PrimitiveNode::Make("y", Repetition::OPTIONAL, ParquetType::INT32, + ConvertedType::NONE, -1, -1, -1, -1, -1); + auto point = GroupNode::Make("point", Repetition::REQUIRED, {x, y}); + auto z = PrimitiveNode::Make("z", Repetition::OPTIONAL, LogicalType::Int(16, true), + ParquetType::INT32, -1, -1, -1); + auto item = GroupNode::Make("element", Repetition::OPTIONAL, {point, z}); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {item}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + + auto arrow_element = ::arrow::field( + "element", + ::arrow::struct_( + {::arrow::field("point", + ::arrow::struct_({::arrow::field("x", FLOAT, false), + ::arrow::field("y", INT32, true)}), + false), + ::arrow::field("z", ::arrow::int16(), true)}), + true); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorStructWithListFallsBackToList) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto item = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32); + auto items_list = GroupNode::Make("list", Repetition::REPEATED, {item}); + auto items = + GroupNode::Make("items", Repetition::REQUIRED, {items_list}, ConvertedType::LIST); + auto element = GroupNode::Make("element", Repetition::REQUIRED, {items}); + auto list = GroupNode::Make("list", Repetition::REPEATED, {element}); + parquet_fields.push_back( + GroupNode::Make("embedding", Repetition::OPTIONAL, {list}, ConvertedType::LIST)); + + auto arrow_element = ::arrow::field( + "element", + ::arrow::struct_({::arrow::field( + "items", ::arrow::list(::arrow::field("element", INT32, false)), false)}), + false); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVector) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + + auto arrow_element = ::arrow::field("element", FLOAT, false); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorElementNameCanonical) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + + auto arrow_element = ::arrow::field("value", FLOAT, false); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.disable_compliant_nested_types()->enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + TEST_F(TestConvertArrowSchema, ParquetNestedComplianceEnabledNullable) { std::vector parquet_fields; std::vector> arrow_fields; diff --git a/cpp/src/parquet/arrow/path_internal.cc b/cpp/src/parquet/arrow/path_internal.cc index c626dcbf48b6..54694a55c681 100644 --- a/cpp/src/parquet/arrow/path_internal.cc +++ b/cpp/src/parquet/arrow/path_internal.cc @@ -86,6 +86,7 @@ #include #include +#include #include #include #include @@ -106,6 +107,7 @@ #include "arrow/util/macros.h" #include "arrow/visit_array_inline.h" +#include "parquet/arrow/schema_internal.h" #include "parquet/properties.h" namespace parquet::arrow { @@ -209,6 +211,29 @@ struct PathWriteContext { visited_elements.push_back(range); } + void RecordVectorPostListVisit(const ElementRange& logical_range, + const ElementRange& physical_range) { + if (!visited_elements.empty() && + visited_elements.back().start == logical_range.start && + visited_elements.back().end == logical_range.end) { + visited_elements.back() = physical_range; + return; + } + RecordPostListVisit(physical_range); + } + + // The enclosing list machinery records the logical (vector-index) child + // range before delegating to a VectorNullableNode, which then records the + // physical (slot-index) ranges of its present runs itself. Remove the + // logical range so only physical ranges remain. + void TrimVectorLogicalVisit(const ElementRange& logical_range) { + if (!visited_elements.empty() && + visited_elements.back().start == logical_range.start && + visited_elements.back().end == logical_range.end) { + visited_elements.pop_back(); + } + } + Status last_status; TypedBufferBuilder rep_levels; TypedBufferBuilder def_levels; @@ -475,6 +500,120 @@ struct FixedSizedRangeSelector { int list_size; }; +struct NoLevelTerminalNode { + IterationResult Run(const ElementRange&, PathWriteContext*) { return kDone; } +}; + +IterationResult ExpandLastRepLevels(int64_t logical_count, int32_t multiplier, + int16_t filler_rep_level, PathWriteContext* context); + +class VectorNullableNode { + public: + VectorNullableNode(const uint8_t* null_bitmap, int64_t entry_offset, + int32_t vector_length, int32_t child_slot_multiplier, + int16_t def_level_if_present, int16_t filler_rep_level, + bool child_emits_present_def_levels, + bool child_records_visited_elements, bool expand_rep_levels) + : null_bitmap_(null_bitmap), + entry_offset_(entry_offset), + vector_length_(vector_length), + child_slot_multiplier_(child_slot_multiplier), + valid_bits_reader_(MakeReader(ElementRange{0, 0})), + def_level_if_present_(def_level_if_present), + def_level_if_null_(def_level_if_present - 1), + filler_rep_level_(filler_rep_level), + child_emits_present_def_levels_(child_emits_present_def_levels), + child_records_visited_elements_(child_records_visited_elements), + expand_rep_levels_(expand_rep_levels), + new_range_(true) {} + + ::arrow::internal::BitRunReader MakeReader(const ElementRange& range) { + return ::arrow::internal::BitRunReader(null_bitmap_, entry_offset_ + range.start, + range.Size()); + } + + IterationResult Run(ElementRange* range, ElementRange* child_range, + PathWriteContext* context) { + if (range->Empty()) { + new_range_ = true; + return kDone; + } + if (new_range_) { + context->TrimVectorLogicalVisit(*range); + } + if (expand_rep_levels_ && new_range_) { + // Expand the trailing repetition levels (one per logical vector in this + // delegation, appended by the enclosing list machinery) once, before + // any run processing. Every vector contributes + // vector_length * child_slot_multiplier leaf slots regardless of + // validity, so the multiplier is constant and per-run expansion (which + // would expand positionally wrong entries for interleaved null and + // present runs) is not needed. Only the outermost vector node of a + // nested chain expands. + RETURN_IF_ERROR(ExpandLastRepLevels(range->Size(), + vector_length_ * child_slot_multiplier_, + filler_rep_level_, context)); + } + if (null_bitmap_ == nullptr) { + ElementRange logical_range = *range; + child_range->start = range->start * vector_length_; + child_range->end = child_range->start + range->Size() * vector_length_; + if (!child_records_visited_elements_) { + context->RecordVectorPostListVisit(logical_range, *child_range); + } + if (!child_emits_present_def_levels_) { + RETURN_IF_ERROR( + context->AppendDefLevels(child_range->Size(), def_level_if_present_)); + } + range->start = range->end; + new_range_ = false; + return kNext; + } + if (new_range_) { + valid_bits_reader_ = MakeReader(*range); + } + ::arrow::internal::BitRun run = valid_bits_reader_.NextRun(); + new_range_ = false; + while (!range->Empty() && !run.set) { + range->start += run.length; + RETURN_IF_ERROR(context->AppendDefLevels( + run.length * vector_length_ * child_slot_multiplier_, def_level_if_null_)); + run = valid_bits_reader_.NextRun(); + } + if (range->Empty()) { + new_range_ = true; + return kDone; + } + + ElementRange logical_range{range->start, range->start + run.length}; + child_range->start = range->start * vector_length_; + child_range->end = child_range->start + run.length * vector_length_; + if (!child_records_visited_elements_) { + context->RecordVectorPostListVisit(logical_range, *child_range); + } + if (!child_emits_present_def_levels_) { + RETURN_IF_ERROR( + context->AppendDefLevels(run.length * vector_length_, def_level_if_present_)); + } + range->start += run.length; + return kNext; + } + + private: + const uint8_t* null_bitmap_; + int64_t entry_offset_; + int32_t vector_length_; + int32_t child_slot_multiplier_; + ::arrow::internal::BitRunReader valid_bits_reader_; + int16_t def_level_if_present_; + int16_t def_level_if_null_; + int16_t filler_rep_level_; + bool child_emits_present_def_levels_; + bool child_records_visited_elements_; + bool expand_rep_levels_; + bool new_range_ = true; +}; + // An intermediate node that handles null values. class NullableNode { public: @@ -544,9 +683,10 @@ struct PathInfo { // The vectors are expected to the same length info. // Note index order matters here. - using Node = std::variant; + using Node = + std::variant; std::vector path; std::shared_ptr primitive_array; @@ -554,6 +694,18 @@ struct PathInfo { int16_t max_rep_level = 0; bool has_dictionary = false; bool leaf_is_nullable = false; + bool leaf_is_vector = false; + int32_t leaf_vector_length = 1; + // Definition level after the last repeated ancestor (0 when there is none). + // For VECTOR leaves, definition levels below it mark ancestors that made + // the vector value absent (empty or null lists); they occupy one level + // entry with no slots, while levels at or above it occupy fixed strides of + // leaf_vector_length slots. + int16_t repeated_ancestor_def_level = 0; + // Definition level at or above which a leaf slot carries a value from the + // leaf array (all vector ancestors present); slots below it (but at or + // above the stride level) belong to null vector values. + int16_t vector_values_def_level = 0; }; struct WritePathVisitor { @@ -569,6 +721,12 @@ struct WritePathVisitor { IterationResult operator()(AllNullsTerminalNode& node) { return node.Run(*stack_position, context); } + IterationResult operator()(VectorNullableNode& node) { + return node.Run(stack_position, stack_position + 1, context); + } + IterationResult operator()(NoLevelTerminalNode& node) { + return node.Run(*stack_position, context); + } template IterationResult operator()(ListPathNode& node) { return node.Run(stack_position, stack_position + 1, context); @@ -585,6 +743,53 @@ struct WritePathVisitor { /// values have been calculated for root_range with the calculated /// values. It is intended to abstract the complexity of writing /// the levels and values to parquet. +// Counts the physical leaf slots of a VECTOR leaf: definition levels at or +// above the stride definition level. Entries below it are absent-ancestor +// markers carrying no slot. +int64_t CountVectorSlots(const int16_t* def_levels, int64_t def_count, + int16_t repeated_ancestor_def_level) { + if (repeated_ancestor_def_level <= 0) { + return def_count; + } + int64_t slots = 0; + for (int64_t i = 0; i < def_count; ++i) { + slots += def_levels[i] >= repeated_ancestor_def_level; + } + return slots; +} + +IterationResult ExpandLastRepLevels(int64_t logical_count, int32_t multiplier, + int16_t filler_rep_level, PathWriteContext* context) { + if (multiplier <= 1 || logical_count == 0 || context->rep_levels.length() == 0) { + return kDone; + } + const int64_t old_length = context->rep_levels.length(); + if (ARROW_PREDICT_FALSE(old_length < logical_count)) { + context->last_status = + Status::Invalid("VECTOR repetition level expansion needed ", logical_count, + " source levels but only ", old_length, " were available"); + return kError; + } + const int64_t prefix_length = old_length - logical_count; + // Grow the buffer by the filler entries, then rewrite the trailing + // logical_count entries in place. Entry i moves from prefix_length + i to + // prefix_length + i * multiplier, so a backward walk never overwrites an + // unread source entry and the cost stays linear in the slots written. + context->last_status = + context->rep_levels.Append((multiplier - 1) * logical_count, filler_rep_level); + if (ARROW_PREDICT_FALSE(!context->last_status.ok())) { + return kError; + } + int16_t* data = context->rep_levels.mutable_data(); + for (int64_t i = logical_count - 1; i >= 0; --i) { + const int16_t source = data[prefix_length + i]; + int16_t* destination = data + prefix_length + i * multiplier; + destination[0] = source; + std::fill_n(destination + 1, multiplier - 1, filler_rep_level); + } + return kDone; +} + Status WritePath(ElementRange root_range, PathInfo* path_info, ArrowWriteContext* arrow_context, MultipathLevelBuilder::CallbackFunction writer) { @@ -592,12 +797,18 @@ Status WritePath(ElementRange root_range, PathInfo* path_info, MultipathLevelBuilderResult builder_result; builder_result.leaf_array = path_info->primitive_array; builder_result.leaf_is_nullable = path_info->leaf_is_nullable; + builder_result.leaf_is_vector = path_info->leaf_is_vector; + builder_result.leaf_vector_length = path_info->leaf_vector_length; + builder_result.vector_repeated_ancestor_def_level = + path_info->repeated_ancestor_def_level; + builder_result.vector_values_def_level = path_info->vector_values_def_level; if (path_info->max_def_level == 0) { // This case only occurs when there are no nullable or repeated // columns in the path from the root to leaf. int64_t leaf_length = builder_result.leaf_array->length(); builder_result.def_rep_level_count = leaf_length; + builder_result.leaf_slot_count = leaf_length; builder_result.post_list_visited_elements.push_back({0, leaf_length}); return writer(builder_result); } @@ -640,6 +851,11 @@ Status WritePath(ElementRange root_range, PathInfo* path_info, // This case only occurs when there was a repeated element that needs to be // processed. builder_result.rep_levels = context.rep_levels.data(); + if (path_info->leaf_is_vector) { + builder_result.leaf_slot_count = + CountVectorSlots(context.def_levels.data(), context.def_levels.length(), + path_info->repeated_ancestor_def_level); + } std::swap(builder_result.post_list_visited_elements, context.visited_elements); // If it is possible when processing lists that all lists where empty. In this // case no elements would have been added to post_list_visited_elements. By @@ -648,9 +864,18 @@ Status WritePath(ElementRange root_range, PathInfo* path_info, builder_result.post_list_visited_elements.push_back({0, 0}); } } else { - builder_result.post_list_visited_elements.push_back( - {0, builder_result.leaf_array->length()}); + if (!context.visited_elements.empty()) { + std::swap(builder_result.post_list_visited_elements, context.visited_elements); + } else { + builder_result.post_list_visited_elements.push_back( + {0, builder_result.leaf_array->length()}); + } builder_result.rep_levels = nullptr; + if (path_info->leaf_is_vector) { + builder_result.leaf_slot_count = + CountVectorSlots(context.def_levels.data(), context.def_levels.length(), + path_info->repeated_ancestor_def_level); + } } builder_result.def_levels = context.def_levels.data(); @@ -682,6 +907,7 @@ struct FixupVisitor { } void operator()(NullableNode& arg) { HandleIntermediateNode(arg); } + void operator()(VectorNullableNode&) {} void operator()(AllNullsTerminalNode& arg) { // Even though no processing happens past this point we @@ -692,6 +918,7 @@ struct FixupVisitor { void operator()(NullableTerminalNode&) {} void operator()(AllPresentTerminalNode&) {} + void operator()(NoLevelTerminalNode&) {} }; PathInfo Fixup(PathInfo info) { @@ -713,7 +940,9 @@ PathInfo Fixup(PathInfo info) { class PathBuilder { public: - explicit PathBuilder(bool start_nullable) : nullable_in_parent_(start_nullable) {} + PathBuilder(bool start_nullable, bool write_fixed_size_list_as_vector) + : nullable_in_parent_(start_nullable), + write_fixed_size_list_as_vector_(write_fixed_size_list_as_vector) {} template void AddTerminalInfo(const T& array) { info_.leaf_is_nullable = nullable_in_parent_; @@ -752,13 +981,18 @@ class PathBuilder { // Increment necessary due to empty lists. info_.max_def_level++; info_.max_rep_level++; + info_.repeated_ancestor_def_level = info_.max_def_level; // raw_value_offsets() accounts for any slice offset. ListPathNode> node( VarRangeSelector{array.raw_value_offsets()}, info_.max_rep_level, info_.max_def_level - 1); info_.path.emplace_back(std::move(node)); nullable_in_parent_ = array.list_type()->value_field()->nullable(); - return VisitInline(*array.values()); + const bool saved_nullable_group = nullable_group_since_repeated_; + nullable_group_since_repeated_ = false; + Status status = VisitInline(*array.values()); + nullable_group_since_repeated_ = saved_nullable_group; + return status; } template @@ -768,6 +1002,7 @@ class PathBuilder { // Increment necessary due to empty lists. info_.max_def_level++; info_.max_rep_level++; + info_.repeated_ancestor_def_level = info_.max_def_level; // raw_value_offsets() and raw_value_sizes() account for any slice offset. ListPathNode> node( ListViewRangeSelector{array.raw_value_offsets(), @@ -775,7 +1010,11 @@ class PathBuilder { info_.max_rep_level, info_.max_def_level - 1); info_.path.emplace_back(std::move(node)); nullable_in_parent_ = array.list_view_type()->value_field()->nullable(); - return VisitInline(*array.values()); + const bool saved_nullable_group = nullable_group_since_repeated_; + nullable_group_since_repeated_ = false; + Status status = VisitInline(*array.values()); + nullable_group_since_repeated_ = saved_nullable_group; + return status; } Status Visit(const ::arrow::DictionaryArray& array) { @@ -828,31 +1067,105 @@ class PathBuilder { } Status Visit(const ::arrow::StructArray& array) { + const bool struct_nullable = nullable_in_parent_; MaybeAddNullable(array); PathInfo info_backup = info_; + const bool saved_nullable_group = nullable_group_since_repeated_; + nullable_group_since_repeated_ = nullable_group_since_repeated_ || struct_nullable; for (int x = 0; x < array.num_fields(); x++) { nullable_in_parent_ = array.type()->field(x)->nullable(); RETURN_NOT_OK(VisitInline(*array.field(x))); info_ = info_backup; } + nullable_group_since_repeated_ = saved_nullable_group; return Status::OK(); } Status Visit(const ::arrow::FixedSizeListArray& array) { - MaybeAddNullable(array); int32_t list_size = array.list_type()->list_size(); + auto vector_leaf_slot_multiplier = VectorLeafSlotMultiplier(*array.type()); + if (write_fixed_size_list_as_vector_ && list_size > 0 && + !nullable_group_since_repeated_ && vector_leaf_slot_multiplier.ok() && + IsSupportedVectorElementType(*array.value_type())) { + const bool element_nullable = array.list_type()->value_field()->nullable(); + const bool nested_vector_path = info_.leaf_is_vector; + info_.leaf_is_vector = true; + if (info_.leaf_vector_length > std::numeric_limits::max() / list_size) { + return Status::Invalid("Nested VECTOR leaf slot multiplier overflow"); + } + info_.leaf_vector_length *= list_size; + const bool parent_nullable = nullable_in_parent_; + if (parent_nullable) { + info_.max_def_level++; + } + // Overwritten by nested vectors so that the final value is the present + // level of the innermost vector. + info_.vector_values_def_level = info_.max_def_level; + const bool value_type_is_struct = array.value_type()->id() == ::arrow::Type::STRUCT; + const bool value_type_is_nested_vector = + array.value_type()->id() == ::arrow::Type::FIXED_SIZE_LIST; + const bool child_emits_present_def_levels = + element_nullable || value_type_is_struct || value_type_is_nested_vector || + nested_vector_path; + const bool child_records_visited_elements = value_type_is_nested_vector; + const bool has_vector_node = parent_nullable || element_nullable || + value_type_is_struct || value_type_is_nested_vector || + nested_vector_path || info_.max_rep_level > 0; + const bool vector_node_emits_def_levels = + has_vector_node && !child_emits_present_def_levels; + if (has_vector_node) { + ARROW_ASSIGN_OR_RAISE(int32_t child_slot_multiplier, + VectorLeafSlotMultiplier(*array.value_type())); + info_.path.emplace_back(VectorNullableNode( + parent_nullable ? array.null_bitmap_data() : nullptr, array.offset(), + list_size, child_slot_multiplier, info_.max_def_level, info_.max_rep_level, + child_emits_present_def_levels, child_records_visited_elements, + /*expand_rep_levels=*/!nested_vector_path)); + } + auto values = + array.values()->Slice(array.value_offset(0), array.length() * list_size); + if (!nested_vector_path && !element_nullable && + !::arrow::is_nested(*array.value_type())) { + info_.leaf_is_nullable = false; + info_.primitive_array = values; + if (!vector_node_emits_def_levels && info_.max_rep_level > 0 && + info_.max_def_level > 0) { + info_.path.emplace_back(AllPresentTerminalNode{info_.max_def_level}); + } else { + info_.path.emplace_back(NoLevelTerminalNode{}); + } + paths_.push_back(Fixup(info_)); + return Status::OK(); + } + nullable_in_parent_ = element_nullable; + const bool saved_nullable_group = nullable_group_since_repeated_; + nullable_group_since_repeated_ = false; + Status status = VisitInline(*values); + nullable_group_since_repeated_ = saved_nullable_group; + return status; + } + + MaybeAddNullable(array); // Technically we could encode fixed size lists with two level encodings // but since we always use 3 level encoding we increment def levels as // well. info_.max_def_level++; info_.max_rep_level++; + info_.repeated_ancestor_def_level = info_.max_def_level; + // def_level_if_empty is max_def_level - 1 ("list present but empty"), not + // max_def_level ("element present"); it is only reachable for zero-length + // fixed-size lists, where every present value is an empty list. info_.path.emplace_back(FixedSizeListNode(FixedSizedRangeSelector{list_size}, - info_.max_rep_level, info_.max_def_level)); + info_.max_rep_level, + info_.max_def_level - 1)); nullable_in_parent_ = array.list_type()->value_field()->nullable(); - if (array.offset() > 0) { - return VisitInline(*array.values()->Slice(array.value_offset(0))); - } - return VisitInline(*array.values()); + const bool saved_nullable_group = nullable_group_since_repeated_; + nullable_group_since_repeated_ = false; + Status status = array.offset() > 0 + ? VisitInline(*array.values()->Slice(array.value_offset(0))) + : VisitInline(*array.values()); + nullable_group_since_repeated_ = saved_nullable_group; + return status; } Status Visit(const ::arrow::ExtensionArray& array) { @@ -876,6 +1189,12 @@ class PathBuilder { PathInfo info_; std::vector paths_; bool nullable_in_parent_; + bool write_fixed_size_list_as_vector_; + // True when a nullable group (struct) sits between the nearest repeated + // ancestor (or the root) and the current position. The writer uses LIST for + // such FixedSizeList fields so it does not have to expand null struct markers + // to vector_length leaf slots. + bool nullable_group_since_repeated_ = false; }; Status PathBuilder::VisitInline(const Array& array) { @@ -919,8 +1238,10 @@ class MultipathLevelBuilderImpl : public MultipathLevelBuilder { // static ::arrow::Result> MultipathLevelBuilder::Make( - const ::arrow::Array& array, bool array_field_nullable) { - auto constructor = std::make_unique(array_field_nullable); + const ::arrow::Array& array, bool array_field_nullable, + bool write_fixed_size_list_as_vector) { + auto constructor = std::make_unique(array_field_nullable, + write_fixed_size_list_as_vector); RETURN_NOT_OK(VisitArrayInline(array, constructor.get())); return std::make_unique(array.data(), std::move(constructor)); @@ -930,8 +1251,12 @@ ::arrow::Result> MultipathLevelBuilder::M Status MultipathLevelBuilder::Write(const Array& array, bool array_field_nullable, ArrowWriteContext* context, MultipathLevelBuilder::CallbackFunction callback) { + const bool write_fixed_size_list_as_vector = + context->properties != nullptr && + context->properties->write_fixed_size_list_as_vector(); ARROW_ASSIGN_OR_RAISE(std::unique_ptr builder, - MultipathLevelBuilder::Make(array, array_field_nullable)); + MultipathLevelBuilder::Make(array, array_field_nullable, + write_fixed_size_list_as_vector)); for (int leaf_idx = 0; leaf_idx < builder->GetLeafCount(); leaf_idx++) { RETURN_NOT_OK(builder->Write(leaf_idx, context, callback)); } diff --git a/cpp/src/parquet/arrow/path_internal.h b/cpp/src/parquet/arrow/path_internal.h index 50d2bf24291a..e2158b4ad6af 100644 --- a/cpp/src/parquet/arrow/path_internal.h +++ b/cpp/src/parquet/arrow/path_internal.h @@ -94,6 +94,32 @@ struct MultipathLevelBuilderResult { /// Whether the leaf array is nullable. bool leaf_is_nullable; + + /// Whether this leaf is produced from an Arrow FixedSizeList being written as + /// Parquet VECTOR. For nullable VECTOR elements the physical Parquet leaf is + /// below the VECTOR node, so checking only the primitive schema node is not + /// sufficient. + bool leaf_is_vector = false; + + /// Fixed number of physical leaf slots per logical VECTOR value for this leaf. + int32_t leaf_vector_length = 1; + + /// Number of physical leaf slots for a VECTOR leaf: the definition levels at + /// or above the vector's stride definition level. Equals def_rep_level_count + /// unless ancestors made some vector values absent (empty or null lists, null + /// structs), which contribute one definition level and no slot. + int64_t leaf_slot_count = 0; + + /// Definition level after the last repeated ancestor of a VECTOR leaf (0 + /// when there is none). Definition levels below it mark absent ancestors + /// (empty or null lists) with no slot. + int16_t vector_repeated_ancestor_def_level = 0; + + /// Definition level at or above which a VECTOR leaf slot carries a value + /// from the leaf array; slots between the repeated-ancestor level and this + /// level belong to null vector values at some nesting depth and carry no + /// value. + int16_t vector_values_def_level = 0; }; /// \brief Logic for being able to write out nesting (rep/def level) data that is @@ -132,7 +158,8 @@ class PARQUET_EXPORT MultipathLevelBuilder { /// the array column as nullable (as determined by its type's parent /// field). static ::arrow::Result> Make( - const ::arrow::Array& array, bool array_field_nullable); + const ::arrow::Array& array, bool array_field_nullable, + bool write_fixed_size_list_as_vector = false); virtual ~MultipathLevelBuilder() = default; diff --git a/cpp/src/parquet/arrow/path_internal_test.cc b/cpp/src/parquet/arrow/path_internal_test.cc index 2daa94e6226c..69640f118a87 100644 --- a/cpp/src/parquet/arrow/path_internal_test.cc +++ b/cpp/src/parquet/arrow/path_internal_test.cc @@ -576,6 +576,88 @@ TEST_F(MultipathLevelBuilderTest, TestFixedSizeList) { EXPECT_THAT(results_[0].post_list_elements[0].end, Eq(6)); } +TEST_F(MultipathLevelBuilderTest, TestFixedSizeListExperimentalVector) { + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + arrow_properties_ = builder.build(); + context_ = ArrowWriteContext(default_memory_pool(), arrow_properties_.get()); + + auto entries = field("Entries", ::arrow::int64(), /*nullable=*/false); + auto list_type = fixed_size_list(entries, 2); + auto array = ArrayFromJSON(list_type, "[[1, 2], [3, 4], [5, 6]]"); + + ASSERT_OK( + MultipathLevelBuilder::Write(*array, /*nullable=*/false, &context_, callback_)); + ASSERT_THAT(results_, SizeIs(1)); + EXPECT_TRUE(results_[0].null_rep_levels); + EXPECT_TRUE(results_[0].null_def_levels); + ASSERT_THAT(results_[0].post_list_elements, SizeIs(1)); + EXPECT_EQ(results_[0].post_list_elements[0].start, 0); + EXPECT_EQ(results_[0].post_list_elements[0].end, 6); +} + +TEST_F(MultipathLevelBuilderTest, TestFixedSizeListExperimentalVectorNullableElements) { + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + arrow_properties_ = builder.build(); + context_ = ArrowWriteContext(default_memory_pool(), arrow_properties_.get()); + + auto entries = field("Entries", ::arrow::int64(), /*nullable=*/true); + auto list_type = fixed_size_list(entries, 2); + auto array = ArrayFromJSON(list_type, "[[1, null], [3, 4], [null, 6]]"); + + ASSERT_OK( + MultipathLevelBuilder::Write(*array, /*nullable=*/false, &context_, callback_)); + ASSERT_THAT(results_, SizeIs(1)); + results_[0].CheckLevelsWithNullRepLevels(std::vector{1, 0, 1, 1, 0, 1}); + ASSERT_THAT(results_[0].post_list_elements, SizeIs(1)); + EXPECT_EQ(results_[0].post_list_elements[0].start, 0); + EXPECT_EQ(results_[0].post_list_elements[0].end, 6); +} + +TEST_F(MultipathLevelBuilderTest, + TestFixedSizeListExperimentalVectorNullableRowsAndElements) { + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + arrow_properties_ = builder.build(); + context_ = ArrowWriteContext(default_memory_pool(), arrow_properties_.get()); + + auto entries = field("Entries", ::arrow::int64(), /*nullable=*/true); + auto list_type = fixed_size_list(entries, 2); + auto array = ArrayFromJSON(list_type, "[[1, null], null, [null, 6]]"); + + ASSERT_OK( + MultipathLevelBuilder::Write(*array, /*nullable=*/true, &context_, callback_)); + ASSERT_THAT(results_, SizeIs(1)); + results_[0].CheckLevelsWithNullRepLevels(std::vector{2, 1, 0, 0, 1, 2}); + ASSERT_THAT(results_[0].post_list_elements, SizeIs(2)); + EXPECT_EQ(results_[0].post_list_elements[0].start, 0); + EXPECT_EQ(results_[0].post_list_elements[0].end, 2); + EXPECT_EQ(results_[0].post_list_elements[1].start, 4); + EXPECT_EQ(results_[0].post_list_elements[1].end, 6); +} + +TEST_F(MultipathLevelBuilderTest, TestFixedSizeListExperimentalVectorNullable) { + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + arrow_properties_ = builder.build(); + context_ = ArrowWriteContext(default_memory_pool(), arrow_properties_.get()); + + auto entries = field("Entries", ::arrow::int64(), /*nullable=*/false); + auto list_type = fixed_size_list(entries, 2); + auto array = ArrayFromJSON(list_type, "[[1, 2], null, [5, 6]]"); + + ASSERT_OK( + MultipathLevelBuilder::Write(*array, /*nullable=*/true, &context_, callback_)); + ASSERT_THAT(results_, SizeIs(1)); + results_[0].CheckLevelsWithNullRepLevels(std::vector{1, 1, 0, 0, 1, 1}); + ASSERT_THAT(results_[0].post_list_elements, SizeIs(2)); + EXPECT_EQ(results_[0].post_list_elements[0].start, 0); + EXPECT_EQ(results_[0].post_list_elements[0].end, 2); + EXPECT_EQ(results_[0].post_list_elements[1].start, 4); + EXPECT_EQ(results_[0].post_list_elements[1].end, 6); +} + TEST_F(MultipathLevelBuilderTest, TestFixedSizeListMissingMiddleHasTwoVisitedRanges) { auto entries = field("Entries", ::arrow::int64(), /*nullable=*/false); auto list_type = fixed_size_list(entries, 2); diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc index cc107c1802e3..4a9973be0bf7 100644 --- a/cpp/src/parquet/arrow/reader.cc +++ b/cpp/src/parquet/arrow/reader.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -26,6 +27,8 @@ #include #include "arrow/array.h" +#include "arrow/array/concatenate.h" +#include "arrow/array/util.h" #include "arrow/buffer.h" #include "arrow/extension_type.h" #include "arrow/io/memory.h" @@ -138,6 +141,16 @@ std::shared_ptr> VectorToSharedSet( return result; } +Result MultiplyVectorLength(int64_t value, int32_t list_size) { + if (list_size <= 0) { + return Status::Invalid("VECTOR FixedSizeList must have positive list_size"); + } + if (value > std::numeric_limits::max() / list_size) { + return Status::Invalid("VECTOR FixedSizeList length overflow"); + } + return value * list_size; +} + // Forward declaration Status GetReader(const SchemaField& field, const std::shared_ptr& context, std::unique_ptr* out); @@ -274,10 +287,24 @@ class FileReaderImpl : public FileReader { // TODO(wesm): This calculation doesn't make much sense when we have repeated // schema nodes int64_t records_to_read = 0; + DCHECK(dynamic_cast(reader) != nullptr); + const bool has_repeated_child = + static_cast(reader)->IsOrHasRepeatedChild(); for (auto row_group : row_groups) { - // Can throw exception - records_to_read += - reader_->metadata()->RowGroup(row_group)->ColumnChunk(i)->num_values(); + // Can throw exception. The column chunk metadata is deserialized + // unconditionally to keep upstream behavior (for example its decryption + // side effects for encrypted columns). + auto row_group_metadata = reader_->metadata()->RowGroup(row_group); + auto column_chunk = row_group_metadata->ColumnChunk(i); + // ColumnReader::NextBatch takes logical parent records. For readers + // without repeated children (flat and VECTOR columns) the row group row + // count is an exact bound; for VECTOR columns in particular, ColumnChunk + // num_values would over-count records by the vector length. + if (has_repeated_child) { + records_to_read += column_chunk->num_values(); + } else { + records_to_read += row_group_metadata->num_rows(); + } } #ifdef ARROW_WITH_OPENTELEMETRY std::string column_name = reader_->metadata()->schema()->Column(i)->name(); @@ -739,6 +766,344 @@ class PARQUET_NO_EXPORT FixedSizeListReader : public ListReader { } }; +// Reads Parquet VECTOR columns into Arrow FixedSizeList arrays. +// +// VECTOR stores one definition level per element even though the public Arrow result is a +// single FixedSizeList slot per row. For nullable VECTOR rows, the child reader therefore +// materializes spaced child slots and this reader collapses each vector's per-element def +// levels back into a parent validity bitmap. +class PARQUET_NO_EXPORT VectorFixedSizeListReader : public ColumnReaderImpl { + public: + VectorFixedSizeListReader(std::shared_ptr ctx, + std::shared_ptr field, + ::parquet::internal::LevelInfo level_info, + int16_t vector_stride_def_level, + std::unique_ptr child_reader) + : ctx_(std::move(ctx)), + field_(std::move(field)), + level_info_(level_info), + vector_stride_def_level_(vector_stride_def_level), + item_reader_(std::move(child_reader)), + list_size_(checked_cast(*field_->type()) + .list_size()) {} + + Status GetDefLevels(const int16_t** data, int64_t* length) override { + if (collapsed_def_levels_.empty()) { + *data = nullptr; + *length = vector_rows_; + } else { + *data = collapsed_def_levels_.data(); + *length = static_cast(collapsed_def_levels_.size()); + } + return Status::OK(); + } + + Status GetRepLevels(const int16_t** data, int64_t* length) override { + if (collapsed_rep_levels_.empty()) { + *data = nullptr; + *length = vector_rows_; + } else { + *data = collapsed_rep_levels_.data(); + *length = static_cast(collapsed_rep_levels_.size()); + } + return Status::OK(); + } + + bool IsOrHasRepeatedChild() const final { return false; } + + Status LoadBatch(int64_t number_of_records) final { + vector_rows_ = 0; + collapsed_def_levels_.clear(); + collapsed_rep_levels_.clear(); + vector_def_levels_.clear(); + int64_t child_records_to_read = number_of_records; + if (level_info_.rep_level == 0) { + ARROW_ASSIGN_OR_RAISE(child_records_to_read, + MultiplyVectorLength(number_of_records, list_size_)); + } + RETURN_NOT_OK(item_reader_->LoadBatch(child_records_to_read)); + + const int16_t* def_levels = nullptr; + int64_t num_levels = 0; + RETURN_NOT_OK(item_reader_->GetDefLevels(&def_levels, &num_levels)); + const int16_t* rep_levels = nullptr; + int64_t num_rep_levels = 0; + RETURN_NOT_OK(item_reader_->GetRepLevels(&rep_levels, &num_rep_levels)); + const bool collapse_rep_levels = level_info_.rep_level > 0 && rep_levels != nullptr; + if (collapse_rep_levels && num_rep_levels != num_levels) { + return Status::Invalid("VECTOR child produced ", num_rep_levels, + " repetition levels for ", num_levels, " definition levels"); + } + if (def_levels == nullptr || num_levels == 0) { + // No level information; the row count comes from the child array length. + return Status::OK(); + } + + // Existing vector values (present, or null when the vector field is + // nullable) occupy exactly list_size_ definition levels. Levels below + // the vector stride level mark ancestors that made the vector value absent + // (empty/null lists or null structs); they occupy a single level entry with + // no physical VECTOR slots and pass through unchanged for the parent reader. + const int16_t stride_def_level = vector_stride_def_level_; + const size_t reserve_hint = static_cast(num_levels / list_size_) + 1; + collapsed_def_levels_.reserve(reserve_hint); + vector_def_levels_.reserve(reserve_hint); + if (collapse_rep_levels) { + collapsed_rep_levels_.reserve(reserve_hint); + } + int64_t i = 0; + while (i < num_levels) { + const int16_t def = def_levels[i]; + // Definition levels arrive in long runs (RLE on disk), so scan the + // maximal run of identical levels and process it in bulk. + int64_t run_end = i + 1; + while (run_end < num_levels && def_levels[run_end] == def) { + ++run_end; + } + if (def < stride_def_level) { + // A run of absent-ancestor markers passes through one-to-one. + const int64_t run_length = run_end - i; + collapsed_def_levels_.insert(collapsed_def_levels_.end(), run_length, def); + if (collapse_rep_levels) { + collapsed_rep_levels_.insert(collapsed_rep_levels_.end(), rep_levels + i, + rep_levels + run_end); + } + i = run_end; + continue; + } + if (i + list_size_ > num_levels) { + return Status::Invalid("VECTOR column ended mid-vector: ", num_levels - i, + " definition levels remaining for list_size=", list_size_); + } + const bool is_present = def >= level_info_.def_level; + const int16_t collapsed = is_present ? level_info_.def_level : def; + const int64_t run_vectors = (run_end - i) / list_size_; + if (run_vectors > 0) { + // Whole vectors covered by one run of equal levels: equal levels imply + // uniform parent validity, so the per-slot validation is subsumed. + collapsed_def_levels_.insert(collapsed_def_levels_.end(), run_vectors, collapsed); + vector_def_levels_.insert(vector_def_levels_.end(), run_vectors, collapsed); + if (collapse_rep_levels) { + for (int64_t v = 0; v < run_vectors; ++v) { + collapsed_rep_levels_.push_back(rep_levels[i + v * list_size_]); + } + } + vector_rows_ += run_vectors; + i += run_vectors * list_size_; + continue; + } + // The run ends inside this vector (for example nullable elements mix + // definition levels); validate parent-validity uniformity per slot. + for (int32_t k = 1; k < list_size_; ++k) { + const bool slot_is_present = def_levels[i + k] >= level_info_.def_level; + if (slot_is_present != is_present) { + return Status::Invalid( + "VECTOR parent validity changed within one fixed-size vector at row ", + vector_rows_, + "; null VECTOR rows must still emit exactly list_size null child " + "slots and all slots for one parent must agree on parent " + "validity"); + } + } + collapsed_def_levels_.push_back(collapsed); + vector_def_levels_.push_back(collapsed); + if (collapse_rep_levels) { + collapsed_rep_levels_.push_back(rep_levels[i]); + } + ++vector_rows_; + i += list_size_; + } + return Status::OK(); + } + + Status BuildArray(int64_t length_upper_bound, + std::shared_ptr* out) override { + std::shared_ptr child_out; + ARROW_ASSIGN_OR_RAISE(const int64_t child_length_upper_bound, + MultiplyVectorLength(length_upper_bound, list_size_)); + RETURN_NOT_OK(item_reader_->BuildArray(child_length_upper_bound, &child_out)); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr child_data, + ChunksToSingle(*child_out)); + std::shared_ptr child_array = ::arrow::MakeArray(child_data); + if (vector_rows_ == 0 && vector_def_levels_.empty()) { + // No level information was loaded; derive the row count from the child. + if (child_array->length() % list_size_ != 0) { + return Status::Invalid("VECTOR FixedSizeList child length ", + child_array->length(), + " was not divisible by list_size=", list_size_); + } + vector_rows_ = child_array->length() / list_size_; + } + + int64_t output_rows = vector_rows_; + std::vector expanded_def_levels; + // Nullable struct ancestors can contribute a single absent level marker for + // a VECTOR value whose Arrow parent still requires child slots. Expand such + // markers here, while leaving empty/null list markers in their compact form. + const std::vector* output_def_levels = &vector_def_levels_; + int64_t expandable_absent_count = 0; + for (const int16_t def_level : collapsed_def_levels_) { + if (def_level >= level_info_.repeated_ancestor_def_level && + def_level < vector_stride_def_level_) { + ++expandable_absent_count; + } + } + // The expansion only triggers below struct (or root) parents: a list + // parent requests exactly the existing vector values, so + // length_upper_bound never exceeds vector_rows_ there, while a struct + // parent requests one output row per logical entry including nullable + // struct markers. + const bool expand_absent_ancestors = + expandable_absent_count > 0 && length_upper_bound > vector_rows_; + if (expand_absent_ancestors) { + const int64_t extra_rows = length_upper_bound - vector_rows_; + if (extra_rows % expandable_absent_count != 0) { + return Status::Invalid("VECTOR FixedSizeList cannot evenly expand ", + expandable_absent_count, " absent ancestor markers to ", + extra_rows, " child rows"); + } + const int64_t rows_per_absent = extra_rows / expandable_absent_count; + // The child array may take one of three shapes, depending on the child + // reader that produced it: + // - exactly the slots of existing vectors (a nested VECTOR reader that + // saw no expandable markers); + // - one placeholder slot per expandable marker in addition (a leaf reader + // without repeated ancestors materializes a slot for every definition + // level entry); + // - rows_per_absent * list_size_ null slots per expandable marker in + // addition (a nested reader that has itself expanded the markers). + ARROW_ASSIGN_OR_RAISE(const int64_t expected_child_slots, + MultiplyVectorLength(vector_rows_, list_size_)); + if (expected_child_slots > + std::numeric_limits::max() - expandable_absent_count) { + return Status::Invalid("VECTOR FixedSizeList length overflow"); + } + const int64_t expected_child_slots_with_placeholders = + expected_child_slots + expandable_absent_count; + ARROW_ASSIGN_OR_RAISE(const int64_t materialized_absent_slots, + MultiplyVectorLength(extra_rows, list_size_)); + if (expected_child_slots > + std::numeric_limits::max() - materialized_absent_slots) { + return Status::Invalid("VECTOR FixedSizeList length overflow"); + } + const int64_t expected_child_slots_with_materialized_absent = + expected_child_slots + materialized_absent_slots; + const bool child_has_absent_placeholders = + child_array->length() == expected_child_slots_with_placeholders; + const bool child_has_materialized_absent_slots = + child_array->length() == expected_child_slots_with_materialized_absent; + if (child_array->length() != expected_child_slots && + !child_has_absent_placeholders && !child_has_materialized_absent_slots) { + return Status::Invalid("VECTOR FixedSizeList child length ", + child_array->length(), " did not match expected ", + expected_child_slots); + } + + ::arrow::ArrayVector parts; + parts.reserve(static_cast(length_upper_bound)); + expanded_def_levels.reserve(static_cast(length_upper_bound)); + ARROW_ASSIGN_OR_RAISE(const int64_t null_slot_count, + MultiplyVectorLength(rows_per_absent, list_size_)); + int64_t child_offset = 0; + for (const int16_t def_level : collapsed_def_levels_) { + if (def_level < vector_stride_def_level_) { + if (def_level < level_info_.repeated_ancestor_def_level) { + continue; + } + if (child_has_absent_placeholders) { + ++child_offset; + } else if (child_has_materialized_absent_slots) { + child_offset += null_slot_count; + } + ARROW_ASSIGN_OR_RAISE( + auto null_values, + ::arrow::MakeArrayOfNull(child_array->type(), null_slot_count, ctx_->pool)); + parts.push_back(std::move(null_values)); + expanded_def_levels.insert(expanded_def_levels.end(), rows_per_absent, + def_level); + } else { + parts.push_back(child_array->Slice(child_offset, list_size_)); + child_offset += list_size_; + expanded_def_levels.push_back(def_level); + } + } + if (child_offset != child_array->length()) { + return Status::Invalid( + "VECTOR FixedSizeList child values out of sync with " + "definition levels"); + } + ARROW_ASSIGN_OR_RAISE(child_array, ::arrow::Concatenate(parts, ctx_->pool)); + child_data = child_array->data(); + output_rows = length_upper_bound; + output_def_levels = &expanded_def_levels; + } else { + ARROW_ASSIGN_OR_RAISE(const int64_t expected_child_slots, + MultiplyVectorLength(vector_rows_, list_size_)); + if (child_array->length() != expected_child_slots) { + return Status::Invalid("VECTOR FixedSizeList child length ", + child_array->length(), " did not match expected ", + expected_child_slots); + } + } + + ARROW_ASSIGN_OR_RAISE(const int64_t expected_output_child_slots, + MultiplyVectorLength(output_rows, list_size_)); + if (child_array->length() != expected_output_child_slots) { + return Status::Invalid("VECTOR FixedSizeList child length ", child_array->length(), + " did not match expected ", expected_output_child_slots); + } + + std::shared_ptr validity_buffer; + int64_t null_count = 0; + if (field_->nullable()) { + ARROW_ASSIGN_OR_RAISE( + validity_buffer, + AllocateResizableBuffer(bit_util::BytesForBits(output_rows), ctx_->pool)); + memset(validity_buffer->mutable_data(), 0, + static_cast(bit_util::BytesForBits(output_rows))); + if (output_def_levels->empty()) { + bit_util::SetBitsTo(validity_buffer->mutable_data(), 0, output_rows, true); + } else { + for (int64_t row = 0; row < output_rows; ++row) { + if ((*output_def_levels)[row] == level_info_.def_level) { + bit_util::SetBit(validity_buffer->mutable_data(), row); + } else { + ++null_count; + } + } + } + validity_buffer->ZeroPadding(); + } + + auto data = std::make_shared( + field_->type(), output_rows, + std::vector>{null_count > 0 ? validity_buffer : nullptr}, + null_count); + data->child_data.push_back(child_data); + *out = std::make_shared(::arrow::MakeArray(std::move(data))); + return Status::OK(); + } + + const std::shared_ptr field() override { return field_; } + + private: + std::shared_ptr ctx_; + std::shared_ptr field_; + ::parquet::internal::LevelInfo level_info_; + int16_t vector_stride_def_level_; + std::unique_ptr item_reader_; + int32_t list_size_; + // Number of existing vector values (present or null); absent ancestors do + // not count. + int64_t vector_rows_ = 0; + // One entry per logical entry (existing vector or absent ancestor), for the + // parent reader. + std::vector collapsed_def_levels_; + std::vector collapsed_rep_levels_; + // One entry per existing vector value, for this reader's validity bitmap. + std::vector vector_def_levels_; +}; + class PARQUET_NO_EXPORT StructReader : public ColumnReaderImpl { public: explicit StructReader(std::shared_ptr ctx, @@ -917,8 +1282,12 @@ Status GetReader(const SchemaField& field, const std::shared_ptr& arrow_f } std::unique_ptr input( ctx->iterator_factory(field.column_index, ctx->reader)); - *out = std::make_unique(ctx, arrow_field, std::move(input), - field.level_info); + auto leaf_field = arrow_field; + if (field.is_vector && field.level_info.def_level > 0) { + leaf_field = leaf_field->WithNullable(true); + } + *out = + std::make_unique(ctx, leaf_field, std::move(input), field.level_info); } else if (type_id == ::arrow::Type::LIST || type_id == ::arrow::Type::MAP || type_id == ::arrow::Type::FIXED_SIZE_LIST || type_id == ::arrow::Type::LARGE_LIST || @@ -999,8 +1368,17 @@ Status GetReader(const SchemaField& field, const std::shared_ptr& arrow_f list_field->WithType(::arrow::fixed_size_list(reader_child_type, list_size)); } - *out = std::make_unique(ctx, list_field, field.level_info, - std::move(child_reader)); + if (field.is_vector) { + if (child->is_leaf() && child->column_index >= 0 && list_field->nullable()) { + DCHECK(child_reader->field()->nullable()); + } + *out = std::make_unique( + ctx, list_field, field.level_info, field.vector_stride_def_level, + std::move(child_reader)); + } else { + *out = std::make_unique(ctx, list_field, field.level_info, + std::move(child_reader)); + } } else { return Status::UnknownError("Unknown list type: ", field.field->ToString()); } diff --git a/cpp/src/parquet/arrow/reader_writer_benchmark.cc b/cpp/src/parquet/arrow/reader_writer_benchmark.cc index 2f288fd2eb0f..63d196892ece 100644 --- a/cpp/src/parquet/arrow/reader_writer_benchmark.cc +++ b/cpp/src/parquet/arrow/reader_writer_benchmark.cc @@ -812,5 +812,84 @@ static void BM_ReadMultipleRowGroupsGenerator(::benchmark::State& state) { BENCHMARK(BM_ReadMultipleRowGroupsGenerator); +// +// Benchmark writing and reading fixed-size list (embedding) columns with the +// experimental VECTOR encoding versus the standard LIST encoding. +// + +constexpr int32_t kVectorBenchmarkDim = 128; + +static std::shared_ptr
MakeFixedSizeListTable(bool nullable) { + const int64_t num_rows = BENCHMARK_SIZE / kVectorBenchmarkDim; + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::float32(), /*nullable=*/false), + kVectorBenchmarkDim); + ::arrow::random::RandomArrayGenerator generator(/*seed=*/42); + auto array = generator.ArrayOf(type, num_rows, nullable ? 0.05 : 0.0); + auto schema = ::arrow::schema({::arrow::field("column", type, nullable)}); + return Table::Make(schema, {array}); +} + +static std::shared_ptr VectorBenchmarkArrowProperties( + bool use_vector) { + ArrowWriterProperties::Builder builder; + builder.store_schema(); + if (use_vector) { + builder.enable_experimental_vector_encoding(); + } + return builder.build(); +} + +template +static void BM_WriteFixedSizeListColumn(::benchmark::State& state) { + auto table = MakeFixedSizeListTable(nullable); + auto arrow_properties = VectorBenchmarkArrowProperties(use_vector); + + while (state.KeepRunning()) { + auto output = CreateOutputStream(); + EXIT_NOT_OK(WriteTable(*table, ::arrow::default_memory_pool(), output, + table->num_rows(), default_writer_properties(), + arrow_properties)); + } + state.SetBytesProcessed(BENCHMARK_SIZE * sizeof(float) * state.iterations()); + state.SetItemsProcessed(table->num_rows() * state.iterations()); +} + +BENCHMARK_TEMPLATE2(BM_WriteFixedSizeListColumn, false, false); +BENCHMARK_TEMPLATE2(BM_WriteFixedSizeListColumn, true, false); +// There is no nullable LIST-encoding baseline: writing null FixedSizeList +// rows through the LIST encoding is NotImplemented upstream ("Lists with +// non-zero length null components"); the VECTOR encoding supports them. +BENCHMARK_TEMPLATE2(BM_WriteFixedSizeListColumn, true, true); + +template +static void BM_ReadFixedSizeListColumn(::benchmark::State& state) { + auto table = MakeFixedSizeListTable(nullable); + auto arrow_properties = VectorBenchmarkArrowProperties(use_vector); + + auto output = CreateOutputStream(); + EXIT_NOT_OK(WriteTable(*table, ::arrow::default_memory_pool(), output, + table->num_rows(), default_writer_properties(), + arrow_properties)); + PARQUET_ASSIGN_OR_THROW(auto buffer, output->Finish()); + + for (auto _ : state) { + auto reader = + ParquetFileReader::Open(std::make_shared<::arrow::io::BufferReader>(buffer)); + auto arrow_reader_result = + FileReader::Make(::arrow::default_memory_pool(), std::move(reader)); + EXIT_NOT_OK(arrow_reader_result.status()); + auto arrow_reader = std::move(*arrow_reader_result); + auto table_result = arrow_reader->ReadTable(); + EXIT_NOT_OK(table_result.status()); + } + state.SetBytesProcessed(BENCHMARK_SIZE * sizeof(float) * state.iterations()); + state.SetItemsProcessed(table->num_rows() * state.iterations()); +} + +BENCHMARK_TEMPLATE2(BM_ReadFixedSizeListColumn, false, false); +BENCHMARK_TEMPLATE2(BM_ReadFixedSizeListColumn, true, false); +BENCHMARK_TEMPLATE2(BM_ReadFixedSizeListColumn, true, true); + } // namespace benchmarks } // namespace parquet diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc index bc4de6c39b5e..2d1b7b481342 100644 --- a/cpp/src/parquet/arrow/schema.cc +++ b/cpp/src/parquet/arrow/schema.cc @@ -31,6 +31,7 @@ #include "arrow/result.h" #include "arrow/status.h" #include "arrow/type.h" +#include "arrow/type_traits.h" #include "arrow/util/base64.h" #include "arrow/util/checked_cast.h" #include "arrow/util/key_value_metadata.h" @@ -92,9 +93,14 @@ Result> MakeArrowList( } } +// nullable_group_since_repeated is true when a nullable group (struct) sits +// between the nearest repeated ancestor (or the root) and the field being +// converted. The writer uses LIST for such FixedSizeList fields so it does +// not have to expand null struct markers to vector_length leaf slots. Status FieldToNode(const std::string& name, const std::shared_ptr& field, const WriterProperties& properties, - const ArrowWriterProperties& arrow_properties, NodePtr* out); + const ArrowWriterProperties& arrow_properties, NodePtr* out, + bool nullable_group_since_repeated = false); Status ListToNode(const std::shared_ptr<::arrow::BaseListType>& type, const std::string& name, bool nullable, int field_id, @@ -112,6 +118,88 @@ Status ListToNode(const std::shared_ptr<::arrow::BaseListType>& type, return Status::OK(); } +bool IsSupportedVectorStructNode(const Node& node) { + if (node.is_repeated() || node.is_vector()) { + return false; + } + if (node.is_primitive()) { + return true; + } + if (!node.is_group()) { + return false; + } + const auto& group = checked_cast(node); + if (group.logical_type() != nullptr && !group.logical_type()->is_none()) { + return false; + } + for (int i = 0; i < group.field_count(); ++i) { + if (!IsSupportedVectorStructNode(*group.field(i))) { + return false; + } + } + return true; +} + +Status ValidateSupportedVectorStructNode(const Node& node) { + if (IsSupportedVectorStructNode(node)) { + return Status::OK(); + } + return Status::Invalid( + "VECTOR element subtrees must not contain repeated, list, or map fields"); +} + +// Option B mapping for Arrow FixedSizeList -> Parquet VECTOR. +// +// The schema always uses a 3-level structure mirroring LIST, regardless of +// vector and element nullability: +// +// group (VECTOR) { +// vector group list [N] { +// element; +// } +// } +// +// VECTOR is a structural repetition type: vector nullability is carried by the +// outer group, the VECTOR-repeated middle group carries vector_length and +// contributes no repetition/definition level, and element nullability is +// carried by the element node. The outer group is annotated with the VECTOR +// logical type to make it explicit that it represents a vector value rather +// than, e.g., a single-field struct containing a vector, mirroring how LIST +// annotates list-shaped groups. +Status FixedSizeListToNode(const std::shared_ptr<::arrow::FixedSizeListType>& type, + const std::string& name, bool nullable, int field_id, + const WriterProperties& properties, + const ArrowWriterProperties& arrow_properties, NodePtr* out) { + if (type->list_size() <= 0) { + return Status::NotImplemented( + "VECTOR repetition does not support zero-length FixedSizeList values"); + } + if (!IsSupportedVectorElementType(*type->value_type())) { + return Status::NotImplemented( + "VECTOR repetition only supports fixed-width primitive, nested FixedSizeList, " + "or struct FixedSizeList elements"); + } + auto leaf_slot_multiplier = VectorLeafSlotMultiplier(*type); + if (!leaf_slot_multiplier.ok()) { + return Status::NotImplemented(leaf_slot_multiplier.status().message()); + } + const auto& value_field = type->value_field(); + const std::string value_name = "element"; + + NodePtr element; + RETURN_NOT_OK( + FieldToNode(value_name, value_field, properties, arrow_properties, &element)); + if (type->value_type()->id() == ::arrow::Type::STRUCT) { + RETURN_NOT_OK(ValidateSupportedVectorStructNode(*element)); + } + NodePtr vector = + GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, /*field_id=*/-1, type->list_size()); + *out = GroupNode::Make(name, RepetitionFromNullable(nullable), {vector}, + LogicalType::Vector(), field_id); + return Status::OK(); +} + Status MapToNode(const std::shared_ptr<::arrow::MapType>& type, const std::string& name, bool nullable, int field_id, const WriterProperties& properties, const ArrowWriterProperties& arrow_properties, NodePtr* out) { @@ -154,12 +242,14 @@ Status VariantToNode( Status StructToNode(const std::shared_ptr<::arrow::StructType>& type, const std::string& name, bool nullable, int field_id, const WriterProperties& properties, - const ArrowWriterProperties& arrow_properties, NodePtr* out) { + const ArrowWriterProperties& arrow_properties, NodePtr* out, + bool nullable_group_since_repeated) { std::vector children(type->num_fields()); if (type->num_fields() != 0) { for (int i = 0; i < type->num_fields(); i++) { RETURN_NOT_OK(FieldToNode(type->field(i)->name(), type->field(i), properties, - arrow_properties, &children[i])); + arrow_properties, &children[i], + nullable_group_since_repeated || nullable)); } } else { // XXX (ARROW-10928) We could add a dummy primitive node but that would @@ -316,7 +406,8 @@ int FieldIdFromMetadata( Status FieldToNode(const std::string& name, const std::shared_ptr& field, const WriterProperties& properties, - const ArrowWriterProperties& arrow_properties, NodePtr* out) { + const ArrowWriterProperties& arrow_properties, NodePtr* out, + bool nullable_group_since_repeated) { std::shared_ptr logical_type = LogicalType::None(); ParquetType::type type; Repetition::type repetition = RepetitionFromNullable(field->nullable()); @@ -452,9 +543,34 @@ Status FieldToNode(const std::string& name, const std::shared_ptr& field, case ArrowTypeId::STRUCT: { auto struct_type = std::static_pointer_cast<::arrow::StructType>(field->type()); return StructToNode(struct_type, name, field->nullable(), field_id, properties, - arrow_properties, out); + arrow_properties, out, nullable_group_since_repeated); + } + case ArrowTypeId::FIXED_SIZE_LIST: { + auto list_type = + std::static_pointer_cast<::arrow::FixedSizeListType>(field->type()); + // Experimental VECTOR encoding is opportunistic: use it for supported + // FixedSizeList fields, and preserve writability by falling back to the + // standard LIST encoding for unsupported fields. This lets schemas mix + // VECTOR-friendly dense numeric/struct vectors with FixedSizeList values + // that still need LIST machinery (for example strings). + if (arrow_properties.write_fixed_size_list_as_vector() && + !nullable_group_since_repeated && list_type->list_size() > 0 && + IsSupportedVectorElementType(*list_type->value_type())) { + Status vector_status = + FixedSizeListToNode(list_type, name, field->nullable(), field_id, properties, + arrow_properties, out); + if (vector_status.ok()) { + return vector_status; + } + if (!vector_status.IsNotImplemented()) { + return vector_status; + } + } + auto base_list_type = + std::static_pointer_cast<::arrow::BaseListType>(field->type()); + return ListToNode(base_list_type, name, field->nullable(), field_id, properties, + arrow_properties, out); } - case ArrowTypeId::FIXED_SIZE_LIST: case ArrowTypeId::LARGE_LIST: case ArrowTypeId::LIST: case ArrowTypeId::LARGE_LIST_VIEW: @@ -470,7 +586,8 @@ Status FieldToNode(const std::string& name, const std::shared_ptr& field, static_cast(*field->type()); std::shared_ptr<::arrow::Field> unpacked_field = ::arrow::field( name, dict_type.value_type(), field->nullable(), field->metadata()); - return FieldToNode(name, unpacked_field, properties, arrow_properties, out); + return FieldToNode(name, unpacked_field, properties, arrow_properties, out, + nullable_group_since_repeated); } case ArrowTypeId::EXTENSION: { auto ext_type = std::static_pointer_cast<::arrow::ExtensionType>(field->type()); @@ -501,7 +618,8 @@ Status FieldToNode(const std::string& name, const std::shared_ptr& field, std::shared_ptr<::arrow::Field> storage_field = ::arrow::field( name, ext_type->storage_type(), field->nullable(), field->metadata()); - return FieldToNode(name, storage_field, properties, arrow_properties, out); + return FieldToNode(name, storage_field, properties, arrow_properties, out, + nullable_group_since_repeated); } case ArrowTypeId::MAP: { auto map_type = std::static_pointer_cast<::arrow::MapType>(field->type()); @@ -577,6 +695,14 @@ Status PopulateLeaf(int column_index, const std::shared_ptr& field, return Status::OK(); } +void MarkVectorSubtree(SchemaField* field, int16_t vector_stride_def_level) { + field->is_vector = true; + field->vector_stride_def_level = vector_stride_def_level; + for (auto& child : field->children) { + MarkVectorSubtree(&child, vector_stride_def_level); + } +} + // Special case mentioned in the format spec: // If the name is array or uses the parent's name with `_tuple` appended, // this should be: @@ -602,7 +728,7 @@ Status GroupToStruct(const GroupNode& node, LevelInfo current_levels, arrow_fields.push_back(out->children[i].field); } auto struct_type = ::arrow::struct_(arrow_fields); - if (ctx->properties.get_arrow_extensions_enabled() && + if (ctx->properties.get_arrow_extensions_enabled() && node.logical_type() != nullptr && node.logical_type()->is_variant()) { auto extension_type = ::arrow::GetExtensionType("arrow.parquet.variant"); if (extension_type) { @@ -621,6 +747,113 @@ Status ListToSchemaField(const GroupNode& group, LevelInfo current_levels, SchemaTreeContext* ctx, const SchemaField* parent, SchemaField* out); +bool IsVectorGroup(const GroupNode& group) { + // The VECTOR logical type annotation is required to identify a vector group; + // this is what distinguishes a vector field from a struct-like group that + // happens to contain a VECTOR-repeated child. An unannotated group with a + // VECTOR-repeated child is rejected during conversion (see + // GroupToSchemaField) rather than silently interpreted as a vector. + return group.logical_type() != nullptr && group.logical_type()->is_vector(); +} + +// Reconstructs an Arrow FixedSizeList from the canonical 3-level vector form: +// +// group (VECTOR) { +// vector group list [N] { +// element; +// } +// } +// +// The element may be a primitive, a struct group, or a nested vector group +// (itself in canonical form). The middle group and element field names are +// part of the canonical VECTOR layout and are validated. +Status VectorToSchemaField(const GroupNode& group, LevelInfo current_levels, + SchemaTreeContext* ctx, const SchemaField* parent, + SchemaField* out) { + if (group.is_repeated() || group.is_vector()) { + return Status::Invalid("VECTOR logical groups must be required or optional"); + } + if (group.field_count() != 1) { + return Status::Invalid("VECTOR groups must have a single VECTOR-repeated child"); + } + const Node& child = *group.field(0); + if (!child.is_vector()) { + return Status::Invalid("VECTOR groups must contain a VECTOR-repeated child"); + } + if (!child.is_group()) { + return Status::Invalid( + "VECTOR-repeated nodes must be groups containing a single element field"); + } + const auto& vector_group = static_cast(child); + if (vector_group.name() != "list") { + return Status::Invalid("VECTOR-repeated middle groups must be named 'list'"); + } + if (vector_group.logical_type() != nullptr && !vector_group.logical_type()->is_none()) { + return Status::Invalid("VECTOR-repeated groups must not have a logical type"); + } + if (vector_group.vector_length() <= 0) { + return Status::Invalid("VECTOR-repeated groups must have positive vector_length"); + } + if (vector_group.field_count() != 1) { + return Status::Invalid("VECTOR-repeated groups must have a single element child"); + } + const Node& element = *vector_group.field(0); + if (element.name() != "element") { + return Status::Invalid("VECTOR element fields must be named 'element'"); + } + if (element.is_repeated() || element.is_vector()) { + return Status::Invalid("VECTOR elements must be required or optional"); + } + + const int16_t vector_stride_def_level = current_levels.def_level; + if (group.is_optional()) { + current_levels.IncrementOptional(); + } + const LevelInfo vector_level = current_levels; + + out->children.resize(1); + SchemaField* child_field = &out->children[0]; + ctx->LinkParent(out, parent); + ctx->LinkParent(child_field, out); + + if (element.is_primitive()) { + const bool element_nullable = element.is_optional(); + if (element_nullable) { + current_levels.IncrementOptional(); + } + const auto& primitive_node = static_cast(element); + int column_index = ctx->schema->GetColumnIndex(primitive_node); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr type, + GetTypeForNode(column_index, primitive_node, ctx)); + auto item_field = ::arrow::field(element.name(), type, element_nullable, + FieldIdMetadata(element.field_id())); + RETURN_NOT_OK( + PopulateLeaf(column_index, item_field, current_levels, ctx, out, child_field)); + } else { + const auto& element_group = static_cast(element); + if (IsVectorGroup(element_group)) { + // Nested vector element; the recursive call handles element nullability. + RETURN_NOT_OK( + VectorToSchemaField(element_group, current_levels, ctx, out, child_field)); + } else { + RETURN_NOT_OK(ValidateSupportedVectorStructNode(element_group)); + if (element_group.is_optional()) { + current_levels.IncrementOptional(); + } + RETURN_NOT_OK(GroupToStruct(element_group, current_levels, ctx, out, child_field)); + } + } + + MarkVectorSubtree(child_field, vector_stride_def_level); + out->field = ::arrow::field( + group.name(), ::arrow::fixed_size_list(child_field->field, child.vector_length()), + group.is_optional(), FieldIdMetadata(group.field_id())); + out->level_info = vector_level; + out->is_vector = true; + out->vector_stride_def_level = vector_stride_def_level; + return Status::OK(); +} + Status MapToSchemaField(const GroupNode& group, LevelInfo current_levels, SchemaTreeContext* ctx, const SchemaField* parent, SchemaField* out) { @@ -845,6 +1078,14 @@ Status ListToSchemaField(const GroupNode& group, LevelInfo current_levels, Status GroupToSchemaField(const GroupNode& node, LevelInfo current_levels, SchemaTreeContext* ctx, const SchemaField* parent, SchemaField* out) { + if (IsVectorGroup(node)) { + return VectorToSchemaField(node, current_levels, ctx, parent, out); + } + if (node.is_vector()) { + return Status::Invalid( + "VECTOR-repeated groups must be the single child of a group annotated with " + "the VECTOR logical type"); + } if (node.logical_type()->is_list()) { return ListToSchemaField(node, current_levels, ctx, parent, out); } else if (node.logical_type()->is_map()) { @@ -906,6 +1147,13 @@ Status NodeToSchemaField(const Node& node, LevelInfo current_levels, int column_index = ctx->schema->GetColumnIndex(primitive_node); ARROW_ASSIGN_OR_RAISE(std::shared_ptr type, GetTypeForNode(column_index, primitive_node, ctx)); + if (node.is_vector()) { + // VECTOR repetition directly on a primitive is not part of the canonical + // 3-level vector structure; require the VECTOR group form instead. + return Status::Invalid( + "VECTOR repetition on primitive nodes is not supported; vectors must use " + "the 3-level VECTOR group structure"); + } if (node.is_repeated()) { // One-level list encoding, e.g. // a: repeated int32; @@ -1004,6 +1252,20 @@ std::function(FieldVector)> GetNestedFactory( return [](FieldVector fields) { return ::arrow::struct_(std::move(fields)); }; } break; + case ::arrow::Type::FIXED_SIZE_LIST: + if (origin_type.id() == ::arrow::Type::FIXED_SIZE_LIST) { + const auto origin_list_size = + checked_cast(origin_type).list_size(); + const auto inferred_list_size = + checked_cast(inferred_type).list_size(); + if (origin_list_size == inferred_list_size) { + return [origin_list_size](FieldVector fields) { + DCHECK_EQ(fields.size(), 1); + return ::arrow::fixed_size_list(std::move(fields[0]), origin_list_size); + }; + } + } + break; case ::arrow::Type::LIST: case ::arrow::Type::LARGE_LIST: { switch (origin_type.id()) { diff --git a/cpp/src/parquet/arrow/schema.h b/cpp/src/parquet/arrow/schema.h index dd60fde43422..f8056545f4e7 100644 --- a/cpp/src/parquet/arrow/schema.h +++ b/cpp/src/parquet/arrow/schema.h @@ -96,6 +96,14 @@ struct PARQUET_EXPORT SchemaField { parquet::internal::LevelInfo level_info; + // True when this Arrow field is backed by a Parquet VECTOR logical group. + bool is_vector = false; + + // For VECTOR fields, the definition level at or above which a vector value + // exists (present or null). Definition levels below this are absent + // ancestors, such as null structs or empty/null lists, and carry no slots. + int16_t vector_stride_def_level = 0; + bool is_leaf() const { return column_index != -1; } }; diff --git a/cpp/src/parquet/arrow/schema_internal.cc b/cpp/src/parquet/arrow/schema_internal.cc index 2e8cf764b27f..c930b70bf9e5 100644 --- a/cpp/src/parquet/arrow/schema_internal.cc +++ b/cpp/src/parquet/arrow/schema_internal.cc @@ -17,9 +17,12 @@ #include "parquet/arrow/schema_internal.h" +#include + #include "arrow/extension/json.h" #include "arrow/extension/uuid.h" #include "arrow/type.h" +#include "arrow/type_traits.h" #include "arrow/util/key_value_metadata.h" #include "arrow/util/logging.h" #include "arrow/util/string.h" @@ -39,6 +42,78 @@ using ::arrow::internal::checked_cast; namespace { +// Whether a type may appear inside a struct VECTOR element. This must mirror +// the writer-side schema validation (ValidateSupportedVectorStructNode): +// list/map descendants require repetition levels and force the whole column +// back to the standard LIST encoding. Both the schema conversion and the +// write-path level generation gate on this predicate so they cannot disagree +// about the VECTOR-vs-LIST fallback. +bool IsSupportedVectorStructFieldType(const ::arrow::DataType& type) { + switch (type.id()) { + case ::arrow::Type::LIST: + case ::arrow::Type::LARGE_LIST: + case ::arrow::Type::LIST_VIEW: + case ::arrow::Type::LARGE_LIST_VIEW: + case ::arrow::Type::MAP: + return false; + case ::arrow::Type::FIXED_SIZE_LIST: + // A fixed-size list nested inside a struct element would give the + // struct's leaves different per-row multiplicities, which the write-path + // level generation does not support; fall back to LIST encoding. + return false; + case ::arrow::Type::STRUCT: { + const auto& struct_type = checked_cast(type); + for (const auto& field : struct_type.fields()) { + if (!IsSupportedVectorStructFieldType(*field->type())) { + return false; + } + } + return true; + } + case ::arrow::Type::DICTIONARY: + return IsSupportedVectorStructFieldType( + *checked_cast(type).value_type()); + case ::arrow::Type::EXTENSION: + return IsSupportedVectorStructFieldType( + *checked_cast(type).storage_type()); + default: + return true; + } +} + +} // namespace + +Result VectorLeafSlotMultiplier(const ::arrow::DataType& type) { + if (type.id() != ::arrow::Type::FIXED_SIZE_LIST) { + return 1; + } + const auto& list_type = checked_cast(type); + if (list_type.list_size() <= 0) { + return Status::Invalid("VECTOR repetition requires positive vector_length"); + } + ARROW_ASSIGN_OR_RAISE(int32_t child_multiplier, + VectorLeafSlotMultiplier(*list_type.value_type())); + if (child_multiplier > std::numeric_limits::max() / list_type.list_size()) { + return Status::Invalid("Nested VECTOR leaf slot multiplier overflow"); + } + return child_multiplier * list_type.list_size(); +} + +bool IsSupportedVectorElementType(const ::arrow::DataType& type) { + if (type.id() == ::arrow::Type::FIXED_SIZE_LIST) { + const auto& list_type = checked_cast(type); + return list_type.list_size() > 0 && VectorLeafSlotMultiplier(type).ok() && + IsSupportedVectorElementType(*list_type.value_type()); + } + if (type.id() == ::arrow::Type::STRUCT) { + return IsSupportedVectorStructFieldType(type); + } + return !::arrow::is_nested(type) && ::arrow::is_fixed_width(type) && + type.id() != ::arrow::Type::DICTIONARY && type.id() != ::arrow::Type::EXTENSION; +} + +namespace { + Result> MakeArrowDecimal(const LogicalType& logical_type, bool smallest_decimal_enabled) { const auto& decimal = checked_cast(logical_type); diff --git a/cpp/src/parquet/arrow/schema_internal.h b/cpp/src/parquet/arrow/schema_internal.h index 09ad891aad3b..f248e8d6b8c4 100644 --- a/cpp/src/parquet/arrow/schema_internal.h +++ b/cpp/src/parquet/arrow/schema_internal.h @@ -40,4 +40,8 @@ Result> GetArrowType( const ArrowReaderProperties& reader_properties, const std::shared_ptr& metadata = nullptr); +::arrow::Result VectorLeafSlotMultiplier(const ::arrow::DataType& type); + +bool IsSupportedVectorElementType(const ::arrow::DataType& type); + } // namespace parquet::arrow diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc index e0fbe308219c..7ea471a8cef8 100644 --- a/cpp/src/parquet/arrow/writer.cc +++ b/cpp/src/parquet/arrow/writer.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -108,6 +109,112 @@ bool HasNullableRoot(const SchemaManifest& schema_manifest, return nullable; } +int64_t CountVisitedValues(const std::vector& visited_elements) { + return std::accumulate( + visited_elements.begin(), visited_elements.end(), int64_t{0}, + [](int64_t total, const ElementRange& range) { return total + range.Size(); }); +} + +Result> MaterializeVectorLeafArray( + const MultipathLevelBuilderResult& result, ArrowWriteContext* ctx, + bool* leaf_is_nullable) { + const auto& visited_elements = result.post_list_visited_elements; + DCHECK_GT(visited_elements.size(), 0); + + // Nullable VECTOR rows still need one child slot per vector element so that the + // generic spaced leaf writer can align leaf slots with the VECTOR def levels. Arrow + // FixedSizeList already stores child slots for null parent rows, so preserve a + // zero-copy slice over the complete child range and let WriteArrow's def-level-derived + // validity bitmap suppress values belonging to null vector rows. + // The materialized array must hold one entry per physical leaf slot; absent + // ancestors (empty or null lists, null structs) contribute definition levels + // but no slots, so the target is leaf_slot_count, not def_rep_level_count. + if (CountVisitedValues(visited_elements) == result.leaf_slot_count && + visited_elements.size() == 1) { + const ElementRange& range = visited_elements[0]; + auto values = result.leaf_array->Slice(range.start, range.Size()); + if (values->null_count() != 0) { + *leaf_is_nullable = true; + } + return values; + } + + // General path, driven by the definition levels (the only stream that + // distinguishes the three kinds of entries): + // def < ancestor level: absent ancestor, no slot, no value; + // def < values level: slot of a null vector value, no value; + // def >= values level: slot carrying the next value from the visited + // ranges (in order; ranges skip null vector slots). + const int16_t ancestor_def_level = result.vector_repeated_ancestor_def_level; + const int16_t values_def_level = result.vector_values_def_level; + const int16_t* def_levels = result.def_levels; + ::arrow::ArrayVector parts; + size_t range_idx = 0; + int64_t range_used = 0; + int64_t pending_nulls = 0; + auto flush_nulls = [&]() -> Status { + if (pending_nulls > 0) { + ARROW_ASSIGN_OR_RAISE(auto null_values, + ::arrow::MakeArrayOfNull(result.leaf_array->type(), + pending_nulls, ctx->memory_pool)); + parts.push_back(std::move(null_values)); + pending_nulls = 0; + } + return Status::OK(); + }; + auto skip_empty_ranges = [&]() { + while (range_idx < visited_elements.size() && range_used == 0 && + visited_elements[range_idx].Empty()) { + ++range_idx; + } + }; + int64_t i = 0; + while (i < result.def_rep_level_count) { + const int16_t def_level = def_levels[i]; + if (def_level < ancestor_def_level) { + ++i; + continue; + } + if (def_level < values_def_level) { + ++pending_nulls; + ++i; + continue; + } + // Batch a run of value slots from the current visited range. + skip_empty_ranges(); + int64_t run = 1; + while (i + run < result.def_rep_level_count && + def_levels[i + run] >= values_def_level && + range_idx < visited_elements.size() && + range_used + run < visited_elements[range_idx].Size()) { + ++run; + } + if (range_idx >= visited_elements.size() || + range_used + run > visited_elements[range_idx].Size()) { + return Status::Invalid("VECTOR leaf values out of sync with definition levels"); + } + RETURN_NOT_OK(flush_nulls()); + const ElementRange& range = visited_elements[range_idx]; + parts.push_back(result.leaf_array->Slice(range.start + range_used, run)); + range_used += run; + if (range_used == static_cast(range.Size())) { + ++range_idx; + range_used = 0; + } + i += run; + } + RETURN_NOT_OK(flush_nulls()); + skip_empty_ranges(); + if (range_idx != visited_elements.size() || range_used != 0) { + return Status::Invalid("VECTOR leaf values out of sync with definition levels"); + } + *leaf_is_nullable = true; + if (parts.empty()) { + return result.leaf_array->Slice(0, 0); + } + return ::arrow::Concatenate(parts, ctx->memory_pool); +} + Status GetSchemaMetadata(const ::arrow::Schema& schema, ::arrow::MemoryPool* pool, const ArrowWriterProperties& properties, std::shared_ptr* out) { @@ -171,7 +278,11 @@ class ArrowColumnWriterV2 { size_t visited_component_size = result.post_list_visited_elements.size(); DCHECK_GT(visited_component_size, 0); std::shared_ptr values_array; - if (visited_component_size == 1) { + bool leaf_is_nullable = result.leaf_is_nullable; + if (result.leaf_is_vector) { + ARROW_ASSIGN_OR_RAISE(values_array, MaterializeVectorLeafArray( + result, ctx, &leaf_is_nullable)); + } else if (visited_component_size == 1) { const ElementRange& range = result.post_list_visited_elements[0]; values_array = result.leaf_array->Slice(range.start, range.Size()); } else { @@ -191,7 +302,7 @@ class ArrowColumnWriterV2 { return column_writer->WriteArrow(result.def_levels, result.rep_levels, result.def_rep_level_count, *values_array, - ctx, result.leaf_is_nullable); + ctx, leaf_is_nullable); })); } @@ -213,7 +324,7 @@ class ArrowColumnWriterV2 { static ::arrow::Result> Make( const ChunkedArray& data, int64_t offset, const int64_t size, const SchemaManifest& schema_manifest, RowGroupWriter* row_group_writer, - int start_leaf_column_index = -1) { + bool write_fixed_size_list_as_vector, int start_leaf_column_index = -1) { int64_t absolute_position = 0; int chunk_index = 0; int64_t chunk_offset = 0; @@ -283,8 +394,10 @@ class ArrowColumnWriterV2 { std::shared_ptr array_to_write = chunk.Slice(chunk_offset, chunk_write_size); if (array_to_write->length() > 0) { - ARROW_ASSIGN_OR_RAISE(std::unique_ptr builder, - MultipathLevelBuilder::Make(*array_to_write, is_nullable)); + ARROW_ASSIGN_OR_RAISE( + std::unique_ptr builder, + MultipathLevelBuilder::Make(*array_to_write, is_nullable, + write_fixed_size_list_as_vector)); if (leaf_count != builder->GetLeafCount()) { return Status::UnknownError("data type leaf_count != builder_leaf_count", leaf_count, " ", builder->GetLeafCount()); @@ -387,10 +500,10 @@ class FileWriterImpl : public FileWriter { if (row_group_writer_->buffered()) { return Status::Invalid("Cannot write column chunk into the buffered row group."); } - ARROW_ASSIGN_OR_RAISE( - std::unique_ptr writer, - ArrowColumnWriterV2::Make(*data, offset, size, schema_manifest_, - row_group_writer_)); + ARROW_ASSIGN_OR_RAISE(std::unique_ptr writer, + ArrowColumnWriterV2::Make( + *data, offset, size, schema_manifest_, row_group_writer_, + arrow_properties_->write_fixed_size_list_as_vector())); return writer->Write(&column_write_context_); } @@ -471,8 +584,10 @@ class FileWriterImpl : public FileWriter { ChunkedArray chunked_array{batch.column(i)}; ARROW_ASSIGN_OR_RAISE( std::unique_ptr writer, - ArrowColumnWriterV2::Make(chunked_array, offset, size, schema_manifest_, - row_group_writer_, column_index_start)); + ArrowColumnWriterV2::Make( + chunked_array, offset, size, schema_manifest_, row_group_writer_, + arrow_properties_->write_fixed_size_list_as_vector(), + column_index_start)); column_index_start += writer->leaf_count(); if (arrow_properties_->use_threads()) { writers.emplace_back(std::move(writer)); diff --git a/cpp/src/parquet/column_writer.cc b/cpp/src/parquet/column_writer.cc index 653f28f64bde..82cb90d62340 100644 --- a/cpp/src/parquet/column_writer.cc +++ b/cpp/src/parquet/column_writer.cc @@ -83,6 +83,43 @@ namespace parquet { namespace { +int64_t CountVectorSlots(const int16_t* def_levels, int64_t level_offset, + int64_t levels_to_write, int16_t repeated_ancestor_def_level) { + if (def_levels == nullptr || repeated_ancestor_def_level <= 0) { + return levels_to_write; + } + int64_t slots = 0; + for (int64_t i = level_offset; i < level_offset + levels_to_write; ++i) { + slots += def_levels[i] >= repeated_ancestor_def_level; + } + return slots; +} + +std::vector<::parquet::internal::Chunk> AlignCdcChunksToVectorBoundaries( + std::vector<::parquet::internal::Chunk> chunks, const int16_t* def_levels, + int32_t vector_length, int16_t repeated_ancestor_def_level) { + if (vector_length <= 1 || chunks.size() <= 1) { + return chunks; + } + + std::vector<::parquet::internal::Chunk> aligned; + aligned.reserve(chunks.size()); + auto pending = chunks.front(); + for (size_t i = 1; i < chunks.size(); ++i) { + const int64_t pending_slots = + CountVectorSlots(def_levels, pending.level_offset, pending.levels_to_write, + repeated_ancestor_def_level); + if (pending_slots % vector_length == 0) { + aligned.push_back(pending); + pending = chunks[i]; + } else { + pending.levels_to_write += chunks[i].levels_to_write; + } + } + aligned.push_back(pending); + return aligned; +} + // Visitor that extracts the value buffer from a FlatArray at a given offset. struct ValueBufferSlicer { template @@ -1185,6 +1222,38 @@ inline void DoInBatchesNonRepeated(int64_t num_levels, int64_t batch_size, } } +// DoInBatches for VECTOR columns without repeated ancestors. num_levels +// counts physical leaf slots and must be a whole number of vectors; batching +// happens on vector boundaries so a vector value is never split across pages. +template +inline void DoInBatchesVectorNonRepeated(int64_t num_levels, int64_t batch_size, + int64_t max_rows_per_page, int32_t vector_length, + Action&& action, + GetBufferedRows&& curr_page_buffered_rows) { + ARROW_DCHECK_GT(vector_length, 0); + if (num_levels % vector_length != 0) { + throw ParquetException("VECTOR columns must be written in whole-vector batches"); + } + const int64_t total_rows = num_levels / vector_length; + int64_t row_offset = 0; + while (row_offset < total_rows) { + int64_t page_buffered_rows = curr_page_buffered_rows(); + ARROW_DCHECK_LE(page_buffered_rows, max_rows_per_page); + + int64_t max_batch_rows = std::max(1, batch_size / vector_length); + max_batch_rows = std::min(max_batch_rows, total_rows - row_offset); + max_batch_rows = std::min(max_batch_rows, max_rows_per_page - page_buffered_rows); + if (max_batch_rows == 0) { + max_batch_rows = 1; + } + int64_t level_offset = row_offset * vector_length; + int64_t level_count = max_batch_rows * vector_length; + + action(level_offset, level_count, /*check_page_limit=*/true); + row_offset += max_batch_rows; + } +} + // DoInBatches for repeated columns template inline void DoInBatchesRepeated(const int16_t* def_levels, const int16_t* rep_levels, @@ -1240,12 +1309,20 @@ inline void DoInBatchesRepeated(const int16_t* def_levels, const int16_t* rep_le template inline void DoInBatches(const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels, int64_t batch_size, int64_t max_rows_per_page, - bool pages_change_on_record_boundaries, Action&& action, + bool pages_change_on_record_boundaries, bool is_vector, + int32_t vector_length, Action&& action, GetBufferedRows&& curr_page_buffered_rows) { if (!rep_levels) { - DoInBatchesNonRepeated(num_levels, batch_size, max_rows_per_page, - std::forward(action), - std::forward(curr_page_buffered_rows)); + if (is_vector) { + DoInBatchesVectorNonRepeated( + num_levels, batch_size, max_rows_per_page, vector_length, + std::forward(action), + std::forward(curr_page_buffered_rows)); + } else { + DoInBatchesNonRepeated(num_levels, batch_size, max_rows_per_page, + std::forward(action), + std::forward(curr_page_buffered_rows)); + } } else { DoInBatchesRepeated(def_levels, rep_levels, num_levels, batch_size, max_rows_per_page, pages_change_on_record_boundaries, std::forward(action), @@ -1323,7 +1400,11 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, } pages_change_on_record_boundaries_ = properties->data_page_version() == ParquetDataPageVersion::V2 || - properties->page_index_enabled(descr_->path()); + properties->page_index_enabled(descr_->path()) || + // A vector value must not be split across pages; for VECTOR columns + // below repeated ancestors a record contains whole vectors, so + // breaking pages on record boundaries preserves the invariant. + descr_->in_vector_column(); } int64_t Close() override { return ColumnWriterImpl::Close(); } @@ -1366,7 +1447,8 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, }; DoInBatches(def_levels, rep_levels, num_values, properties_->write_batch_size(), properties_->max_rows_per_page(), pages_change_on_record_boundaries(), - WriteChunk, [this]() { return num_buffered_rows_; }); + descr_->in_vector_column(), descr_->effective_vector_length(), WriteChunk, + [this]() { return num_buffered_rows_; }); return value_offset; } @@ -1417,16 +1499,20 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, }; DoInBatches(def_levels, rep_levels, num_values, properties_->write_batch_size(), properties_->max_rows_per_page(), pages_change_on_record_boundaries(), - WriteChunk, [this]() { return num_buffered_rows_; }); + descr_->in_vector_column(), descr_->effective_vector_length(), WriteChunk, + [this]() { return num_buffered_rows_; }); } Status WriteArrow(const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels, const ::arrow::Array& leaf_array, ArrowWriteContext* ctx, bool leaf_field_nullable) override { BEGIN_PARQUET_CATCH_EXCEPTIONS + const bool is_vector = descr_->in_vector_column(); // Leaf nulls are canonical when there is only a single null element after a list - // and it is at the leaf. + // and it is at the leaf. VECTOR parent nulls may also be materialized as spaced + // null slots in the leaf array, so do not treat those as single nullable elements. bool single_nullable_element = + !is_vector && (level_info_.def_level == level_info_.repeated_ancestor_def_level + 1) && leaf_field_nullable; if (!leaf_field_nullable && leaf_array.null_count() != 0) { @@ -1446,6 +1532,11 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, DCHECK(content_defined_chunker_.has_value()); auto chunks = content_defined_chunker_->GetChunks(def_levels, rep_levels, num_levels, leaf_array); + if (is_vector) { + chunks = AlignCdcChunksToVectorBoundaries( + std::move(chunks), def_levels, descr_->effective_vector_length(), + level_info_.repeated_ancestor_def_level); + } for (size_t i = 0; i < chunks.size(); i++) { auto chunk = chunks[i]; auto chunk_array = leaf_array.Slice(chunk.value_offset); @@ -1700,6 +1791,13 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, } WriteRepetitionLevels(num_levels, rep_levels); + } else if (descr_->in_vector_column()) { + const int32_t vector_length = descr_->effective_vector_length(); + if (vector_length <= 0 || num_levels % vector_length != 0) { + throw ParquetException("VECTOR columns must be written in whole-vector batches"); + } + rows_written_ += num_levels / vector_length; + num_buffered_rows_ += num_levels / vector_length; } else { // Each value is exactly one row rows_written_ += num_levels; @@ -1794,6 +1892,13 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, } } WriteRepetitionLevels(num_levels, rep_levels); + } else if (descr_->in_vector_column()) { + const int32_t vector_length = descr_->effective_vector_length(); + if (vector_length <= 0 || num_levels % vector_length != 0) { + throw ParquetException("VECTOR columns must be written in whole-vector batches"); + } + rows_written_ += num_levels / vector_length; + num_buffered_rows_ += num_levels / vector_length; } else { // Each value is exactly one row rows_written_ += num_levels; @@ -2078,6 +2183,7 @@ Status TypedColumnWriterImpl::WriteArrowDictionary( PARQUET_CATCH_NOT_OK( DoInBatches(def_levels, rep_levels, num_levels, properties_->write_batch_size(), properties_->max_rows_per_page(), pages_change_on_record_boundaries(), + descr_->in_vector_column(), descr_->effective_vector_length(), WriteIndicesChunk, [this]() { return num_buffered_rows_; })); return Status::OK(); } @@ -2540,6 +2646,7 @@ Status TypedColumnWriterImpl::WriteArrowDense( PARQUET_CATCH_NOT_OK( DoInBatches(def_levels, rep_levels, num_levels, properties_->write_batch_size(), properties_->max_rows_per_page(), pages_change_on_record_boundaries(), + descr_->in_vector_column(), descr_->effective_vector_length(), WriteChunk, [this]() { return num_buffered_rows_; })); return Status::OK(); } diff --git a/cpp/src/parquet/file_reader.cc b/cpp/src/parquet/file_reader.cc index 2f46a5e296f8..df878ffb8779 100644 --- a/cpp/src/parquet/file_reader.cc +++ b/cpp/src/parquet/file_reader.cc @@ -980,10 +980,18 @@ int64_t ScanFileContents(std::vector columns, const int32_t column_batch_si total_rows[col]++; } } - } else { + } else if (!col_reader->descr()->in_vector_column()) { total_rows[col] += levels_read; } } + if (col_reader->descr()->max_repetition_level() == 0 && + col_reader->descr()->in_vector_column()) { + // VECTOR columns without repeated ancestors have one level entry per + // physical vector slot, so levels_read would over-count rows by the + // vector length. Use the row-group row count after still scanning the + // column contents above. + total_rows[col] += group_reader->metadata()->num_rows(); + } col++; } } diff --git a/cpp/src/parquet/parquet.thrift b/cpp/src/parquet/parquet.thrift index 9603cefed388..83575cfc8f75 100644 --- a/cpp/src/parquet/parquet.thrift +++ b/cpp/src/parquet/parquet.thrift @@ -190,6 +190,35 @@ enum FieldRepetitionType { /** The field is repeated and can contain 0 or more values */ REPEATED = 2; + + // EXPERIMENTAL: fixed-size repetition. + // + // VECTOR is used for the middle group of a fixed-size vector encoded with + // the canonical 3-level structure: + // + // group (VECTOR) { + // vector group list [vector_length] { + // element; + // } + // } + // + // The outer group MUST be REQUIRED or OPTIONAL, MUST be annotated with + // LogicalType.VECTOR, and MUST contain exactly one field named "list". The + // middle group MUST have repetition_type = VECTOR, MUST be named "list", + // MUST carry a positive SchemaElement.vector_length, MUST have exactly one + // field named "element", and MUST NOT have a logical or converted type. + // The element field encodes the vector element type and MUST be REQUIRED or + // OPTIONAL. Repeated fields are not allowed inside the vector element subtree. + // + // A VECTOR level contributes no definition or repetition level. Each vector + // occurrence contributes exactly vector_length element slots / level entries + // to each primitive leaf in the element subtree, including null vector values + // and null element slots. Column chunk and page num_values count those slots, + // not necessarily the number of physical values. Writers MUST NOT split the + // slots of one vector value across data pages. + // + // Readers that do not understand VECTOR are expected to reject the file. + VECTOR = 3; } /** @@ -469,6 +498,13 @@ struct GeographyType { 2: optional EdgeInterpolationAlgorithm algorithm; } +// EXPERIMENTAL: Embedded Vector logical type annotation. +// +// The VECTOR annotation marks the outer group of a fixed-size vector field; +// see FieldRepetitionType.VECTOR for the full canonical structure and data +// layout rules. +struct VectorType {} // allowed for group nodes only + /** * LogicalType annotations to replace ConvertedType. * @@ -502,6 +538,7 @@ union LogicalType { 16: VariantType VARIANT // no compatible ConvertedType 17: GeometryType GEOMETRY // no compatible ConvertedType 18: GeographyType GEOGRAPHY // no compatible ConvertedType + 19: VectorType VECTOR // no compatible ConvertedType } /** @@ -564,6 +601,11 @@ struct SchemaElement { * for some logical types to ensure forward-compatibility in format v1. */ 10: optional LogicalType logicalType + + // EXPERIMENTAL: For VECTOR repetition, the fixed number of element slots per + // parent occurrence. This MUST be set, and positive, when repetition_type is + // VECTOR and MUST NOT be set otherwise. + 11: optional i32 vector_length; } /** diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h index e2244a1176e3..23a37b733484 100644 --- a/cpp/src/parquet/properties.h +++ b/cpp/src/parquet/properties.h @@ -1329,6 +1329,7 @@ class PARQUET_EXPORT ArrowWriterProperties { truncated_timestamps_allowed_(false), store_schema_(false), compliant_nested_types_(true), + write_fixed_size_list_as_vector_(false), engine_version_(V2), use_threads_(kArrowDefaultUseThreads), executor_(NULLPTR), @@ -1399,6 +1400,16 @@ class PARQUET_EXPORT ArrowWriterProperties { return this; } + /// \brief EXPERIMENTAL: encode supported Arrow FixedSizeList values as Parquet + /// VECTOR using the canonical 3-level VECTOR layout. Unsupported FixedSizeList + /// values fall back to the standard LIST encoding: zero-length lists, + /// variable-width or dictionary or extension elements, struct elements containing + /// fixed-size lists, and fixed-size lists below nullable groups. + Builder* enable_experimental_vector_encoding() { + write_fixed_size_list_as_vector_ = true; + return this; + } + /// Set the version of the Parquet writer engine. Builder* set_engine_version(EngineVersion version) { engine_version_ = version; @@ -1441,7 +1452,8 @@ class PARQUET_EXPORT ArrowWriterProperties { return std::shared_ptr(new ArrowWriterProperties( write_timestamps_as_int96_, coerce_timestamps_enabled_, coerce_timestamps_unit_, truncated_timestamps_allowed_, store_schema_, compliant_nested_types_, - engine_version_, use_threads_, executor_, write_time_adjusted_to_utc_)); + write_fixed_size_list_as_vector_, engine_version_, use_threads_, executor_, + write_time_adjusted_to_utc_)); } private: @@ -1453,6 +1465,7 @@ class PARQUET_EXPORT ArrowWriterProperties { bool store_schema_; bool compliant_nested_types_; + bool write_fixed_size_list_as_vector_; EngineVersion engine_version_; bool use_threads_; @@ -1479,6 +1492,10 @@ class PARQUET_EXPORT ArrowWriterProperties { /// "element". bool compliant_nested_types() const { return compliant_nested_types_; } + bool write_fixed_size_list_as_vector() const { + return write_fixed_size_list_as_vector_; + } + /// \brief The underlying engine version to use when writing Arrow data. /// /// V2 is currently the latest V1 is considered deprecated but left in @@ -1503,6 +1520,7 @@ class PARQUET_EXPORT ArrowWriterProperties { ::arrow::TimeUnit::type coerce_timestamps_unit, bool truncated_timestamps_allowed, bool store_schema, bool compliant_nested_types, + bool write_fixed_size_list_as_vector, EngineVersion engine_version, bool use_threads, ::arrow::internal::Executor* executor, bool write_time_adjusted_to_utc) @@ -1512,6 +1530,7 @@ class PARQUET_EXPORT ArrowWriterProperties { truncated_timestamps_allowed_(truncated_timestamps_allowed), store_schema_(store_schema), compliant_nested_types_(compliant_nested_types), + write_fixed_size_list_as_vector_(write_fixed_size_list_as_vector), engine_version_(engine_version), use_threads_(use_threads), executor_(executor), @@ -1523,6 +1542,7 @@ class PARQUET_EXPORT ArrowWriterProperties { const bool truncated_timestamps_allowed_; const bool store_schema_; const bool compliant_nested_types_; + const bool write_fixed_size_list_as_vector_; const EngineVersion engine_version_; const bool use_threads_; ::arrow::internal::Executor* executor_; diff --git a/cpp/src/parquet/schema.cc b/cpp/src/parquet/schema.cc index 0cfa49c21c16..45729c1584eb 100644 --- a/cpp/src/parquet/schema.cc +++ b/cpp/src/parquet/schema.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -50,6 +51,16 @@ void CheckColumnBounds(int column_index, size_t max_columns) { } } +void ValidateVectorProperties(Repetition::type repetition, int32_t vector_length) { + if (repetition == Repetition::VECTOR) { + if (vector_length <= 0) { + throw ParquetException("VECTOR nodes must specify a positive vector_length"); + } + } else if (vector_length != -1) { + throw ParquetException("Only VECTOR nodes may specify vector_length"); + } +} + } // namespace namespace schema { @@ -117,7 +128,7 @@ const std::shared_ptr Node::path() const { bool Node::EqualsInternal(const Node* other) const { return type_ == other->type_ && name_ == other->name_ && repetition_ == other->repetition_ && converted_type_ == other->converted_type_ && - field_id_ == other->field_id() && + field_id_ == other->field_id() && vector_length_ == other->vector_length() && logical_type_->Equals(*(other->logical_type())); } @@ -128,10 +139,15 @@ void Node::SetParent(const Node* parent) { parent_ = parent; } PrimitiveNode::PrimitiveNode(const std::string& name, Repetition::type repetition, Type::type type, ConvertedType::type converted_type, - int length, int precision, int scale, int id) - : Node(Node::PRIMITIVE, name, repetition, converted_type, id), + int length, int precision, int scale, int id, + int32_t vector_length) + : Node(Node::PRIMITIVE, name, repetition, converted_type, id, vector_length), physical_type_(type), type_length_(length) { + if (repetition == Repetition::VECTOR) { + throw ParquetException("VECTOR repetition is only allowed on group nodes"); + } + ValidateVectorProperties(repetition, vector_length); std::stringstream ss; // PARQUET-842: In an earlier revision, decimal_metadata_.isset was being @@ -241,10 +257,15 @@ PrimitiveNode::PrimitiveNode(const std::string& name, Repetition::type repetitio PrimitiveNode::PrimitiveNode(const std::string& name, Repetition::type repetition, std::shared_ptr logical_type, - Type::type physical_type, int physical_length, int id) - : Node(Node::PRIMITIVE, name, repetition, std::move(logical_type), id), + Type::type physical_type, int physical_length, int id, + int32_t vector_length) + : Node(Node::PRIMITIVE, name, repetition, std::move(logical_type), id, vector_length), physical_type_(physical_type), type_length_(physical_length) { + if (repetition == Repetition::VECTOR) { + throw ParquetException("VECTOR repetition is only allowed on group nodes"); + } + ValidateVectorProperties(repetition, vector_length); std::stringstream error; if (logical_type_) { // Check for logical type <=> node type consistency @@ -315,8 +336,14 @@ void PrimitiveNode::VisitConst(Node::ConstVisitor* visitor) const { // Group node GroupNode::GroupNode(const std::string& name, Repetition::type repetition, - const NodeVector& fields, ConvertedType::type converted_type, int id) - : Node(Node::GROUP, name, repetition, converted_type, id), fields_(fields) { + const NodeVector& fields, ConvertedType::type converted_type, int id, + int32_t vector_length) + : Node(Node::GROUP, name, repetition, converted_type, id, vector_length), + fields_(fields) { + ValidateVectorProperties(repetition, vector_length); + if (repetition == Repetition::VECTOR && converted_type != ConvertedType::NONE) { + throw ParquetException("VECTOR-repeated groups must not have a converted type"); + } // For forward compatibility, create an equivalent logical type logical_type_ = LogicalType::FromConvertedType(converted_type_); if (!(logical_type_ && (logical_type_->is_nested() || logical_type_->is_none()) && @@ -334,8 +361,14 @@ GroupNode::GroupNode(const std::string& name, Repetition::type repetition, GroupNode::GroupNode(const std::string& name, Repetition::type repetition, const NodeVector& fields, - std::shared_ptr logical_type, int id) - : Node(Node::GROUP, name, repetition, std::move(logical_type), id), fields_(fields) { + std::shared_ptr logical_type, int id, + int32_t vector_length) + : Node(Node::GROUP, name, repetition, std::move(logical_type), id, vector_length), + fields_(fields) { + ValidateVectorProperties(repetition, vector_length); + if (repetition == Repetition::VECTOR && logical_type_ && !logical_type_->is_none()) { + throw ParquetException("VECTOR-repeated groups must not have a logical type"); + } if (logical_type_) { // Check for logical type <=> node type consistency if (logical_type_->is_nested()) { @@ -423,19 +456,23 @@ std::unique_ptr GroupNode::FromParquet(const void* opaque_element, if (element->__isset.field_id) { field_id = element->field_id; } + int32_t vector_length = -1; + if (element->__isset.vector_length) { + vector_length = element->vector_length; + } std::unique_ptr group_node; if (element->__isset.logicalType) { // updated writer with logical type present - group_node = std::unique_ptr( - new GroupNode(element->name, LoadEnumSafe(&element->repetition_type), fields, - LogicalType::FromThrift(element->logicalType), field_id)); + group_node = std::unique_ptr(new GroupNode( + element->name, LoadEnumSafe(&element->repetition_type), fields, + LogicalType::FromThrift(element->logicalType), field_id, vector_length)); } else { group_node = std::unique_ptr(new GroupNode( element->name, LoadEnumSafe(&element->repetition_type), fields, (element->__isset.converted_type ? LoadEnumSafe(&element->converted_type) : ConvertedType::NONE), - field_id)); + field_id, vector_length)); } return std::unique_ptr(group_node.release()); @@ -449,25 +486,30 @@ std::unique_ptr PrimitiveNode::FromParquet(const void* opaque_element) { if (element->__isset.field_id) { field_id = element->field_id; } + int32_t vector_length = -1; + if (element->__isset.vector_length) { + vector_length = element->vector_length; + } std::unique_ptr primitive_node; if (element->__isset.logicalType) { // updated writer with logical type present - primitive_node = std::unique_ptr( - new PrimitiveNode(element->name, LoadEnumSafe(&element->repetition_type), - LogicalType::FromThrift(element->logicalType), - LoadEnumSafe(&element->type), element->type_length, field_id)); - } else if (element->__isset.converted_type) { - // legacy writer with converted type present primitive_node = std::unique_ptr(new PrimitiveNode( element->name, LoadEnumSafe(&element->repetition_type), - LoadEnumSafe(&element->type), LoadEnumSafe(&element->converted_type), - element->type_length, element->precision, element->scale, field_id)); + LogicalType::FromThrift(element->logicalType), LoadEnumSafe(&element->type), + element->type_length, field_id, vector_length)); + } else if (element->__isset.converted_type) { + // legacy writer with converted type present + primitive_node = std::unique_ptr( + new PrimitiveNode(element->name, LoadEnumSafe(&element->repetition_type), + LoadEnumSafe(&element->type), + LoadEnumSafe(&element->converted_type), element->type_length, + element->precision, element->scale, field_id, vector_length)); } else { // logical type not present primitive_node = std::unique_ptr(new PrimitiveNode( element->name, LoadEnumSafe(&element->repetition_type), NoLogicalType::Make(), - LoadEnumSafe(&element->type), element->type_length, field_id)); + LoadEnumSafe(&element->type), element->type_length, field_id, vector_length)); } // Return as unique_ptr to the base type @@ -499,6 +541,9 @@ void GroupNode::ToParquet(void* opaque_element) const { if (field_id_ >= 0) { element->__set_field_id(field_id_); } + if (is_vector()) { + element->__set_vector_length(vector_length_); + } if (logical_type_ && logical_type_->is_serialized()) { element->__set_logicalType(logical_type_->ToThrift()); } @@ -524,6 +569,9 @@ void PrimitiveNode::ToParquet(void* opaque_element) const { if (field_id_ >= 0) { element->__set_field_id(field_id_); } + if (is_vector()) { + element->__set_vector_length(vector_length_); + } if (logical_type_ && logical_type_->is_serialized() && // TODO(tpboudreau): remove the following conjunct to enable serialization // of IntervalTypes after parquet.thrift recognizes them @@ -634,6 +682,9 @@ static void PrintRepLevel(Repetition::type repetition, std::ostream& stream) { case Repetition::REPEATED: stream << "repeated"; break; + case Repetition::VECTOR: + stream << "vector"; + break; default: break; } @@ -710,14 +761,19 @@ struct SchemaPrinter : public Node::ConstVisitor { stream_ << " "; PrintType(node, stream_); stream_ << " field_id=" << node->field_id() << " " << node->name(); + if (node->is_vector()) { + stream_ << " [" << node->vector_length() << "]"; + } PrintConvertedType(node, stream_); stream_ << ";" << std::endl; } void Visit(const GroupNode* node) { PrintRepLevel(node->repetition(), stream_); - stream_ << " group " - << "field_id=" << node->field_id() << " " << node->name(); + stream_ << " group " << "field_id=" << node->field_id() << " " << node->name(); + if (node->is_vector()) { + stream_ << " [" << node->vector_length() << "]"; + } auto lt = node->converted_type(); const auto& la = node->logical_type(); if (la && la->is_valid() && !la->is_none()) { @@ -836,6 +892,9 @@ void SchemaDescriptor::BuildTree(const NodePtr& node, int16_t max_def_level, // between an empty list and a list with an item in it. ++max_rep_level; ++max_def_level; + } else if (node->is_vector()) { + // VECTOR fields repeat a fixed number of times per parent value without + // increasing the maximum definition or repetition level. } // Now, walk the schema and create a ColumnDescriptor for each leaf node @@ -848,8 +907,27 @@ void SchemaDescriptor::BuildTree(const NodePtr& node, int16_t max_def_level, node_to_leaf_index_[static_cast(node.get())] = static_cast(leaves_.size()); + // Determine the product of all VECTOR-repeated ancestors (including this leaf) + // so nested VECTOR shapes such as vector<3, vector<4, int32>> report the total + // number of physical leaf values contributed per parent record. + int32_t effective_vector_length = -1; + for (const Node* cursor = node.get(); cursor != nullptr; cursor = cursor->parent()) { + if (cursor->is_vector()) { + if (effective_vector_length < 0) { + effective_vector_length = cursor->vector_length(); + } else { + if (effective_vector_length > + std::numeric_limits::max() / cursor->vector_length()) { + throw ParquetException("Nested VECTOR effective vector_length overflow"); + } + effective_vector_length *= cursor->vector_length(); + } + } + } + // Primitive node, append to leaves - leaves_.push_back(ColumnDescriptor(node, max_def_level, max_rep_level, this)); + leaves_.push_back(ColumnDescriptor(node, max_def_level, max_rep_level, this, + effective_vector_length)); leaf_to_base_.emplace(static_cast(leaves_.size()) - 1, base); leaf_to_idx_.emplace(node->path()->ToDotString(), static_cast(leaves_.size()) - 1); @@ -866,10 +944,12 @@ int SchemaDescriptor::GetColumnIndex(const PrimitiveNode& node) const { ColumnDescriptor::ColumnDescriptor(schema::NodePtr node, int16_t max_definition_level, int16_t max_repetition_level, - const SchemaDescriptor* schema_descr) + const SchemaDescriptor* schema_descr, + int32_t effective_vector_length) : node_(std::move(node)), max_definition_level_(max_definition_level), - max_repetition_level_(max_repetition_level) { + max_repetition_level_(max_repetition_level), + effective_vector_length_(effective_vector_length) { if (!node_->is_primitive()) { throw ParquetException("Must be a primitive type"); } @@ -879,7 +959,8 @@ ColumnDescriptor::ColumnDescriptor(schema::NodePtr node, int16_t max_definition_ bool ColumnDescriptor::Equals(const ColumnDescriptor& other) const { return primitive_node_->Equals(other.primitive_node_) && max_repetition_level() == other.max_repetition_level() && - max_definition_level() == other.max_definition_level(); + max_definition_level() == other.max_definition_level() && + effective_vector_length() == other.effective_vector_length(); } const ColumnDescriptor* SchemaDescriptor::Column(int i) const { @@ -930,8 +1011,13 @@ std::string ColumnDescriptor::ToString() const { << " physical_type: " << TypeToString(physical_type()) << "," << std::endl << " converted_type: " << ConvertedTypeToString(converted_type()) << "," << std::endl - << " logical_type: " << logical_type()->ToString() << "," << std::endl - << " max_definition_level: " << max_definition_level() << "," << std::endl + << " logical_type: " << logical_type()->ToString() << "," << std::endl; + + if (schema_node()->is_vector()) { + ss << " vector_length: " << schema_node()->vector_length() << "," << std::endl; + } + + ss << " max_definition_level: " << max_definition_level() << "," << std::endl << " max_repetition_level: " << max_repetition_level() << "," << std::endl; if (physical_type() == ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { diff --git a/cpp/src/parquet/schema.h b/cpp/src/parquet/schema.h index 65732603ea1d..6c0d34e90c8d 100644 --- a/cpp/src/parquet/schema.h +++ b/cpp/src/parquet/schema.h @@ -109,6 +109,11 @@ class PARQUET_EXPORT Node { bool is_repeated() const { return repetition_ == Repetition::REPEATED; } + /// \brief True when this node specifically has VECTOR repetition; use + /// ColumnDescriptor::in_vector_column() to ask whether a column is part of + /// a VECTOR subtree. + bool is_vector() const { return repetition_ == Repetition::VECTOR; } + bool is_required() const { return repetition_ == Repetition::REQUIRED; } virtual bool Equals(const Node* other) const = 0; @@ -128,6 +133,10 @@ class PARQUET_EXPORT Node { /// Thrift. int field_id() const { return field_id_; } + /// \brief The fixed number of values per parent when repetition == VECTOR. + /// Returns -1 when this node is not VECTOR-repeated. + int32_t vector_length() const { return vector_length_; } + const Node* parent() const { return parent_; } const std::shared_ptr path() const; @@ -155,21 +164,25 @@ class PARQUET_EXPORT Node { friend class GroupNode; Node(Node::type type, const std::string& name, Repetition::type repetition, - ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1) + ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1, + int32_t vector_length = -1) : type_(type), name_(name), repetition_(repetition), converted_type_(converted_type), field_id_(field_id), + vector_length_(vector_length), parent_(NULLPTR) {} Node(Node::type type, const std::string& name, Repetition::type repetition, - std::shared_ptr logical_type, int field_id = -1) + std::shared_ptr logical_type, int field_id = -1, + int32_t vector_length = -1) : type_(type), name_(name), repetition_(repetition), logical_type_(std::move(logical_type)), field_id_(field_id), + vector_length_(vector_length), parent_(NULLPTR) {} Node::type type_; @@ -178,6 +191,7 @@ class PARQUET_EXPORT Node { ConvertedType::type converted_type_{ConvertedType::NONE}; std::shared_ptr logical_type_; int field_id_; + int32_t vector_length_; // Nodes should not be shared, they have a single parent. const Node* parent_; @@ -205,9 +219,9 @@ class PARQUET_EXPORT PrimitiveNode : public Node { Type::type type, ConvertedType::type converted_type = ConvertedType::NONE, int length = -1, int precision = -1, int scale = -1, - int field_id = -1) { + int field_id = -1, int32_t vector_length = -1) { return NodePtr(new PrimitiveNode(name, repetition, type, converted_type, length, - precision, scale, field_id)); + precision, scale, field_id, vector_length)); } // If no logical type, pass LogicalType::None() or nullptr @@ -215,9 +229,10 @@ class PARQUET_EXPORT PrimitiveNode : public Node { static inline NodePtr Make(const std::string& name, Repetition::type repetition, std::shared_ptr logical_type, Type::type primitive_type, int primitive_length = -1, - int field_id = -1) { + int field_id = -1, int32_t vector_length = -1) { return NodePtr(new PrimitiveNode(name, repetition, std::move(logical_type), - primitive_type, primitive_length, field_id)); + primitive_type, primitive_length, field_id, + vector_length)); } bool Equals(const Node* other) const override; @@ -239,11 +254,13 @@ class PARQUET_EXPORT PrimitiveNode : public Node { private: PrimitiveNode(const std::string& name, Repetition::type repetition, Type::type type, ConvertedType::type converted_type = ConvertedType::NONE, int length = -1, - int precision = -1, int scale = -1, int field_id = -1); + int precision = -1, int scale = -1, int field_id = -1, + int32_t vector_length = -1); PrimitiveNode(const std::string& name, Repetition::type repetition, std::shared_ptr logical_type, - Type::type primitive_type, int primitive_length = -1, int field_id = -1); + Type::type primitive_type, int primitive_length = -1, int field_id = -1, + int32_t vector_length = -1); Type::type physical_type_; int32_t type_length_; @@ -270,8 +287,9 @@ class PARQUET_EXPORT GroupNode : public Node { static inline NodePtr Make(const std::string& name, Repetition::type repetition, const NodeVector& fields, ConvertedType::type converted_type = ConvertedType::NONE, - int field_id = -1) { - return NodePtr(new GroupNode(name, repetition, fields, converted_type, field_id)); + int field_id = -1, int32_t vector_length = -1) { + return NodePtr( + new GroupNode(name, repetition, fields, converted_type, field_id, vector_length)); } // If no logical type, pass nullptr @@ -279,9 +297,9 @@ class PARQUET_EXPORT GroupNode : public Node { static inline NodePtr Make(const std::string& name, Repetition::type repetition, const NodeVector& fields, std::shared_ptr logical_type, - int field_id = -1) { - return NodePtr( - new GroupNode(name, repetition, fields, std::move(logical_type), field_id)); + int field_id = -1, int32_t vector_length = -1) { + return NodePtr(new GroupNode(name, repetition, fields, std::move(logical_type), + field_id, vector_length)); } bool Equals(const Node* other) const override; @@ -307,11 +325,12 @@ class PARQUET_EXPORT GroupNode : public Node { private: GroupNode(const std::string& name, Repetition::type repetition, const NodeVector& fields, - ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1); + ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1, + int32_t vector_length = -1); GroupNode(const std::string& name, Repetition::type repetition, const NodeVector& fields, std::shared_ptr logical_type, - int field_id = -1); + int field_id = -1, int32_t vector_length = -1); NodeVector fields_; bool EqualsInternal(const GroupNode* other) const; @@ -358,7 +377,8 @@ class PARQUET_EXPORT ColumnDescriptor { public: ColumnDescriptor(schema::NodePtr node, int16_t max_definition_level, int16_t max_repetition_level, - const SchemaDescriptor* schema_descr = NULLPTR); + const SchemaDescriptor* schema_descr = NULLPTR, + int32_t effective_vector_length = -1); bool Equals(const ColumnDescriptor& other) const; @@ -366,6 +386,16 @@ class PARQUET_EXPORT ColumnDescriptor { int16_t max_repetition_level() const { return max_repetition_level_; } + /// \brief The fixed number of leaf values per parent record contributed by + /// this column, computed as the product of all VECTOR-repeated ancestors + /// (including the leaf itself if it is VECTOR-repeated). Returns -1 when the + /// column is not part of a VECTOR-repeated subtree. + int32_t effective_vector_length() const { return effective_vector_length_; } + + /// \brief True when this column belongs to a VECTOR-repeated subtree + /// (the leaf or some ancestor has Repetition::VECTOR). + bool in_vector_column() const { return effective_vector_length_ > 0; } + Type::type physical_type() const { return primitive_node_->physical_type(); } ConvertedType::type converted_type() const { return primitive_node_->converted_type(); } @@ -418,6 +448,7 @@ class PARQUET_EXPORT ColumnDescriptor { int16_t max_definition_level_; int16_t max_repetition_level_; + int32_t effective_vector_length_; }; // Container for the converted Parquet schema with a computed information from diff --git a/cpp/src/parquet/schema_test.cc b/cpp/src/parquet/schema_test.cc index 859f14a34d91..0f1b5f96aa4f 100644 --- a/cpp/src/parquet/schema_test.cc +++ b/cpp/src/parquet/schema_test.cc @@ -43,7 +43,8 @@ namespace schema { static inline SchemaElement NewPrimitive(const std::string& name, FieldRepetitionType::type repetition, - Type::type type, int field_id = -1) { + Type::type type, int field_id = -1, + int32_t vector_length = -1) { SchemaElement result; result.__set_name(name); result.__set_repetition_type(repetition); @@ -51,12 +52,16 @@ static inline SchemaElement NewPrimitive(const std::string& name, if (field_id >= 0) { result.__set_field_id(field_id); } + if (vector_length >= 0) { + result.__set_vector_length(vector_length); + } return result; } static inline SchemaElement NewGroup(const std::string& name, FieldRepetitionType::type repetition, - int num_children, int field_id = -1) { + int num_children, int field_id = -1, + int32_t vector_length = -1) { SchemaElement result; result.__set_name(name); result.__set_repetition_type(repetition); @@ -65,6 +70,9 @@ static inline SchemaElement NewGroup(const std::string& name, if (field_id >= 0) { result.__set_field_id(field_id); } + if (vector_length >= 0) { + result.__set_vector_length(vector_length); + } return result; } @@ -217,6 +225,24 @@ TEST_F(TestPrimitiveNode, FromParquet) { ASSERT_EQ(12, prim_node_->decimal_metadata().precision); } +TEST_F(TestPrimitiveNode, VectorFromParquetRejected) { + SchemaElement elt = + NewPrimitive(name_, FieldRepetitionType::VECTOR, Type::FLOAT, field_id_, 8); + + ASSERT_THROW(Convert(&elt), ParquetException); +} + +TEST_F(TestPrimitiveNode, VectorValidation) { + ASSERT_THROW(PrimitiveNode::Make("vec", Repetition::VECTOR, Type::FLOAT), + ParquetException); + ASSERT_THROW(PrimitiveNode::Make("scalar", Repetition::REQUIRED, Type::FLOAT, + ConvertedType::NONE, -1, -1, -1, -1, 4), + ParquetException); + ASSERT_THROW(PrimitiveNode::Make("empty", Repetition::VECTOR, Type::FLOAT, + ConvertedType::NONE, -1, -1, -1, -1, 0), + ParquetException); +} + TEST_F(TestPrimitiveNode, Equals) { PrimitiveNode node1("foo", Repetition::REQUIRED, Type::INT32); PrimitiveNode node2("foo", Repetition::REQUIRED, Type::INT64); @@ -365,6 +391,67 @@ TEST_F(TestGroupNode, Attrs) { ASSERT_EQ(ConvertedType::LIST, node2.converted_type()); } +TEST_F(TestGroupNode, VectorAttrs) { + auto node = GroupNode::Make("vec", Repetition::VECTOR, Fields1(), ConvertedType::NONE, + /*field_id=*/-1, + /*vector_length=*/4); + + ASSERT_TRUE(node->is_vector()); + ASSERT_EQ(Repetition::VECTOR, node->repetition()); + ASSERT_EQ(4, node->vector_length()); +} + +TEST_F(TestGroupNode, VectorValidation) { + ASSERT_THROW(GroupNode::Make("vec", Repetition::VECTOR, Fields1()), ParquetException); + ASSERT_THROW(GroupNode::Make("empty", Repetition::VECTOR, Fields1(), + ConvertedType::NONE, /*field_id=*/-1, + /*vector_length=*/0), + ParquetException); + ASSERT_THROW(GroupNode::Make("scalar", Repetition::REQUIRED, Fields1(), + ConvertedType::NONE, /*field_id=*/-1, + /*vector_length=*/4), + ParquetException); + ASSERT_THROW(GroupNode::Make("list", Repetition::VECTOR, Fields1(), ConvertedType::LIST, + /*field_id=*/-1, + /*vector_length=*/4), + ParquetException); + ASSERT_THROW(GroupNode::Make("list", Repetition::VECTOR, Fields1(), LogicalType::List(), + /*field_id=*/-1, + /*vector_length=*/4), + ParquetException); +} + +TEST_F(TestGroupNode, VectorLogicalTypeRoundTrip) { + // A canonical 3-level vector group annotated with the Vector logical type + // round-trips through the Thrift intermediary. + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::FLOAT); + auto list = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, /*field_id=*/-1, + /*vector_length=*/3); + auto group = + GroupNode::Make("embedding", Repetition::OPTIONAL, {list}, LogicalType::Vector()); + + ASSERT_TRUE(group->logical_type()->is_vector()); + ASSERT_TRUE(group->logical_type()->is_nested()); + ASSERT_EQ(ConvertedType::NONE, group->converted_type()); + + format::SchemaElement group_element; + group->ToParquet(&group_element); + ASSERT_TRUE(group_element.__isset.logicalType); + ASSERT_TRUE(group_element.logicalType.__isset.VECTOR); + + format::SchemaElement list_element; + list->ToParquet(&list_element); + format::SchemaElement element_element; + element->ToParquet(&element_element); + std::unique_ptr roundtripped = GroupNode::FromParquet( + &group_element, + {GroupNode::FromParquet(&list_element, + {PrimitiveNode::FromParquet(&element_element)})}); + ASSERT_TRUE(roundtripped->logical_type()->is_vector()); + ASSERT_TRUE(group->Equals(roundtripped.get())); +} + TEST_F(TestGroupNode, Equals) { NodeVector f1 = Fields1(); NodeVector f2 = Fields1(); @@ -825,6 +912,63 @@ TEST_F(TestSchemaDescriptor, BuildTree) { ASSERT_EQ(nleaves, descr_.num_columns()); } +TEST_F(TestSchemaDescriptor, BuildTreeVector) { + NodePtr element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::FLOAT); + NodePtr vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/3); + NodePtr embedding = + GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, LogicalType::Vector()); + + descr_.Init(GroupNode::Make("schema", Repetition::REPEATED, {embedding})); + + ASSERT_EQ(1, descr_.num_columns()); + const ColumnDescriptor* col = descr_.Column(0); + EXPECT_EQ(1, col->max_definition_level()); + EXPECT_EQ(0, col->max_repetition_level()); + EXPECT_FALSE(col->schema_node()->is_vector()); + EXPECT_EQ(3, col->effective_vector_length()); +} + +TEST_F(TestSchemaDescriptor, BuildTreeNestedVector) { + NodePtr element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::FLOAT); + NodePtr inner_list = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/4); + NodePtr inner_vector = GroupNode::Make("element", Repetition::REQUIRED, {inner_list}, + LogicalType::Vector()); + NodePtr outer_list = GroupNode::Make("list", Repetition::VECTOR, {inner_vector}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/3); + NodePtr embedding = GroupNode::Make("embedding", Repetition::OPTIONAL, {outer_list}, + LogicalType::Vector()); + + descr_.Init(GroupNode::Make("schema", Repetition::REPEATED, {embedding})); + + ASSERT_EQ(1, descr_.num_columns()); + const ColumnDescriptor* col = descr_.Column(0); + EXPECT_EQ(1, col->max_definition_level()); + EXPECT_EQ(0, col->max_repetition_level()); + EXPECT_EQ(12, col->effective_vector_length()); +} + +TEST_F(TestSchemaDescriptor, BuildTreeNestedVectorLengthOverflow) { + NodePtr element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::FLOAT); + NodePtr inner_list = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/46341); + NodePtr inner_vector = GroupNode::Make("element", Repetition::REQUIRED, {inner_list}, + LogicalType::Vector()); + NodePtr outer_list = GroupNode::Make("list", Repetition::VECTOR, {inner_vector}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/46341); + NodePtr embedding = GroupNode::Make("embedding", Repetition::OPTIONAL, {outer_list}, + LogicalType::Vector()); + + ASSERT_THROW(descr_.Init(GroupNode::Make("schema", Repetition::REPEATED, {embedding})), + ParquetException); +} + TEST_F(TestSchemaDescriptor, HasRepeatedFields) { NodeVector fields; NodePtr schema; @@ -912,6 +1056,28 @@ TEST(TestSchemaPrinter, Examples) { ASSERT_EQ(expected, result); } +TEST(TestSchemaPrinter, Vector) { + NodePtr schema = GroupNode::Make( + "schema", Repetition::REPEATED, + {GroupNode::Make( + "embedding", Repetition::OPTIONAL, + {GroupNode::Make( + "list", Repetition::VECTOR, + {PrimitiveNode::Make("element", Repetition::REQUIRED, Type::FLOAT)}, + /*logical_type=*/nullptr, -1, 3)}, + LogicalType::Vector())}); + + std::string expected = R"(repeated group field_id=-1 schema { + optional group field_id=-1 embedding (Vector) { + vector group field_id=-1 list [3] { + required float field_id=-1 element; + } + } +} +)"; + ASSERT_EQ(expected, Print(schema)); +} + static void ConfirmFactoryEquivalence( ConvertedType::type converted_type, const std::shared_ptr& from_make, diff --git a/cpp/src/parquet/types.cc b/cpp/src/parquet/types.cc index 9d7604faec30..827e001a5f71 100644 --- a/cpp/src/parquet/types.cc +++ b/cpp/src/parquet/types.cc @@ -614,6 +614,8 @@ std::shared_ptr LogicalType::FromThrift( } return VariantLogicalType::Make(spec_version); + } else if (type.__isset.VECTOR) { + return VectorLogicalType::Make(); } else { // Sentinel type for one we do not recognize return UndefinedLogicalType::Make(); @@ -685,6 +687,10 @@ std::shared_ptr LogicalType::Variant(int8_t spec_version) { return VariantLogicalType::Make(spec_version); } +std::shared_ptr LogicalType::Vector() { + return VectorLogicalType::Make(); +} + std::shared_ptr LogicalType::None() { return NoLogicalType::Make(); } /* @@ -770,6 +776,7 @@ class LogicalType::Impl { class Geometry; class Geography; class Variant; + class Vector; class No; class Undefined; @@ -851,6 +858,7 @@ bool LogicalType::is_geography() const { bool LogicalType::is_variant() const { return impl_->type() == LogicalType::Type::VARIANT; } +bool LogicalType::is_vector() const { return impl_->type() == LogicalType::Type::VECTOR; } bool LogicalType::is_none() const { return impl_->type() == LogicalType::Type::NONE; } bool LogicalType::is_valid() const { return impl_->type() != LogicalType::Type::UNDEFINED; @@ -859,7 +867,8 @@ bool LogicalType::is_invalid() const { return !is_valid(); } bool LogicalType::is_nested() const { return impl_->type() == LogicalType::Type::LIST || impl_->type() == LogicalType::Type::MAP || - impl_->type() == LogicalType::Type::VARIANT; + impl_->type() == LogicalType::Type::VARIANT || + impl_->type() == LogicalType::Type::VECTOR; } bool LogicalType::is_nonnested() const { return !is_nested(); } bool LogicalType::is_serialized() const { return impl_->is_serialized(); } @@ -2028,6 +2037,20 @@ std::shared_ptr VariantLogicalType::Make(const int8_t spec_ve return logical_type; } +class LogicalType::Impl::Vector final : public LogicalType::Impl::Incompatible, + public LogicalType::Impl::Inapplicable { + public: + friend class VectorLogicalType; + + OVERRIDE_TOSTRING(Vector) + OVERRIDE_TOTHRIFT(VectorType, VECTOR) + + private: + Vector() : LogicalType::Impl(LogicalType::Type::VECTOR, SortOrder::UNKNOWN) {} +}; + +GENERATE_MAKE(Vector) + class LogicalType::Impl::No final : public LogicalType::Impl::SimpleCompatible, public LogicalType::Impl::UniversalApplicable { public: diff --git a/cpp/src/parquet/types.h b/cpp/src/parquet/types.h index 687353aa9bcb..6fcb24943f80 100644 --- a/cpp/src/parquet/types.h +++ b/cpp/src/parquet/types.h @@ -112,7 +112,13 @@ class LogicalType; // Mirrors parquet::FieldRepetitionType struct Repetition { - enum type { REQUIRED = 0, OPTIONAL = 1, REPEATED = 2, /*Always last*/ UNDEFINED = 3 }; + enum type { + REQUIRED = 0, + OPTIONAL = 1, + REPEATED = 2, + VECTOR = 3, + /*Always last*/ UNDEFINED = 4 + }; }; // Reference: @@ -162,6 +168,7 @@ class PARQUET_EXPORT LogicalType { GEOMETRY, GEOGRAPHY, VARIANT, + VECTOR, NONE // Not a real logical type; should always be last element }; }; @@ -231,6 +238,10 @@ class PARQUET_EXPORT LogicalType { static std::shared_ptr Variant( int8_t specVersion = kVariantSpecVersion); + /// \brief Create an experimental Vector logical type, annotating a group + /// node whose single child has VECTOR repetition. + static std::shared_ptr Vector(); + static std::shared_ptr Geometry(std::string crs = ""); static std::shared_ptr Geography( @@ -293,6 +304,7 @@ class PARQUET_EXPORT LogicalType { bool is_geometry() const; bool is_geography() const; bool is_variant() const; + bool is_vector() const; bool is_none() const; /// \brief Return true if this logical type is of a known type. bool is_valid() const; @@ -342,6 +354,16 @@ class PARQUET_EXPORT ListLogicalType : public LogicalType { ListLogicalType() = default; }; +/// \brief Allowed for group nodes only. EXPERIMENTAL: annotates a group whose +/// single child has VECTOR repetition. +class PARQUET_EXPORT VectorLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + VectorLogicalType() = default; +}; + /// \brief Allowed for physical type BYTE_ARRAY, must be encoded as UTF-8. class PARQUET_EXPORT EnumLogicalType : public LogicalType { public: diff --git a/python/pyarrow/_dataset_parquet.pyx b/python/pyarrow/_dataset_parquet.pyx index 534f7790923a..6567b95a9ab1 100644 --- a/python/pyarrow/_dataset_parquet.pyx +++ b/python/pyarrow/_dataset_parquet.pyx @@ -620,6 +620,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): "coerce_timestamps", "allow_truncated_timestamps", "use_compliant_nested_type", + "experimental_vector_encoding", } setters = set() @@ -676,6 +677,11 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): writer_engine_version="V2", use_compliant_nested_type=( self._properties["use_compliant_nested_type"] + ), + store_schema=True, + write_time_adjusted_to_utc=False, + experimental_vector_encoding=( + self._properties["experimental_vector_encoding"] ) ) @@ -705,6 +711,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): coerce_timestamps=None, allow_truncated_timestamps=False, use_compliant_nested_type=True, + experimental_vector_encoding=False, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 36fc2ccf2f33..fa0a06e53c7e 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -69,6 +69,7 @@ cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties( use_compliant_nested_type=*, store_schema=*, write_time_adjusted_to_utc=*, + experimental_vector_encoding=*, ) except * diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index 2358a961ebd9..39cb5dac42f0 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -2281,7 +2281,8 @@ cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties( writer_engine_version=None, use_compliant_nested_type=True, store_schema=True, - write_time_adjusted_to_utc=False) except *: + write_time_adjusted_to_utc=False, + experimental_vector_encoding=False) except *: """Arrow writer properties""" cdef: shared_ptr[ArrowWriterProperties] arrow_properties @@ -2322,6 +2323,11 @@ cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties( else: arrow_props.disable_compliant_nested_types() + # experimental_vector_encoding + + if experimental_vector_encoding: + arrow_props.enable_experimental_vector_encoding() + # writer_engine_version if writer_engine_version == "V1": @@ -2396,7 +2402,8 @@ cdef class ParquetWriter(_Weakrefable): store_decimal_as_integer=False, use_content_defined_chunking=False, write_time_adjusted_to_utc=False, - bloom_filter_options=None): + bloom_filter_options=None, + experimental_vector_encoding=False): cdef: shared_ptr[WriterProperties] properties shared_ptr[ArrowWriterProperties] arrow_properties @@ -2443,6 +2450,7 @@ cdef class ParquetWriter(_Weakrefable): use_compliant_nested_type=use_compliant_nested_type, store_schema=store_schema, write_time_adjusted_to_utc=write_time_adjusted_to_utc, + experimental_vector_encoding=experimental_vector_encoding, ) pool = maybe_unbox_memory_pool(memory_pool) diff --git a/python/pyarrow/includes/libparquet.pxd b/python/pyarrow/includes/libparquet.pxd index df353cc7805f..6515e6f1b75d 100644 --- a/python/pyarrow/includes/libparquet.pxd +++ b/python/pyarrow/includes/libparquet.pxd @@ -529,6 +529,7 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil: Builder* store_schema() Builder* enable_compliant_nested_types() Builder* disable_compliant_nested_types() + Builder* enable_experimental_vector_encoding() Builder* set_engine_version(ArrowWriterEngineVersion version) Builder* set_time_adjusted_to_utc(c_bool adjusted) shared_ptr[ArrowWriterProperties] build() diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index ff880fdcf52c..4ce1a495e69e 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -876,6 +876,13 @@ def _sanitize_table(table, new_schema, flavor): dictionary_pagesize_limit : int, default None Specify the dictionary page size limit per row group. If None, use the default 1MB. +experimental_vector_encoding : bool, default False + EXPERIMENTAL: Encode supported fixed-size list values, including the storage + of fixed-shape tensor extension arrays, using the canonical 3-level Parquet + VECTOR layout. Unsupported fixed-size list values fall back to the standard + LIST encoding: zero-length lists, variable-width or dictionary or extension + elements, struct elements containing fixed-size lists, and fixed-size lists + below nullable groups. store_schema : bool, default True By default, the Arrow schema is serialized and stored in the Parquet file metadata (in the "ARROW:schema" key). When reading the file, @@ -1083,6 +1090,7 @@ def __init__(self, where, schema, filesystem=None, store_decimal_as_integer=False, write_time_adjusted_to_utc=False, max_rows_per_page=None, + experimental_vector_encoding=False, **options): if use_deprecated_int96_timestamps is None: # Use int96 timestamps for Spark @@ -1138,6 +1146,7 @@ def __init__(self, where, schema, filesystem=None, store_decimal_as_integer=store_decimal_as_integer, write_time_adjusted_to_utc=write_time_adjusted_to_utc, max_rows_per_page=max_rows_per_page, + experimental_vector_encoding=experimental_vector_encoding, **options) self.is_open = True @@ -2017,6 +2026,7 @@ def write_table(table, where, row_group_size=None, version='2.6', write_time_adjusted_to_utc=False, max_rows_per_page=None, bloom_filter_options=None, + experimental_vector_encoding=False, **kwargs): # Implementor's note: when adding keywords here / updating defaults, also # update it in write_to_dataset and _dataset_parquet.pyx ParquetFileWriteOptions @@ -2051,6 +2061,7 @@ def write_table(table, where, row_group_size=None, version='2.6', write_time_adjusted_to_utc=write_time_adjusted_to_utc, max_rows_per_page=max_rows_per_page, bloom_filter_options=bloom_filter_options, + experimental_vector_encoding=experimental_vector_encoding, **kwargs) as writer: writer.write_table(table, row_group_size=row_group_size) except Exception: diff --git a/python/pyarrow/tests/parquet/test_data_types.py b/python/pyarrow/tests/parquet/test_data_types.py index c546bc1532ac..2d674a20036c 100644 --- a/python/pyarrow/tests/parquet/test_data_types.py +++ b/python/pyarrow/tests/parquet/test_data_types.py @@ -581,6 +581,34 @@ def test_json_extension_type(storage_type): store_schema=False) +def test_fixed_shape_tensor_vector_encoding(tempdir): + tensor_type = pa.fixed_shape_tensor(pa.float32(), [2, 3]) + storage = pa.array([ + [1, 2, 3, 4, 5, 6], + None, + [7, 8, 9, 10, 11, 12], + ], pa.list_(pa.float32(), 6)) + arr = pa.ExtensionArray.from_storage(tensor_type, storage) + table = pa.table({"tensor": arr}) + + sink = pa.BufferOutputStream() + pq.write_table(table, sink, experimental_vector_encoding=True) + buf = sink.getvalue() + + metadata = pq.read_metadata(pa.BufferReader(buf)) + assert "vector" in str(metadata.schema).lower() + + result = pq.read_table(pa.BufferReader(buf)) + assert result.schema == table.schema + assert result.equals(table) + + dataset_dir = tempdir / "vector_dataset" + pq.write_to_dataset(table, dataset_dir, experimental_vector_encoding=True) + dataset_result = pq.read_table(dataset_dir) + assert dataset_result.schema == table.schema + assert dataset_result.equals(table) + + def test_uuid_extension_type(): data = [ b'\xe4`\xf9p\x83QGN\xac\x7f\xa4g>K\xa8\xcb',