From 2d0482938f1f5ac69b94765c0d6066d8a98e75e7 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Thu, 11 Jun 2026 12:44:22 +0200 Subject: [PATCH 1/8] Add experimental VECTOR repetition and Vector logical type to the Parquet format VECTOR is a fourth FieldRepetitionType: the field repeats exactly SchemaElement.vector_length times per parent value without increasing the maximum definition or repetition level of its descendants. The Vector logical type annotates the outer group of the 3-level vector structure. The normative layout rules (canonical structure, fixed stride for null vector values, filler repetition levels, whole-vector pages, nested length product, positive length) live as non-doc comments in parquet.thrift so they are not copied into generated code. Generated files regenerated with Thrift 0.21.0. --- cpp/src/generated/parquet_types.cpp | 1733 ++++++++++++++------------- cpp/src/generated/parquet_types.h | 48 +- cpp/src/generated/parquet_types.tcc | 554 +++++---- cpp/src/parquet/parquet.thrift | 79 ++ cpp/src/parquet/types.cc | 25 +- cpp/src/parquet/types.h | 24 +- 6 files changed, 1386 insertions(+), 1077 deletions(-) diff --git a/cpp/src/generated/parquet_types.cpp b/cpp/src/generated/parquet_types.cpp index cf8debb79e73..01b7b5b4a969 100644 --- a/cpp/src/generated/parquet_types.cpp +++ b/cpp/src/generated/parquet_types.cpp @@ -335,7 +335,8 @@ int _kFieldRepetitionTypeValues[] = { /** * The field is repeated and can contain 0 or more values */ - FieldRepetitionType::REPEATED + FieldRepetitionType::REPEATED, + FieldRepetitionType::VECTOR }; const char* _kFieldRepetitionTypeNames[] = { /** @@ -349,9 +350,10 @@ const char* _kFieldRepetitionTypeNames[] = { /** * The field is repeated and can contain 0 or more values */ - "REPEATED" + "REPEATED", + "VECTOR" }; -const std::map _FieldRepetitionType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kFieldRepetitionTypeValues, _kFieldRepetitionTypeNames), ::apache::thrift::TEnumIterator(-1, nullptr, nullptr)); +const std::map _FieldRepetitionType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(4, _kFieldRepetitionTypeValues, _kFieldRepetitionTypeNames), ::apache::thrift::TEnumIterator(-1, nullptr, nullptr)); std::ostream& operator<<(std::ostream& out, const FieldRepetitionType::type& val) { std::map::const_iterator it = _FieldRepetitionType_VALUES_TO_NAMES.find(val); @@ -2273,6 +2275,50 @@ void GeographyType::printTo(std::ostream& out) const { } +VectorType::~VectorType() noexcept { +} + +VectorType::VectorType() noexcept { +} +std::ostream& operator<<(std::ostream& out, const VectorType& obj) +{ + obj.printTo(out); + return out; +} + + +void swap(VectorType &a, VectorType &b) noexcept { + using ::std::swap; + (void) a; + (void) b; +} + +bool VectorType::operator==(const VectorType & /* rhs */) const +{ + return true; +} + +VectorType::VectorType(const VectorType& other119) noexcept { + (void) other119; +} +VectorType::VectorType(VectorType&& other120) noexcept { + (void) other120; +} +VectorType& VectorType::operator=(const VectorType& other121) noexcept { + (void) other121; + return *this; +} +VectorType& VectorType::operator=(VectorType&& other122) noexcept { + (void) other122; + return *this; +} +void VectorType::printTo(std::ostream& out) const { + using ::apache::thrift::to_string; + out << "VectorType("; + out << ")"; +} + + LogicalType::~LogicalType() noexcept { } @@ -2363,6 +2409,11 @@ void LogicalType::__set_GEOGRAPHY(const GeographyType& val) { this->GEOGRAPHY = val; __isset.GEOGRAPHY = true; } + +void LogicalType::__set_VECTOR(const VectorType& val) { + this->VECTOR = val; +__isset.VECTOR = true; +} std::ostream& operator<<(std::ostream& out, const LogicalType& obj) { obj.printTo(out); @@ -2389,6 +2440,7 @@ void swap(LogicalType &a, LogicalType &b) noexcept { swap(a.VARIANT, b.VARIANT); swap(a.GEOMETRY, b.GEOMETRY); swap(a.GEOGRAPHY, b.GEOGRAPHY); + swap(a.VECTOR, b.VECTOR); swap(a.__isset, b.__isset); } @@ -2462,89 +2514,97 @@ bool LogicalType::operator==(const LogicalType & rhs) const return false; else if (__isset.GEOGRAPHY && !(GEOGRAPHY == rhs.GEOGRAPHY)) return false; + if (__isset.VECTOR != rhs.__isset.VECTOR) + return false; + else if (__isset.VECTOR && !(VECTOR == rhs.VECTOR)) + return false; return true; } -LogicalType::LogicalType(const LogicalType& other119) { - STRING = other119.STRING; - MAP = other119.MAP; - LIST = other119.LIST; - ENUM = other119.ENUM; - DECIMAL = other119.DECIMAL; - DATE = other119.DATE; - TIME = other119.TIME; - TIMESTAMP = other119.TIMESTAMP; - INTEGER = other119.INTEGER; - UNKNOWN = other119.UNKNOWN; - JSON = other119.JSON; - BSON = other119.BSON; - UUID = other119.UUID; - FLOAT16 = other119.FLOAT16; - VARIANT = other119.VARIANT; - GEOMETRY = other119.GEOMETRY; - GEOGRAPHY = other119.GEOGRAPHY; - __isset = other119.__isset; -} -LogicalType::LogicalType(LogicalType&& other120) noexcept { - STRING = std::move(other120.STRING); - MAP = std::move(other120.MAP); - LIST = std::move(other120.LIST); - ENUM = std::move(other120.ENUM); - DECIMAL = std::move(other120.DECIMAL); - DATE = std::move(other120.DATE); - TIME = std::move(other120.TIME); - TIMESTAMP = std::move(other120.TIMESTAMP); - INTEGER = std::move(other120.INTEGER); - UNKNOWN = std::move(other120.UNKNOWN); - JSON = std::move(other120.JSON); - BSON = std::move(other120.BSON); - UUID = std::move(other120.UUID); - FLOAT16 = std::move(other120.FLOAT16); - VARIANT = std::move(other120.VARIANT); - GEOMETRY = std::move(other120.GEOMETRY); - GEOGRAPHY = std::move(other120.GEOGRAPHY); - __isset = other120.__isset; -} -LogicalType& LogicalType::operator=(const LogicalType& other121) { - STRING = other121.STRING; - MAP = other121.MAP; - LIST = other121.LIST; - ENUM = other121.ENUM; - DECIMAL = other121.DECIMAL; - DATE = other121.DATE; - TIME = other121.TIME; - TIMESTAMP = other121.TIMESTAMP; - INTEGER = other121.INTEGER; - UNKNOWN = other121.UNKNOWN; - JSON = other121.JSON; - BSON = other121.BSON; - UUID = other121.UUID; - FLOAT16 = other121.FLOAT16; - VARIANT = other121.VARIANT; - GEOMETRY = other121.GEOMETRY; - GEOGRAPHY = other121.GEOGRAPHY; - __isset = other121.__isset; +LogicalType::LogicalType(const LogicalType& other123) { + STRING = other123.STRING; + MAP = other123.MAP; + LIST = other123.LIST; + ENUM = other123.ENUM; + DECIMAL = other123.DECIMAL; + DATE = other123.DATE; + TIME = other123.TIME; + TIMESTAMP = other123.TIMESTAMP; + INTEGER = other123.INTEGER; + UNKNOWN = other123.UNKNOWN; + JSON = other123.JSON; + BSON = other123.BSON; + UUID = other123.UUID; + FLOAT16 = other123.FLOAT16; + VARIANT = other123.VARIANT; + GEOMETRY = other123.GEOMETRY; + GEOGRAPHY = other123.GEOGRAPHY; + VECTOR = other123.VECTOR; + __isset = other123.__isset; +} +LogicalType::LogicalType(LogicalType&& other124) noexcept { + STRING = std::move(other124.STRING); + MAP = std::move(other124.MAP); + LIST = std::move(other124.LIST); + ENUM = std::move(other124.ENUM); + DECIMAL = std::move(other124.DECIMAL); + DATE = std::move(other124.DATE); + TIME = std::move(other124.TIME); + TIMESTAMP = std::move(other124.TIMESTAMP); + INTEGER = std::move(other124.INTEGER); + UNKNOWN = std::move(other124.UNKNOWN); + JSON = std::move(other124.JSON); + BSON = std::move(other124.BSON); + UUID = std::move(other124.UUID); + FLOAT16 = std::move(other124.FLOAT16); + VARIANT = std::move(other124.VARIANT); + GEOMETRY = std::move(other124.GEOMETRY); + GEOGRAPHY = std::move(other124.GEOGRAPHY); + VECTOR = std::move(other124.VECTOR); + __isset = other124.__isset; +} +LogicalType& LogicalType::operator=(const LogicalType& other125) { + STRING = other125.STRING; + MAP = other125.MAP; + LIST = other125.LIST; + ENUM = other125.ENUM; + DECIMAL = other125.DECIMAL; + DATE = other125.DATE; + TIME = other125.TIME; + TIMESTAMP = other125.TIMESTAMP; + INTEGER = other125.INTEGER; + UNKNOWN = other125.UNKNOWN; + JSON = other125.JSON; + BSON = other125.BSON; + UUID = other125.UUID; + FLOAT16 = other125.FLOAT16; + VARIANT = other125.VARIANT; + GEOMETRY = other125.GEOMETRY; + GEOGRAPHY = other125.GEOGRAPHY; + VECTOR = other125.VECTOR; + __isset = other125.__isset; return *this; } -LogicalType& LogicalType::operator=(LogicalType&& other122) noexcept { - STRING = std::move(other122.STRING); - MAP = std::move(other122.MAP); - LIST = std::move(other122.LIST); - ENUM = std::move(other122.ENUM); - DECIMAL = std::move(other122.DECIMAL); - DATE = std::move(other122.DATE); - TIME = std::move(other122.TIME); - TIMESTAMP = std::move(other122.TIMESTAMP); - INTEGER = std::move(other122.INTEGER); - UNKNOWN = std::move(other122.UNKNOWN); - JSON = std::move(other122.JSON); - BSON = std::move(other122.BSON); - UUID = std::move(other122.UUID); - FLOAT16 = std::move(other122.FLOAT16); - VARIANT = std::move(other122.VARIANT); - GEOMETRY = std::move(other122.GEOMETRY); - GEOGRAPHY = std::move(other122.GEOGRAPHY); - __isset = other122.__isset; +LogicalType& LogicalType::operator=(LogicalType&& other126) noexcept { + STRING = std::move(other126.STRING); + MAP = std::move(other126.MAP); + LIST = std::move(other126.LIST); + ENUM = std::move(other126.ENUM); + DECIMAL = std::move(other126.DECIMAL); + DATE = std::move(other126.DATE); + TIME = std::move(other126.TIME); + TIMESTAMP = std::move(other126.TIMESTAMP); + INTEGER = std::move(other126.INTEGER); + UNKNOWN = std::move(other126.UNKNOWN); + JSON = std::move(other126.JSON); + BSON = std::move(other126.BSON); + UUID = std::move(other126.UUID); + FLOAT16 = std::move(other126.FLOAT16); + VARIANT = std::move(other126.VARIANT); + GEOMETRY = std::move(other126.GEOMETRY); + GEOGRAPHY = std::move(other126.GEOGRAPHY); + VECTOR = std::move(other126.VECTOR); + __isset = other126.__isset; return *this; } void LogicalType::printTo(std::ostream& out) const { @@ -2567,6 +2627,7 @@ void LogicalType::printTo(std::ostream& out) const { out << ", " << "VARIANT="; (__isset.VARIANT ? (out << to_string(VARIANT)) : (out << "")); out << ", " << "GEOMETRY="; (__isset.GEOMETRY ? (out << to_string(GEOMETRY)) : (out << "")); out << ", " << "GEOGRAPHY="; (__isset.GEOGRAPHY ? (out << to_string(GEOGRAPHY)) : (out << "")); + out << ", " << "VECTOR="; (__isset.VECTOR ? (out << to_string(VECTOR)) : (out << "")); out << ")"; } @@ -2583,7 +2644,8 @@ SchemaElement::SchemaElement() noexcept converted_type(static_cast(0)), scale(0), precision(0), - field_id(0) { + field_id(0), + vector_length(0) { } void SchemaElement::__set_type(const Type::type val) { @@ -2634,6 +2696,11 @@ void SchemaElement::__set_logicalType(const LogicalType& val) { this->logicalType = val; __isset.logicalType = true; } + +void SchemaElement::__set_vector_length(const int32_t val) { + this->vector_length = val; +__isset.vector_length = true; +} std::ostream& operator<<(std::ostream& out, const SchemaElement& obj) { obj.printTo(out); @@ -2653,6 +2720,7 @@ void swap(SchemaElement &a, SchemaElement &b) noexcept { swap(a.precision, b.precision); swap(a.field_id, b.field_id); swap(a.logicalType, b.logicalType); + swap(a.vector_length, b.vector_length); swap(a.__isset, b.__isset); } @@ -2696,61 +2764,69 @@ bool SchemaElement::operator==(const SchemaElement & rhs) const return false; else if (__isset.logicalType && !(logicalType == rhs.logicalType)) return false; + if (__isset.vector_length != rhs.__isset.vector_length) + return false; + else if (__isset.vector_length && !(vector_length == rhs.vector_length)) + return false; return true; } -SchemaElement::SchemaElement(const SchemaElement& other126) { - type = other126.type; - type_length = other126.type_length; - repetition_type = other126.repetition_type; - name = other126.name; - num_children = other126.num_children; - converted_type = other126.converted_type; - scale = other126.scale; - precision = other126.precision; - field_id = other126.field_id; - logicalType = other126.logicalType; - __isset = other126.__isset; -} -SchemaElement::SchemaElement(SchemaElement&& other127) noexcept { - type = other127.type; - type_length = other127.type_length; - repetition_type = other127.repetition_type; - name = std::move(other127.name); - num_children = other127.num_children; - converted_type = other127.converted_type; - scale = other127.scale; - precision = other127.precision; - field_id = other127.field_id; - logicalType = std::move(other127.logicalType); - __isset = other127.__isset; -} -SchemaElement& SchemaElement::operator=(const SchemaElement& other128) { - type = other128.type; - type_length = other128.type_length; - repetition_type = other128.repetition_type; - name = other128.name; - num_children = other128.num_children; - converted_type = other128.converted_type; - scale = other128.scale; - precision = other128.precision; - field_id = other128.field_id; - logicalType = other128.logicalType; - __isset = other128.__isset; +SchemaElement::SchemaElement(const SchemaElement& other130) { + type = other130.type; + type_length = other130.type_length; + repetition_type = other130.repetition_type; + name = other130.name; + num_children = other130.num_children; + converted_type = other130.converted_type; + scale = other130.scale; + precision = other130.precision; + field_id = other130.field_id; + logicalType = other130.logicalType; + vector_length = other130.vector_length; + __isset = other130.__isset; +} +SchemaElement::SchemaElement(SchemaElement&& other131) noexcept { + type = other131.type; + type_length = other131.type_length; + repetition_type = other131.repetition_type; + name = std::move(other131.name); + num_children = other131.num_children; + converted_type = other131.converted_type; + scale = other131.scale; + precision = other131.precision; + field_id = other131.field_id; + logicalType = std::move(other131.logicalType); + vector_length = other131.vector_length; + __isset = other131.__isset; +} +SchemaElement& SchemaElement::operator=(const SchemaElement& other132) { + type = other132.type; + type_length = other132.type_length; + repetition_type = other132.repetition_type; + name = other132.name; + num_children = other132.num_children; + converted_type = other132.converted_type; + scale = other132.scale; + precision = other132.precision; + field_id = other132.field_id; + logicalType = other132.logicalType; + vector_length = other132.vector_length; + __isset = other132.__isset; return *this; } -SchemaElement& SchemaElement::operator=(SchemaElement&& other129) noexcept { - type = other129.type; - type_length = other129.type_length; - repetition_type = other129.repetition_type; - name = std::move(other129.name); - num_children = other129.num_children; - converted_type = other129.converted_type; - scale = other129.scale; - precision = other129.precision; - field_id = other129.field_id; - logicalType = std::move(other129.logicalType); - __isset = other129.__isset; +SchemaElement& SchemaElement::operator=(SchemaElement&& other133) noexcept { + type = other133.type; + type_length = other133.type_length; + repetition_type = other133.repetition_type; + name = std::move(other133.name); + num_children = other133.num_children; + converted_type = other133.converted_type; + scale = other133.scale; + precision = other133.precision; + field_id = other133.field_id; + logicalType = std::move(other133.logicalType); + vector_length = other133.vector_length; + __isset = other133.__isset; return *this; } void SchemaElement::printTo(std::ostream& out) const { @@ -2766,6 +2842,7 @@ void SchemaElement::printTo(std::ostream& out) const { out << ", " << "precision="; (__isset.precision ? (out << to_string(precision)) : (out << "")); out << ", " << "field_id="; (__isset.field_id ? (out << to_string(field_id)) : (out << "")); out << ", " << "logicalType="; (__isset.logicalType ? (out << to_string(logicalType)) : (out << "")); + out << ", " << "vector_length="; (__isset.vector_length ? (out << to_string(vector_length)) : (out << "")); out << ")"; } @@ -2834,38 +2911,38 @@ bool DataPageHeader::operator==(const DataPageHeader & rhs) const return true; } -DataPageHeader::DataPageHeader(const DataPageHeader& other133) { - num_values = other133.num_values; - encoding = other133.encoding; - definition_level_encoding = other133.definition_level_encoding; - repetition_level_encoding = other133.repetition_level_encoding; - statistics = other133.statistics; - __isset = other133.__isset; -} -DataPageHeader::DataPageHeader(DataPageHeader&& other134) noexcept { - num_values = other134.num_values; - encoding = other134.encoding; - definition_level_encoding = other134.definition_level_encoding; - repetition_level_encoding = other134.repetition_level_encoding; - statistics = std::move(other134.statistics); - __isset = other134.__isset; -} -DataPageHeader& DataPageHeader::operator=(const DataPageHeader& other135) { - num_values = other135.num_values; - encoding = other135.encoding; - definition_level_encoding = other135.definition_level_encoding; - repetition_level_encoding = other135.repetition_level_encoding; - statistics = other135.statistics; - __isset = other135.__isset; +DataPageHeader::DataPageHeader(const DataPageHeader& other137) { + num_values = other137.num_values; + encoding = other137.encoding; + definition_level_encoding = other137.definition_level_encoding; + repetition_level_encoding = other137.repetition_level_encoding; + statistics = other137.statistics; + __isset = other137.__isset; +} +DataPageHeader::DataPageHeader(DataPageHeader&& other138) noexcept { + num_values = other138.num_values; + encoding = other138.encoding; + definition_level_encoding = other138.definition_level_encoding; + repetition_level_encoding = other138.repetition_level_encoding; + statistics = std::move(other138.statistics); + __isset = other138.__isset; +} +DataPageHeader& DataPageHeader::operator=(const DataPageHeader& other139) { + num_values = other139.num_values; + encoding = other139.encoding; + definition_level_encoding = other139.definition_level_encoding; + repetition_level_encoding = other139.repetition_level_encoding; + statistics = other139.statistics; + __isset = other139.__isset; return *this; } -DataPageHeader& DataPageHeader::operator=(DataPageHeader&& other136) noexcept { - num_values = other136.num_values; - encoding = other136.encoding; - definition_level_encoding = other136.definition_level_encoding; - repetition_level_encoding = other136.repetition_level_encoding; - statistics = std::move(other136.statistics); - __isset = other136.__isset; +DataPageHeader& DataPageHeader::operator=(DataPageHeader&& other140) noexcept { + num_values = other140.num_values; + encoding = other140.encoding; + definition_level_encoding = other140.definition_level_encoding; + repetition_level_encoding = other140.repetition_level_encoding; + statistics = std::move(other140.statistics); + __isset = other140.__isset; return *this; } void DataPageHeader::printTo(std::ostream& out) const { @@ -2903,18 +2980,18 @@ bool IndexPageHeader::operator==(const IndexPageHeader & /* rhs */) const return true; } -IndexPageHeader::IndexPageHeader(const IndexPageHeader& other137) noexcept { - (void) other137; +IndexPageHeader::IndexPageHeader(const IndexPageHeader& other141) noexcept { + (void) other141; } -IndexPageHeader::IndexPageHeader(IndexPageHeader&& other138) noexcept { - (void) other138; +IndexPageHeader::IndexPageHeader(IndexPageHeader&& other142) noexcept { + (void) other142; } -IndexPageHeader& IndexPageHeader::operator=(const IndexPageHeader& other139) noexcept { - (void) other139; +IndexPageHeader& IndexPageHeader::operator=(const IndexPageHeader& other143) noexcept { + (void) other143; return *this; } -IndexPageHeader& IndexPageHeader::operator=(IndexPageHeader&& other140) noexcept { - (void) other140; +IndexPageHeader& IndexPageHeader::operator=(IndexPageHeader&& other144) noexcept { + (void) other144; return *this; } void IndexPageHeader::printTo(std::ostream& out) const { @@ -2973,30 +3050,30 @@ bool DictionaryPageHeader::operator==(const DictionaryPageHeader & rhs) const return true; } -DictionaryPageHeader::DictionaryPageHeader(const DictionaryPageHeader& other142) noexcept { - num_values = other142.num_values; - encoding = other142.encoding; - is_sorted = other142.is_sorted; - __isset = other142.__isset; +DictionaryPageHeader::DictionaryPageHeader(const DictionaryPageHeader& other146) noexcept { + num_values = other146.num_values; + encoding = other146.encoding; + is_sorted = other146.is_sorted; + __isset = other146.__isset; } -DictionaryPageHeader::DictionaryPageHeader(DictionaryPageHeader&& other143) noexcept { - num_values = other143.num_values; - encoding = other143.encoding; - is_sorted = other143.is_sorted; - __isset = other143.__isset; +DictionaryPageHeader::DictionaryPageHeader(DictionaryPageHeader&& other147) noexcept { + num_values = other147.num_values; + encoding = other147.encoding; + is_sorted = other147.is_sorted; + __isset = other147.__isset; } -DictionaryPageHeader& DictionaryPageHeader::operator=(const DictionaryPageHeader& other144) noexcept { - num_values = other144.num_values; - encoding = other144.encoding; - is_sorted = other144.is_sorted; - __isset = other144.__isset; +DictionaryPageHeader& DictionaryPageHeader::operator=(const DictionaryPageHeader& other148) noexcept { + num_values = other148.num_values; + encoding = other148.encoding; + is_sorted = other148.is_sorted; + __isset = other148.__isset; return *this; } -DictionaryPageHeader& DictionaryPageHeader::operator=(DictionaryPageHeader&& other145) noexcept { - num_values = other145.num_values; - encoding = other145.encoding; - is_sorted = other145.is_sorted; - __isset = other145.__isset; +DictionaryPageHeader& DictionaryPageHeader::operator=(DictionaryPageHeader&& other149) noexcept { + num_values = other149.num_values; + encoding = other149.encoding; + is_sorted = other149.is_sorted; + __isset = other149.__isset; return *this; } void DictionaryPageHeader::printTo(std::ostream& out) const { @@ -3100,50 +3177,50 @@ bool DataPageHeaderV2::operator==(const DataPageHeaderV2 & rhs) const return true; } -DataPageHeaderV2::DataPageHeaderV2(const DataPageHeaderV2& other147) { - num_values = other147.num_values; - num_nulls = other147.num_nulls; - num_rows = other147.num_rows; - encoding = other147.encoding; - definition_levels_byte_length = other147.definition_levels_byte_length; - repetition_levels_byte_length = other147.repetition_levels_byte_length; - is_compressed = other147.is_compressed; - statistics = other147.statistics; - __isset = other147.__isset; -} -DataPageHeaderV2::DataPageHeaderV2(DataPageHeaderV2&& other148) noexcept { - num_values = other148.num_values; - num_nulls = other148.num_nulls; - num_rows = other148.num_rows; - encoding = other148.encoding; - definition_levels_byte_length = other148.definition_levels_byte_length; - repetition_levels_byte_length = other148.repetition_levels_byte_length; - is_compressed = other148.is_compressed; - statistics = std::move(other148.statistics); - __isset = other148.__isset; -} -DataPageHeaderV2& DataPageHeaderV2::operator=(const DataPageHeaderV2& other149) { - num_values = other149.num_values; - num_nulls = other149.num_nulls; - num_rows = other149.num_rows; - encoding = other149.encoding; - definition_levels_byte_length = other149.definition_levels_byte_length; - repetition_levels_byte_length = other149.repetition_levels_byte_length; - is_compressed = other149.is_compressed; - statistics = other149.statistics; - __isset = other149.__isset; +DataPageHeaderV2::DataPageHeaderV2(const DataPageHeaderV2& other151) { + num_values = other151.num_values; + num_nulls = other151.num_nulls; + num_rows = other151.num_rows; + encoding = other151.encoding; + definition_levels_byte_length = other151.definition_levels_byte_length; + repetition_levels_byte_length = other151.repetition_levels_byte_length; + is_compressed = other151.is_compressed; + statistics = other151.statistics; + __isset = other151.__isset; +} +DataPageHeaderV2::DataPageHeaderV2(DataPageHeaderV2&& other152) noexcept { + num_values = other152.num_values; + num_nulls = other152.num_nulls; + num_rows = other152.num_rows; + encoding = other152.encoding; + definition_levels_byte_length = other152.definition_levels_byte_length; + repetition_levels_byte_length = other152.repetition_levels_byte_length; + is_compressed = other152.is_compressed; + statistics = std::move(other152.statistics); + __isset = other152.__isset; +} +DataPageHeaderV2& DataPageHeaderV2::operator=(const DataPageHeaderV2& other153) { + num_values = other153.num_values; + num_nulls = other153.num_nulls; + num_rows = other153.num_rows; + encoding = other153.encoding; + definition_levels_byte_length = other153.definition_levels_byte_length; + repetition_levels_byte_length = other153.repetition_levels_byte_length; + is_compressed = other153.is_compressed; + statistics = other153.statistics; + __isset = other153.__isset; return *this; } -DataPageHeaderV2& DataPageHeaderV2::operator=(DataPageHeaderV2&& other150) noexcept { - num_values = other150.num_values; - num_nulls = other150.num_nulls; - num_rows = other150.num_rows; - encoding = other150.encoding; - definition_levels_byte_length = other150.definition_levels_byte_length; - repetition_levels_byte_length = other150.repetition_levels_byte_length; - is_compressed = other150.is_compressed; - statistics = std::move(other150.statistics); - __isset = other150.__isset; +DataPageHeaderV2& DataPageHeaderV2::operator=(DataPageHeaderV2&& other154) noexcept { + num_values = other154.num_values; + num_nulls = other154.num_nulls; + num_rows = other154.num_rows; + encoding = other154.encoding; + definition_levels_byte_length = other154.definition_levels_byte_length; + repetition_levels_byte_length = other154.repetition_levels_byte_length; + is_compressed = other154.is_compressed; + statistics = std::move(other154.statistics); + __isset = other154.__isset; return *this; } void DataPageHeaderV2::printTo(std::ostream& out) const { @@ -3184,18 +3261,18 @@ bool SplitBlockAlgorithm::operator==(const SplitBlockAlgorithm & /* rhs */) cons return true; } -SplitBlockAlgorithm::SplitBlockAlgorithm(const SplitBlockAlgorithm& other151) noexcept { - (void) other151; +SplitBlockAlgorithm::SplitBlockAlgorithm(const SplitBlockAlgorithm& other155) noexcept { + (void) other155; } -SplitBlockAlgorithm::SplitBlockAlgorithm(SplitBlockAlgorithm&& other152) noexcept { - (void) other152; +SplitBlockAlgorithm::SplitBlockAlgorithm(SplitBlockAlgorithm&& other156) noexcept { + (void) other156; } -SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(const SplitBlockAlgorithm& other153) noexcept { - (void) other153; +SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(const SplitBlockAlgorithm& other157) noexcept { + (void) other157; return *this; } -SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(SplitBlockAlgorithm&& other154) noexcept { - (void) other154; +SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(SplitBlockAlgorithm&& other158) noexcept { + (void) other158; return *this; } void SplitBlockAlgorithm::printTo(std::ostream& out) const { @@ -3237,22 +3314,22 @@ bool BloomFilterAlgorithm::operator==(const BloomFilterAlgorithm & rhs) const return true; } -BloomFilterAlgorithm::BloomFilterAlgorithm(const BloomFilterAlgorithm& other155) noexcept { - BLOCK = other155.BLOCK; - __isset = other155.__isset; +BloomFilterAlgorithm::BloomFilterAlgorithm(const BloomFilterAlgorithm& other159) noexcept { + BLOCK = other159.BLOCK; + __isset = other159.__isset; } -BloomFilterAlgorithm::BloomFilterAlgorithm(BloomFilterAlgorithm&& other156) noexcept { - BLOCK = std::move(other156.BLOCK); - __isset = other156.__isset; +BloomFilterAlgorithm::BloomFilterAlgorithm(BloomFilterAlgorithm&& other160) noexcept { + BLOCK = std::move(other160.BLOCK); + __isset = other160.__isset; } -BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(const BloomFilterAlgorithm& other157) noexcept { - BLOCK = other157.BLOCK; - __isset = other157.__isset; +BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(const BloomFilterAlgorithm& other161) noexcept { + BLOCK = other161.BLOCK; + __isset = other161.__isset; return *this; } -BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(BloomFilterAlgorithm&& other158) noexcept { - BLOCK = std::move(other158.BLOCK); - __isset = other158.__isset; +BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(BloomFilterAlgorithm&& other162) noexcept { + BLOCK = std::move(other162.BLOCK); + __isset = other162.__isset; return *this; } void BloomFilterAlgorithm::printTo(std::ostream& out) const { @@ -3286,18 +3363,18 @@ bool XxHash::operator==(const XxHash & /* rhs */) const return true; } -XxHash::XxHash(const XxHash& other159) noexcept { - (void) other159; +XxHash::XxHash(const XxHash& other163) noexcept { + (void) other163; } -XxHash::XxHash(XxHash&& other160) noexcept { - (void) other160; +XxHash::XxHash(XxHash&& other164) noexcept { + (void) other164; } -XxHash& XxHash::operator=(const XxHash& other161) noexcept { - (void) other161; +XxHash& XxHash::operator=(const XxHash& other165) noexcept { + (void) other165; return *this; } -XxHash& XxHash::operator=(XxHash&& other162) noexcept { - (void) other162; +XxHash& XxHash::operator=(XxHash&& other166) noexcept { + (void) other166; return *this; } void XxHash::printTo(std::ostream& out) const { @@ -3339,22 +3416,22 @@ bool BloomFilterHash::operator==(const BloomFilterHash & rhs) const return true; } -BloomFilterHash::BloomFilterHash(const BloomFilterHash& other163) noexcept { - XXHASH = other163.XXHASH; - __isset = other163.__isset; +BloomFilterHash::BloomFilterHash(const BloomFilterHash& other167) noexcept { + XXHASH = other167.XXHASH; + __isset = other167.__isset; } -BloomFilterHash::BloomFilterHash(BloomFilterHash&& other164) noexcept { - XXHASH = std::move(other164.XXHASH); - __isset = other164.__isset; +BloomFilterHash::BloomFilterHash(BloomFilterHash&& other168) noexcept { + XXHASH = std::move(other168.XXHASH); + __isset = other168.__isset; } -BloomFilterHash& BloomFilterHash::operator=(const BloomFilterHash& other165) noexcept { - XXHASH = other165.XXHASH; - __isset = other165.__isset; +BloomFilterHash& BloomFilterHash::operator=(const BloomFilterHash& other169) noexcept { + XXHASH = other169.XXHASH; + __isset = other169.__isset; return *this; } -BloomFilterHash& BloomFilterHash::operator=(BloomFilterHash&& other166) noexcept { - XXHASH = std::move(other166.XXHASH); - __isset = other166.__isset; +BloomFilterHash& BloomFilterHash::operator=(BloomFilterHash&& other170) noexcept { + XXHASH = std::move(other170.XXHASH); + __isset = other170.__isset; return *this; } void BloomFilterHash::printTo(std::ostream& out) const { @@ -3388,18 +3465,18 @@ bool Uncompressed::operator==(const Uncompressed & /* rhs */) const return true; } -Uncompressed::Uncompressed(const Uncompressed& other167) noexcept { - (void) other167; +Uncompressed::Uncompressed(const Uncompressed& other171) noexcept { + (void) other171; } -Uncompressed::Uncompressed(Uncompressed&& other168) noexcept { - (void) other168; +Uncompressed::Uncompressed(Uncompressed&& other172) noexcept { + (void) other172; } -Uncompressed& Uncompressed::operator=(const Uncompressed& other169) noexcept { - (void) other169; +Uncompressed& Uncompressed::operator=(const Uncompressed& other173) noexcept { + (void) other173; return *this; } -Uncompressed& Uncompressed::operator=(Uncompressed&& other170) noexcept { - (void) other170; +Uncompressed& Uncompressed::operator=(Uncompressed&& other174) noexcept { + (void) other174; return *this; } void Uncompressed::printTo(std::ostream& out) const { @@ -3441,22 +3518,22 @@ bool BloomFilterCompression::operator==(const BloomFilterCompression & rhs) cons return true; } -BloomFilterCompression::BloomFilterCompression(const BloomFilterCompression& other171) noexcept { - UNCOMPRESSED = other171.UNCOMPRESSED; - __isset = other171.__isset; +BloomFilterCompression::BloomFilterCompression(const BloomFilterCompression& other175) noexcept { + UNCOMPRESSED = other175.UNCOMPRESSED; + __isset = other175.__isset; } -BloomFilterCompression::BloomFilterCompression(BloomFilterCompression&& other172) noexcept { - UNCOMPRESSED = std::move(other172.UNCOMPRESSED); - __isset = other172.__isset; +BloomFilterCompression::BloomFilterCompression(BloomFilterCompression&& other176) noexcept { + UNCOMPRESSED = std::move(other176.UNCOMPRESSED); + __isset = other176.__isset; } -BloomFilterCompression& BloomFilterCompression::operator=(const BloomFilterCompression& other173) noexcept { - UNCOMPRESSED = other173.UNCOMPRESSED; - __isset = other173.__isset; +BloomFilterCompression& BloomFilterCompression::operator=(const BloomFilterCompression& other177) noexcept { + UNCOMPRESSED = other177.UNCOMPRESSED; + __isset = other177.__isset; return *this; } -BloomFilterCompression& BloomFilterCompression::operator=(BloomFilterCompression&& other174) noexcept { - UNCOMPRESSED = std::move(other174.UNCOMPRESSED); - __isset = other174.__isset; +BloomFilterCompression& BloomFilterCompression::operator=(BloomFilterCompression&& other178) noexcept { + UNCOMPRESSED = std::move(other178.UNCOMPRESSED); + __isset = other178.__isset; return *this; } void BloomFilterCompression::printTo(std::ostream& out) const { @@ -3517,30 +3594,30 @@ bool BloomFilterHeader::operator==(const BloomFilterHeader & rhs) const return true; } -BloomFilterHeader::BloomFilterHeader(const BloomFilterHeader& other175) noexcept { - numBytes = other175.numBytes; - algorithm = other175.algorithm; - hash = other175.hash; - compression = other175.compression; +BloomFilterHeader::BloomFilterHeader(const BloomFilterHeader& other179) noexcept { + numBytes = other179.numBytes; + algorithm = other179.algorithm; + hash = other179.hash; + compression = other179.compression; } -BloomFilterHeader::BloomFilterHeader(BloomFilterHeader&& other176) noexcept { - numBytes = other176.numBytes; - algorithm = std::move(other176.algorithm); - hash = std::move(other176.hash); - compression = std::move(other176.compression); +BloomFilterHeader::BloomFilterHeader(BloomFilterHeader&& other180) noexcept { + numBytes = other180.numBytes; + algorithm = std::move(other180.algorithm); + hash = std::move(other180.hash); + compression = std::move(other180.compression); } -BloomFilterHeader& BloomFilterHeader::operator=(const BloomFilterHeader& other177) noexcept { - numBytes = other177.numBytes; - algorithm = other177.algorithm; - hash = other177.hash; - compression = other177.compression; +BloomFilterHeader& BloomFilterHeader::operator=(const BloomFilterHeader& other181) noexcept { + numBytes = other181.numBytes; + algorithm = other181.algorithm; + hash = other181.hash; + compression = other181.compression; return *this; } -BloomFilterHeader& BloomFilterHeader::operator=(BloomFilterHeader&& other178) noexcept { - numBytes = other178.numBytes; - algorithm = std::move(other178.algorithm); - hash = std::move(other178.hash); - compression = std::move(other178.compression); +BloomFilterHeader& BloomFilterHeader::operator=(BloomFilterHeader&& other182) noexcept { + numBytes = other182.numBytes; + algorithm = std::move(other182.algorithm); + hash = std::move(other182.hash); + compression = std::move(other182.compression); return *this; } void BloomFilterHeader::printTo(std::ostream& out) const { @@ -3651,50 +3728,50 @@ bool PageHeader::operator==(const PageHeader & rhs) const return true; } -PageHeader::PageHeader(const PageHeader& other180) { - type = other180.type; - uncompressed_page_size = other180.uncompressed_page_size; - compressed_page_size = other180.compressed_page_size; - crc = other180.crc; - data_page_header = other180.data_page_header; - index_page_header = other180.index_page_header; - dictionary_page_header = other180.dictionary_page_header; - data_page_header_v2 = other180.data_page_header_v2; - __isset = other180.__isset; -} -PageHeader::PageHeader(PageHeader&& other181) noexcept { - type = other181.type; - uncompressed_page_size = other181.uncompressed_page_size; - compressed_page_size = other181.compressed_page_size; - crc = other181.crc; - data_page_header = std::move(other181.data_page_header); - index_page_header = std::move(other181.index_page_header); - dictionary_page_header = std::move(other181.dictionary_page_header); - data_page_header_v2 = std::move(other181.data_page_header_v2); - __isset = other181.__isset; -} -PageHeader& PageHeader::operator=(const PageHeader& other182) { - type = other182.type; - uncompressed_page_size = other182.uncompressed_page_size; - compressed_page_size = other182.compressed_page_size; - crc = other182.crc; - data_page_header = other182.data_page_header; - index_page_header = other182.index_page_header; - dictionary_page_header = other182.dictionary_page_header; - data_page_header_v2 = other182.data_page_header_v2; - __isset = other182.__isset; +PageHeader::PageHeader(const PageHeader& other184) { + type = other184.type; + uncompressed_page_size = other184.uncompressed_page_size; + compressed_page_size = other184.compressed_page_size; + crc = other184.crc; + data_page_header = other184.data_page_header; + index_page_header = other184.index_page_header; + dictionary_page_header = other184.dictionary_page_header; + data_page_header_v2 = other184.data_page_header_v2; + __isset = other184.__isset; +} +PageHeader::PageHeader(PageHeader&& other185) noexcept { + type = other185.type; + uncompressed_page_size = other185.uncompressed_page_size; + compressed_page_size = other185.compressed_page_size; + crc = other185.crc; + data_page_header = std::move(other185.data_page_header); + index_page_header = std::move(other185.index_page_header); + dictionary_page_header = std::move(other185.dictionary_page_header); + data_page_header_v2 = std::move(other185.data_page_header_v2); + __isset = other185.__isset; +} +PageHeader& PageHeader::operator=(const PageHeader& other186) { + type = other186.type; + uncompressed_page_size = other186.uncompressed_page_size; + compressed_page_size = other186.compressed_page_size; + crc = other186.crc; + data_page_header = other186.data_page_header; + index_page_header = other186.index_page_header; + dictionary_page_header = other186.dictionary_page_header; + data_page_header_v2 = other186.data_page_header_v2; + __isset = other186.__isset; return *this; } -PageHeader& PageHeader::operator=(PageHeader&& other183) noexcept { - type = other183.type; - uncompressed_page_size = other183.uncompressed_page_size; - compressed_page_size = other183.compressed_page_size; - crc = other183.crc; - data_page_header = std::move(other183.data_page_header); - index_page_header = std::move(other183.index_page_header); - dictionary_page_header = std::move(other183.dictionary_page_header); - data_page_header_v2 = std::move(other183.data_page_header_v2); - __isset = other183.__isset; +PageHeader& PageHeader::operator=(PageHeader&& other187) noexcept { + type = other187.type; + uncompressed_page_size = other187.uncompressed_page_size; + compressed_page_size = other187.compressed_page_size; + crc = other187.crc; + data_page_header = std::move(other187.data_page_header); + index_page_header = std::move(other187.index_page_header); + dictionary_page_header = std::move(other187.dictionary_page_header); + data_page_header_v2 = std::move(other187.data_page_header_v2); + __isset = other187.__isset; return *this; } void PageHeader::printTo(std::ostream& out) const { @@ -3753,26 +3830,26 @@ bool KeyValue::operator==(const KeyValue & rhs) const return true; } -KeyValue::KeyValue(const KeyValue& other184) { - key = other184.key; - value = other184.value; - __isset = other184.__isset; +KeyValue::KeyValue(const KeyValue& other188) { + key = other188.key; + value = other188.value; + __isset = other188.__isset; } -KeyValue::KeyValue(KeyValue&& other185) noexcept { - key = std::move(other185.key); - value = std::move(other185.value); - __isset = other185.__isset; +KeyValue::KeyValue(KeyValue&& other189) noexcept { + key = std::move(other189.key); + value = std::move(other189.value); + __isset = other189.__isset; } -KeyValue& KeyValue::operator=(const KeyValue& other186) { - key = other186.key; - value = other186.value; - __isset = other186.__isset; +KeyValue& KeyValue::operator=(const KeyValue& other190) { + key = other190.key; + value = other190.value; + __isset = other190.__isset; return *this; } -KeyValue& KeyValue::operator=(KeyValue&& other187) noexcept { - key = std::move(other187.key); - value = std::move(other187.value); - __isset = other187.__isset; +KeyValue& KeyValue::operator=(KeyValue&& other191) noexcept { + key = std::move(other191.key); + value = std::move(other191.value); + __isset = other191.__isset; return *this; } void KeyValue::printTo(std::ostream& out) const { @@ -3829,26 +3906,26 @@ bool SortingColumn::operator==(const SortingColumn & rhs) const return true; } -SortingColumn::SortingColumn(const SortingColumn& other188) noexcept { - column_idx = other188.column_idx; - descending = other188.descending; - nulls_first = other188.nulls_first; +SortingColumn::SortingColumn(const SortingColumn& other192) noexcept { + column_idx = other192.column_idx; + descending = other192.descending; + nulls_first = other192.nulls_first; } -SortingColumn::SortingColumn(SortingColumn&& other189) noexcept { - column_idx = other189.column_idx; - descending = other189.descending; - nulls_first = other189.nulls_first; +SortingColumn::SortingColumn(SortingColumn&& other193) noexcept { + column_idx = other193.column_idx; + descending = other193.descending; + nulls_first = other193.nulls_first; } -SortingColumn& SortingColumn::operator=(const SortingColumn& other190) noexcept { - column_idx = other190.column_idx; - descending = other190.descending; - nulls_first = other190.nulls_first; +SortingColumn& SortingColumn::operator=(const SortingColumn& other194) noexcept { + column_idx = other194.column_idx; + descending = other194.descending; + nulls_first = other194.nulls_first; return *this; } -SortingColumn& SortingColumn::operator=(SortingColumn&& other191) noexcept { - column_idx = other191.column_idx; - descending = other191.descending; - nulls_first = other191.nulls_first; +SortingColumn& SortingColumn::operator=(SortingColumn&& other195) noexcept { + column_idx = other195.column_idx; + descending = other195.descending; + nulls_first = other195.nulls_first; return *this; } void SortingColumn::printTo(std::ostream& out) const { @@ -3906,26 +3983,26 @@ bool PageEncodingStats::operator==(const PageEncodingStats & rhs) const return true; } -PageEncodingStats::PageEncodingStats(const PageEncodingStats& other194) noexcept { - page_type = other194.page_type; - encoding = other194.encoding; - count = other194.count; +PageEncodingStats::PageEncodingStats(const PageEncodingStats& other198) noexcept { + page_type = other198.page_type; + encoding = other198.encoding; + count = other198.count; } -PageEncodingStats::PageEncodingStats(PageEncodingStats&& other195) noexcept { - page_type = other195.page_type; - encoding = other195.encoding; - count = other195.count; +PageEncodingStats::PageEncodingStats(PageEncodingStats&& other199) noexcept { + page_type = other199.page_type; + encoding = other199.encoding; + count = other199.count; } -PageEncodingStats& PageEncodingStats::operator=(const PageEncodingStats& other196) noexcept { - page_type = other196.page_type; - encoding = other196.encoding; - count = other196.count; +PageEncodingStats& PageEncodingStats::operator=(const PageEncodingStats& other200) noexcept { + page_type = other200.page_type; + encoding = other200.encoding; + count = other200.count; return *this; } -PageEncodingStats& PageEncodingStats::operator=(PageEncodingStats&& other197) noexcept { - page_type = other197.page_type; - encoding = other197.encoding; - count = other197.count; +PageEncodingStats& PageEncodingStats::operator=(PageEncodingStats&& other201) noexcept { + page_type = other201.page_type; + encoding = other201.encoding; + count = other201.count; return *this; } void PageEncodingStats::printTo(std::ostream& out) const { @@ -4116,86 +4193,86 @@ bool ColumnMetaData::operator==(const ColumnMetaData & rhs) const return true; } -ColumnMetaData::ColumnMetaData(const ColumnMetaData& other225) { - type = other225.type; - encodings = other225.encodings; - path_in_schema = other225.path_in_schema; - codec = other225.codec; - num_values = other225.num_values; - total_uncompressed_size = other225.total_uncompressed_size; - total_compressed_size = other225.total_compressed_size; - key_value_metadata = other225.key_value_metadata; - data_page_offset = other225.data_page_offset; - index_page_offset = other225.index_page_offset; - dictionary_page_offset = other225.dictionary_page_offset; - statistics = other225.statistics; - encoding_stats = other225.encoding_stats; - bloom_filter_offset = other225.bloom_filter_offset; - bloom_filter_length = other225.bloom_filter_length; - size_statistics = other225.size_statistics; - geospatial_statistics = other225.geospatial_statistics; - __isset = other225.__isset; -} -ColumnMetaData::ColumnMetaData(ColumnMetaData&& other226) noexcept { - type = other226.type; - encodings = std::move(other226.encodings); - path_in_schema = std::move(other226.path_in_schema); - codec = other226.codec; - num_values = other226.num_values; - total_uncompressed_size = other226.total_uncompressed_size; - total_compressed_size = other226.total_compressed_size; - key_value_metadata = std::move(other226.key_value_metadata); - data_page_offset = other226.data_page_offset; - index_page_offset = other226.index_page_offset; - dictionary_page_offset = other226.dictionary_page_offset; - statistics = std::move(other226.statistics); - encoding_stats = std::move(other226.encoding_stats); - bloom_filter_offset = other226.bloom_filter_offset; - bloom_filter_length = other226.bloom_filter_length; - size_statistics = std::move(other226.size_statistics); - geospatial_statistics = std::move(other226.geospatial_statistics); - __isset = other226.__isset; -} -ColumnMetaData& ColumnMetaData::operator=(const ColumnMetaData& other227) { - type = other227.type; - encodings = other227.encodings; - path_in_schema = other227.path_in_schema; - codec = other227.codec; - num_values = other227.num_values; - total_uncompressed_size = other227.total_uncompressed_size; - total_compressed_size = other227.total_compressed_size; - key_value_metadata = other227.key_value_metadata; - data_page_offset = other227.data_page_offset; - index_page_offset = other227.index_page_offset; - dictionary_page_offset = other227.dictionary_page_offset; - statistics = other227.statistics; - encoding_stats = other227.encoding_stats; - bloom_filter_offset = other227.bloom_filter_offset; - bloom_filter_length = other227.bloom_filter_length; - size_statistics = other227.size_statistics; - geospatial_statistics = other227.geospatial_statistics; - __isset = other227.__isset; +ColumnMetaData::ColumnMetaData(const ColumnMetaData& other229) { + type = other229.type; + encodings = other229.encodings; + path_in_schema = other229.path_in_schema; + codec = other229.codec; + num_values = other229.num_values; + total_uncompressed_size = other229.total_uncompressed_size; + total_compressed_size = other229.total_compressed_size; + key_value_metadata = other229.key_value_metadata; + data_page_offset = other229.data_page_offset; + index_page_offset = other229.index_page_offset; + dictionary_page_offset = other229.dictionary_page_offset; + statistics = other229.statistics; + encoding_stats = other229.encoding_stats; + bloom_filter_offset = other229.bloom_filter_offset; + bloom_filter_length = other229.bloom_filter_length; + size_statistics = other229.size_statistics; + geospatial_statistics = other229.geospatial_statistics; + __isset = other229.__isset; +} +ColumnMetaData::ColumnMetaData(ColumnMetaData&& other230) noexcept { + type = other230.type; + encodings = std::move(other230.encodings); + path_in_schema = std::move(other230.path_in_schema); + codec = other230.codec; + num_values = other230.num_values; + total_uncompressed_size = other230.total_uncompressed_size; + total_compressed_size = other230.total_compressed_size; + key_value_metadata = std::move(other230.key_value_metadata); + data_page_offset = other230.data_page_offset; + index_page_offset = other230.index_page_offset; + dictionary_page_offset = other230.dictionary_page_offset; + statistics = std::move(other230.statistics); + encoding_stats = std::move(other230.encoding_stats); + bloom_filter_offset = other230.bloom_filter_offset; + bloom_filter_length = other230.bloom_filter_length; + size_statistics = std::move(other230.size_statistics); + geospatial_statistics = std::move(other230.geospatial_statistics); + __isset = other230.__isset; +} +ColumnMetaData& ColumnMetaData::operator=(const ColumnMetaData& other231) { + type = other231.type; + encodings = other231.encodings; + path_in_schema = other231.path_in_schema; + codec = other231.codec; + num_values = other231.num_values; + total_uncompressed_size = other231.total_uncompressed_size; + total_compressed_size = other231.total_compressed_size; + key_value_metadata = other231.key_value_metadata; + data_page_offset = other231.data_page_offset; + index_page_offset = other231.index_page_offset; + dictionary_page_offset = other231.dictionary_page_offset; + statistics = other231.statistics; + encoding_stats = other231.encoding_stats; + bloom_filter_offset = other231.bloom_filter_offset; + bloom_filter_length = other231.bloom_filter_length; + size_statistics = other231.size_statistics; + geospatial_statistics = other231.geospatial_statistics; + __isset = other231.__isset; return *this; } -ColumnMetaData& ColumnMetaData::operator=(ColumnMetaData&& other228) noexcept { - type = other228.type; - encodings = std::move(other228.encodings); - path_in_schema = std::move(other228.path_in_schema); - codec = other228.codec; - num_values = other228.num_values; - total_uncompressed_size = other228.total_uncompressed_size; - total_compressed_size = other228.total_compressed_size; - key_value_metadata = std::move(other228.key_value_metadata); - data_page_offset = other228.data_page_offset; - index_page_offset = other228.index_page_offset; - dictionary_page_offset = other228.dictionary_page_offset; - statistics = std::move(other228.statistics); - encoding_stats = std::move(other228.encoding_stats); - bloom_filter_offset = other228.bloom_filter_offset; - bloom_filter_length = other228.bloom_filter_length; - size_statistics = std::move(other228.size_statistics); - geospatial_statistics = std::move(other228.geospatial_statistics); - __isset = other228.__isset; +ColumnMetaData& ColumnMetaData::operator=(ColumnMetaData&& other232) noexcept { + type = other232.type; + encodings = std::move(other232.encodings); + path_in_schema = std::move(other232.path_in_schema); + codec = other232.codec; + num_values = other232.num_values; + total_uncompressed_size = other232.total_uncompressed_size; + total_compressed_size = other232.total_compressed_size; + key_value_metadata = std::move(other232.key_value_metadata); + data_page_offset = other232.data_page_offset; + index_page_offset = other232.index_page_offset; + dictionary_page_offset = other232.dictionary_page_offset; + statistics = std::move(other232.statistics); + encoding_stats = std::move(other232.encoding_stats); + bloom_filter_offset = other232.bloom_filter_offset; + bloom_filter_length = other232.bloom_filter_length; + size_statistics = std::move(other232.size_statistics); + geospatial_statistics = std::move(other232.geospatial_statistics); + __isset = other232.__isset; return *this; } void ColumnMetaData::printTo(std::ostream& out) const { @@ -4245,18 +4322,18 @@ bool EncryptionWithFooterKey::operator==(const EncryptionWithFooterKey & /* rhs return true; } -EncryptionWithFooterKey::EncryptionWithFooterKey(const EncryptionWithFooterKey& other229) noexcept { - (void) other229; +EncryptionWithFooterKey::EncryptionWithFooterKey(const EncryptionWithFooterKey& other233) noexcept { + (void) other233; } -EncryptionWithFooterKey::EncryptionWithFooterKey(EncryptionWithFooterKey&& other230) noexcept { - (void) other230; +EncryptionWithFooterKey::EncryptionWithFooterKey(EncryptionWithFooterKey&& other234) noexcept { + (void) other234; } -EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(const EncryptionWithFooterKey& other231) noexcept { - (void) other231; +EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(const EncryptionWithFooterKey& other235) noexcept { + (void) other235; return *this; } -EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(EncryptionWithFooterKey&& other232) noexcept { - (void) other232; +EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(EncryptionWithFooterKey&& other236) noexcept { + (void) other236; return *this; } void EncryptionWithFooterKey::printTo(std::ostream& out) const { @@ -4306,26 +4383,26 @@ bool EncryptionWithColumnKey::operator==(const EncryptionWithColumnKey & rhs) co return true; } -EncryptionWithColumnKey::EncryptionWithColumnKey(const EncryptionWithColumnKey& other239) { - path_in_schema = other239.path_in_schema; - key_metadata = other239.key_metadata; - __isset = other239.__isset; +EncryptionWithColumnKey::EncryptionWithColumnKey(const EncryptionWithColumnKey& other243) { + path_in_schema = other243.path_in_schema; + key_metadata = other243.key_metadata; + __isset = other243.__isset; } -EncryptionWithColumnKey::EncryptionWithColumnKey(EncryptionWithColumnKey&& other240) noexcept { - path_in_schema = std::move(other240.path_in_schema); - key_metadata = std::move(other240.key_metadata); - __isset = other240.__isset; +EncryptionWithColumnKey::EncryptionWithColumnKey(EncryptionWithColumnKey&& other244) noexcept { + path_in_schema = std::move(other244.path_in_schema); + key_metadata = std::move(other244.key_metadata); + __isset = other244.__isset; } -EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(const EncryptionWithColumnKey& other241) { - path_in_schema = other241.path_in_schema; - key_metadata = other241.key_metadata; - __isset = other241.__isset; +EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(const EncryptionWithColumnKey& other245) { + path_in_schema = other245.path_in_schema; + key_metadata = other245.key_metadata; + __isset = other245.__isset; return *this; } -EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(EncryptionWithColumnKey&& other242) noexcept { - path_in_schema = std::move(other242.path_in_schema); - key_metadata = std::move(other242.key_metadata); - __isset = other242.__isset; +EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(EncryptionWithColumnKey&& other246) noexcept { + path_in_schema = std::move(other246.path_in_schema); + key_metadata = std::move(other246.key_metadata); + __isset = other246.__isset; return *this; } void EncryptionWithColumnKey::printTo(std::ostream& out) const { @@ -4379,26 +4456,26 @@ bool ColumnCryptoMetaData::operator==(const ColumnCryptoMetaData & rhs) const return true; } -ColumnCryptoMetaData::ColumnCryptoMetaData(const ColumnCryptoMetaData& other243) { - ENCRYPTION_WITH_FOOTER_KEY = other243.ENCRYPTION_WITH_FOOTER_KEY; - ENCRYPTION_WITH_COLUMN_KEY = other243.ENCRYPTION_WITH_COLUMN_KEY; - __isset = other243.__isset; +ColumnCryptoMetaData::ColumnCryptoMetaData(const ColumnCryptoMetaData& other247) { + ENCRYPTION_WITH_FOOTER_KEY = other247.ENCRYPTION_WITH_FOOTER_KEY; + ENCRYPTION_WITH_COLUMN_KEY = other247.ENCRYPTION_WITH_COLUMN_KEY; + __isset = other247.__isset; } -ColumnCryptoMetaData::ColumnCryptoMetaData(ColumnCryptoMetaData&& other244) noexcept { - ENCRYPTION_WITH_FOOTER_KEY = std::move(other244.ENCRYPTION_WITH_FOOTER_KEY); - ENCRYPTION_WITH_COLUMN_KEY = std::move(other244.ENCRYPTION_WITH_COLUMN_KEY); - __isset = other244.__isset; +ColumnCryptoMetaData::ColumnCryptoMetaData(ColumnCryptoMetaData&& other248) noexcept { + ENCRYPTION_WITH_FOOTER_KEY = std::move(other248.ENCRYPTION_WITH_FOOTER_KEY); + ENCRYPTION_WITH_COLUMN_KEY = std::move(other248.ENCRYPTION_WITH_COLUMN_KEY); + __isset = other248.__isset; } -ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(const ColumnCryptoMetaData& other245) { - ENCRYPTION_WITH_FOOTER_KEY = other245.ENCRYPTION_WITH_FOOTER_KEY; - ENCRYPTION_WITH_COLUMN_KEY = other245.ENCRYPTION_WITH_COLUMN_KEY; - __isset = other245.__isset; +ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(const ColumnCryptoMetaData& other249) { + ENCRYPTION_WITH_FOOTER_KEY = other249.ENCRYPTION_WITH_FOOTER_KEY; + ENCRYPTION_WITH_COLUMN_KEY = other249.ENCRYPTION_WITH_COLUMN_KEY; + __isset = other249.__isset; return *this; } -ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(ColumnCryptoMetaData&& other246) noexcept { - ENCRYPTION_WITH_FOOTER_KEY = std::move(other246.ENCRYPTION_WITH_FOOTER_KEY); - ENCRYPTION_WITH_COLUMN_KEY = std::move(other246.ENCRYPTION_WITH_COLUMN_KEY); - __isset = other246.__isset; +ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(ColumnCryptoMetaData&& other250) noexcept { + ENCRYPTION_WITH_FOOTER_KEY = std::move(other250.ENCRYPTION_WITH_FOOTER_KEY); + ENCRYPTION_WITH_COLUMN_KEY = std::move(other250.ENCRYPTION_WITH_COLUMN_KEY); + __isset = other250.__isset; return *this; } void ColumnCryptoMetaData::printTo(std::ostream& out) const { @@ -4526,54 +4603,54 @@ bool ColumnChunk::operator==(const ColumnChunk & rhs) const return true; } -ColumnChunk::ColumnChunk(const ColumnChunk& other247) { - file_path = other247.file_path; - file_offset = other247.file_offset; - meta_data = other247.meta_data; - offset_index_offset = other247.offset_index_offset; - offset_index_length = other247.offset_index_length; - column_index_offset = other247.column_index_offset; - column_index_length = other247.column_index_length; - crypto_metadata = other247.crypto_metadata; - encrypted_column_metadata = other247.encrypted_column_metadata; - __isset = other247.__isset; -} -ColumnChunk::ColumnChunk(ColumnChunk&& other248) noexcept { - file_path = std::move(other248.file_path); - file_offset = other248.file_offset; - meta_data = std::move(other248.meta_data); - offset_index_offset = other248.offset_index_offset; - offset_index_length = other248.offset_index_length; - column_index_offset = other248.column_index_offset; - column_index_length = other248.column_index_length; - crypto_metadata = std::move(other248.crypto_metadata); - encrypted_column_metadata = std::move(other248.encrypted_column_metadata); - __isset = other248.__isset; -} -ColumnChunk& ColumnChunk::operator=(const ColumnChunk& other249) { - file_path = other249.file_path; - file_offset = other249.file_offset; - meta_data = other249.meta_data; - offset_index_offset = other249.offset_index_offset; - offset_index_length = other249.offset_index_length; - column_index_offset = other249.column_index_offset; - column_index_length = other249.column_index_length; - crypto_metadata = other249.crypto_metadata; - encrypted_column_metadata = other249.encrypted_column_metadata; - __isset = other249.__isset; +ColumnChunk::ColumnChunk(const ColumnChunk& other251) { + file_path = other251.file_path; + file_offset = other251.file_offset; + meta_data = other251.meta_data; + offset_index_offset = other251.offset_index_offset; + offset_index_length = other251.offset_index_length; + column_index_offset = other251.column_index_offset; + column_index_length = other251.column_index_length; + crypto_metadata = other251.crypto_metadata; + encrypted_column_metadata = other251.encrypted_column_metadata; + __isset = other251.__isset; +} +ColumnChunk::ColumnChunk(ColumnChunk&& other252) noexcept { + file_path = std::move(other252.file_path); + file_offset = other252.file_offset; + meta_data = std::move(other252.meta_data); + offset_index_offset = other252.offset_index_offset; + offset_index_length = other252.offset_index_length; + column_index_offset = other252.column_index_offset; + column_index_length = other252.column_index_length; + crypto_metadata = std::move(other252.crypto_metadata); + encrypted_column_metadata = std::move(other252.encrypted_column_metadata); + __isset = other252.__isset; +} +ColumnChunk& ColumnChunk::operator=(const ColumnChunk& other253) { + file_path = other253.file_path; + file_offset = other253.file_offset; + meta_data = other253.meta_data; + offset_index_offset = other253.offset_index_offset; + offset_index_length = other253.offset_index_length; + column_index_offset = other253.column_index_offset; + column_index_length = other253.column_index_length; + crypto_metadata = other253.crypto_metadata; + encrypted_column_metadata = other253.encrypted_column_metadata; + __isset = other253.__isset; return *this; } -ColumnChunk& ColumnChunk::operator=(ColumnChunk&& other250) noexcept { - file_path = std::move(other250.file_path); - file_offset = other250.file_offset; - meta_data = std::move(other250.meta_data); - offset_index_offset = other250.offset_index_offset; - offset_index_length = other250.offset_index_length; - column_index_offset = other250.column_index_offset; - column_index_length = other250.column_index_length; - crypto_metadata = std::move(other250.crypto_metadata); - encrypted_column_metadata = std::move(other250.encrypted_column_metadata); - __isset = other250.__isset; +ColumnChunk& ColumnChunk::operator=(ColumnChunk&& other254) noexcept { + file_path = std::move(other254.file_path); + file_offset = other254.file_offset; + meta_data = std::move(other254.meta_data); + offset_index_offset = other254.offset_index_offset; + offset_index_length = other254.offset_index_length; + column_index_offset = other254.column_index_offset; + column_index_length = other254.column_index_length; + crypto_metadata = std::move(other254.crypto_metadata); + encrypted_column_metadata = std::move(other254.encrypted_column_metadata); + __isset = other254.__isset; return *this; } void ColumnChunk::printTo(std::ostream& out) const { @@ -4680,46 +4757,46 @@ bool RowGroup::operator==(const RowGroup & rhs) const return true; } -RowGroup::RowGroup(const RowGroup& other263) { - columns = other263.columns; - total_byte_size = other263.total_byte_size; - num_rows = other263.num_rows; - sorting_columns = other263.sorting_columns; - file_offset = other263.file_offset; - total_compressed_size = other263.total_compressed_size; - ordinal = other263.ordinal; - __isset = other263.__isset; -} -RowGroup::RowGroup(RowGroup&& other264) noexcept { - columns = std::move(other264.columns); - total_byte_size = other264.total_byte_size; - num_rows = other264.num_rows; - sorting_columns = std::move(other264.sorting_columns); - file_offset = other264.file_offset; - total_compressed_size = other264.total_compressed_size; - ordinal = other264.ordinal; - __isset = other264.__isset; -} -RowGroup& RowGroup::operator=(const RowGroup& other265) { - columns = other265.columns; - total_byte_size = other265.total_byte_size; - num_rows = other265.num_rows; - sorting_columns = other265.sorting_columns; - file_offset = other265.file_offset; - total_compressed_size = other265.total_compressed_size; - ordinal = other265.ordinal; - __isset = other265.__isset; +RowGroup::RowGroup(const RowGroup& other267) { + columns = other267.columns; + total_byte_size = other267.total_byte_size; + num_rows = other267.num_rows; + sorting_columns = other267.sorting_columns; + file_offset = other267.file_offset; + total_compressed_size = other267.total_compressed_size; + ordinal = other267.ordinal; + __isset = other267.__isset; +} +RowGroup::RowGroup(RowGroup&& other268) noexcept { + columns = std::move(other268.columns); + total_byte_size = other268.total_byte_size; + num_rows = other268.num_rows; + sorting_columns = std::move(other268.sorting_columns); + file_offset = other268.file_offset; + total_compressed_size = other268.total_compressed_size; + ordinal = other268.ordinal; + __isset = other268.__isset; +} +RowGroup& RowGroup::operator=(const RowGroup& other269) { + columns = other269.columns; + total_byte_size = other269.total_byte_size; + num_rows = other269.num_rows; + sorting_columns = other269.sorting_columns; + file_offset = other269.file_offset; + total_compressed_size = other269.total_compressed_size; + ordinal = other269.ordinal; + __isset = other269.__isset; return *this; } -RowGroup& RowGroup::operator=(RowGroup&& other266) noexcept { - columns = std::move(other266.columns); - total_byte_size = other266.total_byte_size; - num_rows = other266.num_rows; - sorting_columns = std::move(other266.sorting_columns); - file_offset = other266.file_offset; - total_compressed_size = other266.total_compressed_size; - ordinal = other266.ordinal; - __isset = other266.__isset; +RowGroup& RowGroup::operator=(RowGroup&& other270) noexcept { + columns = std::move(other270.columns); + total_byte_size = other270.total_byte_size; + num_rows = other270.num_rows; + sorting_columns = std::move(other270.sorting_columns); + file_offset = other270.file_offset; + total_compressed_size = other270.total_compressed_size; + ordinal = other270.ordinal; + __isset = other270.__isset; return *this; } void RowGroup::printTo(std::ostream& out) const { @@ -4759,18 +4836,18 @@ bool TypeDefinedOrder::operator==(const TypeDefinedOrder & /* rhs */) const return true; } -TypeDefinedOrder::TypeDefinedOrder(const TypeDefinedOrder& other267) noexcept { - (void) other267; +TypeDefinedOrder::TypeDefinedOrder(const TypeDefinedOrder& other271) noexcept { + (void) other271; } -TypeDefinedOrder::TypeDefinedOrder(TypeDefinedOrder&& other268) noexcept { - (void) other268; +TypeDefinedOrder::TypeDefinedOrder(TypeDefinedOrder&& other272) noexcept { + (void) other272; } -TypeDefinedOrder& TypeDefinedOrder::operator=(const TypeDefinedOrder& other269) noexcept { - (void) other269; +TypeDefinedOrder& TypeDefinedOrder::operator=(const TypeDefinedOrder& other273) noexcept { + (void) other273; return *this; } -TypeDefinedOrder& TypeDefinedOrder::operator=(TypeDefinedOrder&& other270) noexcept { - (void) other270; +TypeDefinedOrder& TypeDefinedOrder::operator=(TypeDefinedOrder&& other274) noexcept { + (void) other274; return *this; } void TypeDefinedOrder::printTo(std::ostream& out) const { @@ -4803,18 +4880,18 @@ bool IEEE754TotalOrder::operator==(const IEEE754TotalOrder & /* rhs */) const return true; } -IEEE754TotalOrder::IEEE754TotalOrder(const IEEE754TotalOrder& other271) noexcept { - (void) other271; +IEEE754TotalOrder::IEEE754TotalOrder(const IEEE754TotalOrder& other275) noexcept { + (void) other275; } -IEEE754TotalOrder::IEEE754TotalOrder(IEEE754TotalOrder&& other272) noexcept { - (void) other272; +IEEE754TotalOrder::IEEE754TotalOrder(IEEE754TotalOrder&& other276) noexcept { + (void) other276; } -IEEE754TotalOrder& IEEE754TotalOrder::operator=(const IEEE754TotalOrder& other273) noexcept { - (void) other273; +IEEE754TotalOrder& IEEE754TotalOrder::operator=(const IEEE754TotalOrder& other277) noexcept { + (void) other277; return *this; } -IEEE754TotalOrder& IEEE754TotalOrder::operator=(IEEE754TotalOrder&& other274) noexcept { - (void) other274; +IEEE754TotalOrder& IEEE754TotalOrder::operator=(IEEE754TotalOrder&& other278) noexcept { + (void) other278; return *this; } void IEEE754TotalOrder::printTo(std::ostream& out) const { @@ -4866,26 +4943,26 @@ bool ColumnOrder::operator==(const ColumnOrder & rhs) const return true; } -ColumnOrder::ColumnOrder(const ColumnOrder& other275) noexcept { - TYPE_ORDER = other275.TYPE_ORDER; - IEEE_754_TOTAL_ORDER = other275.IEEE_754_TOTAL_ORDER; - __isset = other275.__isset; +ColumnOrder::ColumnOrder(const ColumnOrder& other279) noexcept { + TYPE_ORDER = other279.TYPE_ORDER; + IEEE_754_TOTAL_ORDER = other279.IEEE_754_TOTAL_ORDER; + __isset = other279.__isset; } -ColumnOrder::ColumnOrder(ColumnOrder&& other276) noexcept { - TYPE_ORDER = std::move(other276.TYPE_ORDER); - IEEE_754_TOTAL_ORDER = std::move(other276.IEEE_754_TOTAL_ORDER); - __isset = other276.__isset; +ColumnOrder::ColumnOrder(ColumnOrder&& other280) noexcept { + TYPE_ORDER = std::move(other280.TYPE_ORDER); + IEEE_754_TOTAL_ORDER = std::move(other280.IEEE_754_TOTAL_ORDER); + __isset = other280.__isset; } -ColumnOrder& ColumnOrder::operator=(const ColumnOrder& other277) noexcept { - TYPE_ORDER = other277.TYPE_ORDER; - IEEE_754_TOTAL_ORDER = other277.IEEE_754_TOTAL_ORDER; - __isset = other277.__isset; +ColumnOrder& ColumnOrder::operator=(const ColumnOrder& other281) noexcept { + TYPE_ORDER = other281.TYPE_ORDER; + IEEE_754_TOTAL_ORDER = other281.IEEE_754_TOTAL_ORDER; + __isset = other281.__isset; return *this; } -ColumnOrder& ColumnOrder::operator=(ColumnOrder&& other278) noexcept { - TYPE_ORDER = std::move(other278.TYPE_ORDER); - IEEE_754_TOTAL_ORDER = std::move(other278.IEEE_754_TOTAL_ORDER); - __isset = other278.__isset; +ColumnOrder& ColumnOrder::operator=(ColumnOrder&& other282) noexcept { + TYPE_ORDER = std::move(other282.TYPE_ORDER); + IEEE_754_TOTAL_ORDER = std::move(other282.IEEE_754_TOTAL_ORDER); + __isset = other282.__isset; return *this; } void ColumnOrder::printTo(std::ostream& out) const { @@ -4942,26 +5019,26 @@ bool PageLocation::operator==(const PageLocation & rhs) const return true; } -PageLocation::PageLocation(const PageLocation& other279) noexcept { - offset = other279.offset; - compressed_page_size = other279.compressed_page_size; - first_row_index = other279.first_row_index; +PageLocation::PageLocation(const PageLocation& other283) noexcept { + offset = other283.offset; + compressed_page_size = other283.compressed_page_size; + first_row_index = other283.first_row_index; } -PageLocation::PageLocation(PageLocation&& other280) noexcept { - offset = other280.offset; - compressed_page_size = other280.compressed_page_size; - first_row_index = other280.first_row_index; +PageLocation::PageLocation(PageLocation&& other284) noexcept { + offset = other284.offset; + compressed_page_size = other284.compressed_page_size; + first_row_index = other284.first_row_index; } -PageLocation& PageLocation::operator=(const PageLocation& other281) noexcept { - offset = other281.offset; - compressed_page_size = other281.compressed_page_size; - first_row_index = other281.first_row_index; +PageLocation& PageLocation::operator=(const PageLocation& other285) noexcept { + offset = other285.offset; + compressed_page_size = other285.compressed_page_size; + first_row_index = other285.first_row_index; return *this; } -PageLocation& PageLocation::operator=(PageLocation&& other282) noexcept { - offset = other282.offset; - compressed_page_size = other282.compressed_page_size; - first_row_index = other282.first_row_index; +PageLocation& PageLocation::operator=(PageLocation&& other286) noexcept { + offset = other286.offset; + compressed_page_size = other286.compressed_page_size; + first_row_index = other286.first_row_index; return *this; } void PageLocation::printTo(std::ostream& out) const { @@ -5013,26 +5090,26 @@ bool OffsetIndex::operator==(const OffsetIndex & rhs) const return true; } -OffsetIndex::OffsetIndex(const OffsetIndex& other295) { - page_locations = other295.page_locations; - unencoded_byte_array_data_bytes = other295.unencoded_byte_array_data_bytes; - __isset = other295.__isset; +OffsetIndex::OffsetIndex(const OffsetIndex& other299) { + page_locations = other299.page_locations; + unencoded_byte_array_data_bytes = other299.unencoded_byte_array_data_bytes; + __isset = other299.__isset; } -OffsetIndex::OffsetIndex(OffsetIndex&& other296) noexcept { - page_locations = std::move(other296.page_locations); - unencoded_byte_array_data_bytes = std::move(other296.unencoded_byte_array_data_bytes); - __isset = other296.__isset; +OffsetIndex::OffsetIndex(OffsetIndex&& other300) noexcept { + page_locations = std::move(other300.page_locations); + unencoded_byte_array_data_bytes = std::move(other300.unencoded_byte_array_data_bytes); + __isset = other300.__isset; } -OffsetIndex& OffsetIndex::operator=(const OffsetIndex& other297) { - page_locations = other297.page_locations; - unencoded_byte_array_data_bytes = other297.unencoded_byte_array_data_bytes; - __isset = other297.__isset; +OffsetIndex& OffsetIndex::operator=(const OffsetIndex& other301) { + page_locations = other301.page_locations; + unencoded_byte_array_data_bytes = other301.unencoded_byte_array_data_bytes; + __isset = other301.__isset; return *this; } -OffsetIndex& OffsetIndex::operator=(OffsetIndex&& other298) noexcept { - page_locations = std::move(other298.page_locations); - unencoded_byte_array_data_bytes = std::move(other298.unencoded_byte_array_data_bytes); - __isset = other298.__isset; +OffsetIndex& OffsetIndex::operator=(OffsetIndex&& other302) noexcept { + page_locations = std::move(other302.page_locations); + unencoded_byte_array_data_bytes = std::move(other302.unencoded_byte_array_data_bytes); + __isset = other302.__isset; return *this; } void OffsetIndex::printTo(std::ostream& out) const { @@ -5135,50 +5212,50 @@ bool ColumnIndex::operator==(const ColumnIndex & rhs) const return true; } -ColumnIndex::ColumnIndex(const ColumnIndex& other342) { - null_pages = other342.null_pages; - min_values = other342.min_values; - max_values = other342.max_values; - boundary_order = other342.boundary_order; - null_counts = other342.null_counts; - repetition_level_histograms = other342.repetition_level_histograms; - definition_level_histograms = other342.definition_level_histograms; - nan_counts = other342.nan_counts; - __isset = other342.__isset; -} -ColumnIndex::ColumnIndex(ColumnIndex&& other343) noexcept { - null_pages = std::move(other343.null_pages); - min_values = std::move(other343.min_values); - max_values = std::move(other343.max_values); - boundary_order = other343.boundary_order; - null_counts = std::move(other343.null_counts); - repetition_level_histograms = std::move(other343.repetition_level_histograms); - definition_level_histograms = std::move(other343.definition_level_histograms); - nan_counts = std::move(other343.nan_counts); - __isset = other343.__isset; -} -ColumnIndex& ColumnIndex::operator=(const ColumnIndex& other344) { - null_pages = other344.null_pages; - min_values = other344.min_values; - max_values = other344.max_values; - boundary_order = other344.boundary_order; - null_counts = other344.null_counts; - repetition_level_histograms = other344.repetition_level_histograms; - definition_level_histograms = other344.definition_level_histograms; - nan_counts = other344.nan_counts; - __isset = other344.__isset; +ColumnIndex::ColumnIndex(const ColumnIndex& other346) { + null_pages = other346.null_pages; + min_values = other346.min_values; + max_values = other346.max_values; + boundary_order = other346.boundary_order; + null_counts = other346.null_counts; + repetition_level_histograms = other346.repetition_level_histograms; + definition_level_histograms = other346.definition_level_histograms; + nan_counts = other346.nan_counts; + __isset = other346.__isset; +} +ColumnIndex::ColumnIndex(ColumnIndex&& other347) noexcept { + null_pages = std::move(other347.null_pages); + min_values = std::move(other347.min_values); + max_values = std::move(other347.max_values); + boundary_order = other347.boundary_order; + null_counts = std::move(other347.null_counts); + repetition_level_histograms = std::move(other347.repetition_level_histograms); + definition_level_histograms = std::move(other347.definition_level_histograms); + nan_counts = std::move(other347.nan_counts); + __isset = other347.__isset; +} +ColumnIndex& ColumnIndex::operator=(const ColumnIndex& other348) { + null_pages = other348.null_pages; + min_values = other348.min_values; + max_values = other348.max_values; + boundary_order = other348.boundary_order; + null_counts = other348.null_counts; + repetition_level_histograms = other348.repetition_level_histograms; + definition_level_histograms = other348.definition_level_histograms; + nan_counts = other348.nan_counts; + __isset = other348.__isset; return *this; } -ColumnIndex& ColumnIndex::operator=(ColumnIndex&& other345) noexcept { - null_pages = std::move(other345.null_pages); - min_values = std::move(other345.min_values); - max_values = std::move(other345.max_values); - boundary_order = other345.boundary_order; - null_counts = std::move(other345.null_counts); - repetition_level_histograms = std::move(other345.repetition_level_histograms); - definition_level_histograms = std::move(other345.definition_level_histograms); - nan_counts = std::move(other345.nan_counts); - __isset = other345.__isset; +ColumnIndex& ColumnIndex::operator=(ColumnIndex&& other349) noexcept { + null_pages = std::move(other349.null_pages); + min_values = std::move(other349.min_values); + max_values = std::move(other349.max_values); + boundary_order = other349.boundary_order; + null_counts = std::move(other349.null_counts); + repetition_level_histograms = std::move(other349.repetition_level_histograms); + definition_level_histograms = std::move(other349.definition_level_histograms); + nan_counts = std::move(other349.nan_counts); + __isset = other349.__isset; return *this; } void ColumnIndex::printTo(std::ostream& out) const { @@ -5251,30 +5328,30 @@ bool AesGcmV1::operator==(const AesGcmV1 & rhs) const return true; } -AesGcmV1::AesGcmV1(const AesGcmV1& other346) { - aad_prefix = other346.aad_prefix; - aad_file_unique = other346.aad_file_unique; - supply_aad_prefix = other346.supply_aad_prefix; - __isset = other346.__isset; +AesGcmV1::AesGcmV1(const AesGcmV1& other350) { + aad_prefix = other350.aad_prefix; + aad_file_unique = other350.aad_file_unique; + supply_aad_prefix = other350.supply_aad_prefix; + __isset = other350.__isset; } -AesGcmV1::AesGcmV1(AesGcmV1&& other347) noexcept { - aad_prefix = std::move(other347.aad_prefix); - aad_file_unique = std::move(other347.aad_file_unique); - supply_aad_prefix = other347.supply_aad_prefix; - __isset = other347.__isset; +AesGcmV1::AesGcmV1(AesGcmV1&& other351) noexcept { + aad_prefix = std::move(other351.aad_prefix); + aad_file_unique = std::move(other351.aad_file_unique); + supply_aad_prefix = other351.supply_aad_prefix; + __isset = other351.__isset; } -AesGcmV1& AesGcmV1::operator=(const AesGcmV1& other348) { - aad_prefix = other348.aad_prefix; - aad_file_unique = other348.aad_file_unique; - supply_aad_prefix = other348.supply_aad_prefix; - __isset = other348.__isset; +AesGcmV1& AesGcmV1::operator=(const AesGcmV1& other352) { + aad_prefix = other352.aad_prefix; + aad_file_unique = other352.aad_file_unique; + supply_aad_prefix = other352.supply_aad_prefix; + __isset = other352.__isset; return *this; } -AesGcmV1& AesGcmV1::operator=(AesGcmV1&& other349) noexcept { - aad_prefix = std::move(other349.aad_prefix); - aad_file_unique = std::move(other349.aad_file_unique); - supply_aad_prefix = other349.supply_aad_prefix; - __isset = other349.__isset; +AesGcmV1& AesGcmV1::operator=(AesGcmV1&& other353) noexcept { + aad_prefix = std::move(other353.aad_prefix); + aad_file_unique = std::move(other353.aad_file_unique); + supply_aad_prefix = other353.supply_aad_prefix; + __isset = other353.__isset; return *this; } void AesGcmV1::printTo(std::ostream& out) const { @@ -5342,30 +5419,30 @@ bool AesGcmCtrV1::operator==(const AesGcmCtrV1 & rhs) const return true; } -AesGcmCtrV1::AesGcmCtrV1(const AesGcmCtrV1& other350) { - aad_prefix = other350.aad_prefix; - aad_file_unique = other350.aad_file_unique; - supply_aad_prefix = other350.supply_aad_prefix; - __isset = other350.__isset; +AesGcmCtrV1::AesGcmCtrV1(const AesGcmCtrV1& other354) { + aad_prefix = other354.aad_prefix; + aad_file_unique = other354.aad_file_unique; + supply_aad_prefix = other354.supply_aad_prefix; + __isset = other354.__isset; } -AesGcmCtrV1::AesGcmCtrV1(AesGcmCtrV1&& other351) noexcept { - aad_prefix = std::move(other351.aad_prefix); - aad_file_unique = std::move(other351.aad_file_unique); - supply_aad_prefix = other351.supply_aad_prefix; - __isset = other351.__isset; +AesGcmCtrV1::AesGcmCtrV1(AesGcmCtrV1&& other355) noexcept { + aad_prefix = std::move(other355.aad_prefix); + aad_file_unique = std::move(other355.aad_file_unique); + supply_aad_prefix = other355.supply_aad_prefix; + __isset = other355.__isset; } -AesGcmCtrV1& AesGcmCtrV1::operator=(const AesGcmCtrV1& other352) { - aad_prefix = other352.aad_prefix; - aad_file_unique = other352.aad_file_unique; - supply_aad_prefix = other352.supply_aad_prefix; - __isset = other352.__isset; +AesGcmCtrV1& AesGcmCtrV1::operator=(const AesGcmCtrV1& other356) { + aad_prefix = other356.aad_prefix; + aad_file_unique = other356.aad_file_unique; + supply_aad_prefix = other356.supply_aad_prefix; + __isset = other356.__isset; return *this; } -AesGcmCtrV1& AesGcmCtrV1::operator=(AesGcmCtrV1&& other353) noexcept { - aad_prefix = std::move(other353.aad_prefix); - aad_file_unique = std::move(other353.aad_file_unique); - supply_aad_prefix = other353.supply_aad_prefix; - __isset = other353.__isset; +AesGcmCtrV1& AesGcmCtrV1::operator=(AesGcmCtrV1&& other357) noexcept { + aad_prefix = std::move(other357.aad_prefix); + aad_file_unique = std::move(other357.aad_file_unique); + supply_aad_prefix = other357.supply_aad_prefix; + __isset = other357.__isset; return *this; } void AesGcmCtrV1::printTo(std::ostream& out) const { @@ -5420,26 +5497,26 @@ bool EncryptionAlgorithm::operator==(const EncryptionAlgorithm & rhs) const return true; } -EncryptionAlgorithm::EncryptionAlgorithm(const EncryptionAlgorithm& other354) { - AES_GCM_V1 = other354.AES_GCM_V1; - AES_GCM_CTR_V1 = other354.AES_GCM_CTR_V1; - __isset = other354.__isset; +EncryptionAlgorithm::EncryptionAlgorithm(const EncryptionAlgorithm& other358) { + AES_GCM_V1 = other358.AES_GCM_V1; + AES_GCM_CTR_V1 = other358.AES_GCM_CTR_V1; + __isset = other358.__isset; } -EncryptionAlgorithm::EncryptionAlgorithm(EncryptionAlgorithm&& other355) noexcept { - AES_GCM_V1 = std::move(other355.AES_GCM_V1); - AES_GCM_CTR_V1 = std::move(other355.AES_GCM_CTR_V1); - __isset = other355.__isset; +EncryptionAlgorithm::EncryptionAlgorithm(EncryptionAlgorithm&& other359) noexcept { + AES_GCM_V1 = std::move(other359.AES_GCM_V1); + AES_GCM_CTR_V1 = std::move(other359.AES_GCM_CTR_V1); + __isset = other359.__isset; } -EncryptionAlgorithm& EncryptionAlgorithm::operator=(const EncryptionAlgorithm& other356) { - AES_GCM_V1 = other356.AES_GCM_V1; - AES_GCM_CTR_V1 = other356.AES_GCM_CTR_V1; - __isset = other356.__isset; +EncryptionAlgorithm& EncryptionAlgorithm::operator=(const EncryptionAlgorithm& other360) { + AES_GCM_V1 = other360.AES_GCM_V1; + AES_GCM_CTR_V1 = other360.AES_GCM_CTR_V1; + __isset = other360.__isset; return *this; } -EncryptionAlgorithm& EncryptionAlgorithm::operator=(EncryptionAlgorithm&& other357) noexcept { - AES_GCM_V1 = std::move(other357.AES_GCM_V1); - AES_GCM_CTR_V1 = std::move(other357.AES_GCM_CTR_V1); - __isset = other357.__isset; +EncryptionAlgorithm& EncryptionAlgorithm::operator=(EncryptionAlgorithm&& other361) noexcept { + AES_GCM_V1 = std::move(other361.AES_GCM_V1); + AES_GCM_CTR_V1 = std::move(other361.AES_GCM_CTR_V1); + __isset = other361.__isset; return *this; } void EncryptionAlgorithm::printTo(std::ostream& out) const { @@ -5555,54 +5632,54 @@ bool FileMetaData::operator==(const FileMetaData & rhs) const return true; } -FileMetaData::FileMetaData(const FileMetaData& other382) { - version = other382.version; - schema = other382.schema; - num_rows = other382.num_rows; - row_groups = other382.row_groups; - key_value_metadata = other382.key_value_metadata; - created_by = other382.created_by; - column_orders = other382.column_orders; - encryption_algorithm = other382.encryption_algorithm; - footer_signing_key_metadata = other382.footer_signing_key_metadata; - __isset = other382.__isset; -} -FileMetaData::FileMetaData(FileMetaData&& other383) noexcept { - version = other383.version; - schema = std::move(other383.schema); - num_rows = other383.num_rows; - row_groups = std::move(other383.row_groups); - key_value_metadata = std::move(other383.key_value_metadata); - created_by = std::move(other383.created_by); - column_orders = std::move(other383.column_orders); - encryption_algorithm = std::move(other383.encryption_algorithm); - footer_signing_key_metadata = std::move(other383.footer_signing_key_metadata); - __isset = other383.__isset; -} -FileMetaData& FileMetaData::operator=(const FileMetaData& other384) { - version = other384.version; - schema = other384.schema; - num_rows = other384.num_rows; - row_groups = other384.row_groups; - key_value_metadata = other384.key_value_metadata; - created_by = other384.created_by; - column_orders = other384.column_orders; - encryption_algorithm = other384.encryption_algorithm; - footer_signing_key_metadata = other384.footer_signing_key_metadata; - __isset = other384.__isset; +FileMetaData::FileMetaData(const FileMetaData& other386) { + version = other386.version; + schema = other386.schema; + num_rows = other386.num_rows; + row_groups = other386.row_groups; + key_value_metadata = other386.key_value_metadata; + created_by = other386.created_by; + column_orders = other386.column_orders; + encryption_algorithm = other386.encryption_algorithm; + footer_signing_key_metadata = other386.footer_signing_key_metadata; + __isset = other386.__isset; +} +FileMetaData::FileMetaData(FileMetaData&& other387) noexcept { + version = other387.version; + schema = std::move(other387.schema); + num_rows = other387.num_rows; + row_groups = std::move(other387.row_groups); + key_value_metadata = std::move(other387.key_value_metadata); + created_by = std::move(other387.created_by); + column_orders = std::move(other387.column_orders); + encryption_algorithm = std::move(other387.encryption_algorithm); + footer_signing_key_metadata = std::move(other387.footer_signing_key_metadata); + __isset = other387.__isset; +} +FileMetaData& FileMetaData::operator=(const FileMetaData& other388) { + version = other388.version; + schema = other388.schema; + num_rows = other388.num_rows; + row_groups = other388.row_groups; + key_value_metadata = other388.key_value_metadata; + created_by = other388.created_by; + column_orders = other388.column_orders; + encryption_algorithm = other388.encryption_algorithm; + footer_signing_key_metadata = other388.footer_signing_key_metadata; + __isset = other388.__isset; return *this; } -FileMetaData& FileMetaData::operator=(FileMetaData&& other385) noexcept { - version = other385.version; - schema = std::move(other385.schema); - num_rows = other385.num_rows; - row_groups = std::move(other385.row_groups); - key_value_metadata = std::move(other385.key_value_metadata); - created_by = std::move(other385.created_by); - column_orders = std::move(other385.column_orders); - encryption_algorithm = std::move(other385.encryption_algorithm); - footer_signing_key_metadata = std::move(other385.footer_signing_key_metadata); - __isset = other385.__isset; +FileMetaData& FileMetaData::operator=(FileMetaData&& other389) noexcept { + version = other389.version; + schema = std::move(other389.schema); + num_rows = other389.num_rows; + row_groups = std::move(other389.row_groups); + key_value_metadata = std::move(other389.key_value_metadata); + created_by = std::move(other389.created_by); + column_orders = std::move(other389.column_orders); + encryption_algorithm = std::move(other389.encryption_algorithm); + footer_signing_key_metadata = std::move(other389.footer_signing_key_metadata); + __isset = other389.__isset; return *this; } void FileMetaData::printTo(std::ostream& out) const { @@ -5661,26 +5738,26 @@ bool FileCryptoMetaData::operator==(const FileCryptoMetaData & rhs) const return true; } -FileCryptoMetaData::FileCryptoMetaData(const FileCryptoMetaData& other386) { - encryption_algorithm = other386.encryption_algorithm; - key_metadata = other386.key_metadata; - __isset = other386.__isset; +FileCryptoMetaData::FileCryptoMetaData(const FileCryptoMetaData& other390) { + encryption_algorithm = other390.encryption_algorithm; + key_metadata = other390.key_metadata; + __isset = other390.__isset; } -FileCryptoMetaData::FileCryptoMetaData(FileCryptoMetaData&& other387) noexcept { - encryption_algorithm = std::move(other387.encryption_algorithm); - key_metadata = std::move(other387.key_metadata); - __isset = other387.__isset; +FileCryptoMetaData::FileCryptoMetaData(FileCryptoMetaData&& other391) noexcept { + encryption_algorithm = std::move(other391.encryption_algorithm); + key_metadata = std::move(other391.key_metadata); + __isset = other391.__isset; } -FileCryptoMetaData& FileCryptoMetaData::operator=(const FileCryptoMetaData& other388) { - encryption_algorithm = other388.encryption_algorithm; - key_metadata = other388.key_metadata; - __isset = other388.__isset; +FileCryptoMetaData& FileCryptoMetaData::operator=(const FileCryptoMetaData& other392) { + encryption_algorithm = other392.encryption_algorithm; + key_metadata = other392.key_metadata; + __isset = other392.__isset; return *this; } -FileCryptoMetaData& FileCryptoMetaData::operator=(FileCryptoMetaData&& other389) noexcept { - encryption_algorithm = std::move(other389.encryption_algorithm); - key_metadata = std::move(other389.key_metadata); - __isset = other389.__isset; +FileCryptoMetaData& FileCryptoMetaData::operator=(FileCryptoMetaData&& other393) noexcept { + encryption_algorithm = std::move(other393.encryption_algorithm); + key_metadata = std::move(other393.key_metadata); + __isset = other393.__isset; return *this; } void FileCryptoMetaData::printTo(std::ostream& out) const { diff --git a/cpp/src/generated/parquet_types.h b/cpp/src/generated/parquet_types.h index 7dc3ccc2de2c..7072da89c2cb 100644 --- a/cpp/src/generated/parquet_types.h +++ b/cpp/src/generated/parquet_types.h @@ -202,7 +202,8 @@ struct FieldRepetitionType { /** * The field is repeated and can contain 0 or more values */ - REPEATED = 2 + REPEATED = 2, + VECTOR = 3 }; }; @@ -418,6 +419,8 @@ class GeometryType; class GeographyType; +class VectorType; + class LogicalType; class SchemaElement; @@ -1633,8 +1636,39 @@ void swap(GeographyType &a, GeographyType &b) noexcept; std::ostream& operator<<(std::ostream& out, const GeographyType& obj); + +class VectorType { + public: + + VectorType(const VectorType&) noexcept; + VectorType(VectorType&&) noexcept; + VectorType& operator=(const VectorType&) noexcept; + VectorType& operator=(VectorType&&) noexcept; + VectorType() noexcept; + + ~VectorType() noexcept; + + bool operator == (const VectorType & /* rhs */) const; + bool operator != (const VectorType &rhs) const { + return !(*this == rhs); + } + + bool operator < (const VectorType & ) const; + + template + uint32_t read(Protocol_* iprot); + template + uint32_t write(Protocol_* oprot) const; + + void printTo(std::ostream& out) const; +}; + +void swap(VectorType &a, VectorType &b) noexcept; + +std::ostream& operator<<(std::ostream& out, const VectorType& obj); + typedef struct _LogicalType__isset { - _LogicalType__isset() : STRING(false), MAP(false), LIST(false), ENUM(false), DECIMAL(false), DATE(false), TIME(false), TIMESTAMP(false), INTEGER(false), UNKNOWN(false), JSON(false), BSON(false), UUID(false), FLOAT16(false), VARIANT(false), GEOMETRY(false), GEOGRAPHY(false) {} + _LogicalType__isset() : STRING(false), MAP(false), LIST(false), ENUM(false), DECIMAL(false), DATE(false), TIME(false), TIMESTAMP(false), INTEGER(false), UNKNOWN(false), JSON(false), BSON(false), UUID(false), FLOAT16(false), VARIANT(false), GEOMETRY(false), GEOGRAPHY(false), VECTOR(false) {} bool STRING :1; bool MAP :1; bool LIST :1; @@ -1652,6 +1686,7 @@ typedef struct _LogicalType__isset { bool VARIANT :1; bool GEOMETRY :1; bool GEOGRAPHY :1; + bool VECTOR :1; } _LogicalType__isset; /** @@ -1688,6 +1723,7 @@ class LogicalType { VariantType VARIANT; GeometryType GEOMETRY; GeographyType GEOGRAPHY; + VectorType VECTOR; _LogicalType__isset __isset; @@ -1725,6 +1761,8 @@ class LogicalType { void __set_GEOGRAPHY(const GeographyType& val); + void __set_VECTOR(const VectorType& val); + bool operator == (const LogicalType & rhs) const; bool operator != (const LogicalType &rhs) const { return !(*this == rhs); @@ -1745,7 +1783,7 @@ void swap(LogicalType &a, LogicalType &b) noexcept; std::ostream& operator<<(std::ostream& out, const LogicalType& obj); typedef struct _SchemaElement__isset { - _SchemaElement__isset() : type(false), type_length(false), repetition_type(false), num_children(false), converted_type(false), scale(false), precision(false), field_id(false), logicalType(false) {} + _SchemaElement__isset() : type(false), type_length(false), repetition_type(false), num_children(false), converted_type(false), scale(false), precision(false), field_id(false), logicalType(false), vector_length(false) {} bool type :1; bool type_length :1; bool repetition_type :1; @@ -1755,6 +1793,7 @@ typedef struct _SchemaElement__isset { bool precision :1; bool field_id :1; bool logicalType :1; + bool vector_length :1; } _SchemaElement__isset; /** @@ -1833,6 +1872,7 @@ class SchemaElement { * for some logical types to ensure forward-compatibility in format v1. */ LogicalType logicalType; + int32_t vector_length; _SchemaElement__isset __isset; @@ -1856,6 +1896,8 @@ class SchemaElement { void __set_logicalType(const LogicalType& val); + void __set_vector_length(const int32_t val); + bool operator == (const SchemaElement & rhs) const; bool operator != (const SchemaElement &rhs) const { return !(*this == rhs); diff --git a/cpp/src/generated/parquet_types.tcc b/cpp/src/generated/parquet_types.tcc index 01559f897372..073741b125f7 100644 --- a/cpp/src/generated/parquet_types.tcc +++ b/cpp/src/generated/parquet_types.tcc @@ -1638,6 +1638,46 @@ uint32_t GeographyType::write(Protocol_* oprot) const { return xfer; } +template +uint32_t VectorType::read(Protocol_* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + xfer += iprot->skip(ftype); + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +template +uint32_t VectorType::write(Protocol_* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("VectorType"); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + template uint32_t LogicalType::read(Protocol_* iprot) { @@ -1796,6 +1836,14 @@ uint32_t LogicalType::read(Protocol_* iprot) { xfer += iprot->skip(ftype); } break; + case 19: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->VECTOR.read(iprot); + this->__isset.VECTOR = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -1899,6 +1947,11 @@ uint32_t LogicalType::write(Protocol_* oprot) const { xfer += this->GEOGRAPHY.write(oprot); xfer += oprot->writeFieldEnd(); } + if (this->__isset.VECTOR) { + xfer += oprot->writeFieldBegin("VECTOR", ::apache::thrift::protocol::T_STRUCT, 19); + xfer += this->VECTOR.write(oprot); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -1929,9 +1982,9 @@ uint32_t SchemaElement::read(Protocol_* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast123; - xfer += iprot->readI32(ecast123); - this->type = static_cast(ecast123); + int32_t ecast127; + xfer += iprot->readI32(ecast127); + this->type = static_cast(ecast127); this->__isset.type = true; } else { xfer += iprot->skip(ftype); @@ -1947,9 +2000,9 @@ uint32_t SchemaElement::read(Protocol_* iprot) { break; case 3: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast124; - xfer += iprot->readI32(ecast124); - this->repetition_type = static_cast(ecast124); + int32_t ecast128; + xfer += iprot->readI32(ecast128); + this->repetition_type = static_cast(ecast128); this->__isset.repetition_type = true; } else { xfer += iprot->skip(ftype); @@ -1973,9 +2026,9 @@ uint32_t SchemaElement::read(Protocol_* iprot) { break; case 6: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast125; - xfer += iprot->readI32(ecast125); - this->converted_type = static_cast(ecast125); + int32_t ecast129; + xfer += iprot->readI32(ecast129); + this->converted_type = static_cast(ecast129); this->__isset.converted_type = true; } else { xfer += iprot->skip(ftype); @@ -2013,6 +2066,14 @@ uint32_t SchemaElement::read(Protocol_* iprot) { xfer += iprot->skip(ftype); } break; + case 12: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->vector_length); + this->__isset.vector_length = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -2082,6 +2143,11 @@ uint32_t SchemaElement::write(Protocol_* oprot) const { xfer += this->logicalType.write(oprot); xfer += oprot->writeFieldEnd(); } + if (this->__isset.vector_length) { + xfer += oprot->writeFieldBegin("vector_length", ::apache::thrift::protocol::T_I32, 12); + xfer += oprot->writeI32(this->vector_length); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -2123,9 +2189,9 @@ uint32_t DataPageHeader::read(Protocol_* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast130; - xfer += iprot->readI32(ecast130); - this->encoding = static_cast(ecast130); + int32_t ecast134; + xfer += iprot->readI32(ecast134); + this->encoding = static_cast(ecast134); isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2133,9 +2199,9 @@ uint32_t DataPageHeader::read(Protocol_* iprot) { break; case 3: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast131; - xfer += iprot->readI32(ecast131); - this->definition_level_encoding = static_cast(ecast131); + int32_t ecast135; + xfer += iprot->readI32(ecast135); + this->definition_level_encoding = static_cast(ecast135); isset_definition_level_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2143,9 +2209,9 @@ uint32_t DataPageHeader::read(Protocol_* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast132; - xfer += iprot->readI32(ecast132); - this->repetition_level_encoding = static_cast(ecast132); + int32_t ecast136; + xfer += iprot->readI32(ecast136); + this->repetition_level_encoding = static_cast(ecast136); isset_repetition_level_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2285,9 +2351,9 @@ uint32_t DictionaryPageHeader::read(Protocol_* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast141; - xfer += iprot->readI32(ecast141); - this->encoding = static_cast(ecast141); + int32_t ecast145; + xfer += iprot->readI32(ecast145); + this->encoding = static_cast(ecast145); isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2395,9 +2461,9 @@ uint32_t DataPageHeaderV2::read(Protocol_* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast146; - xfer += iprot->readI32(ecast146); - this->encoding = static_cast(ecast146); + int32_t ecast150; + xfer += iprot->readI32(ecast150); + this->encoding = static_cast(ecast150); isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2930,9 +2996,9 @@ uint32_t PageHeader::read(Protocol_* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast179; - xfer += iprot->readI32(ecast179); - this->type = static_cast(ecast179); + int32_t ecast183; + xfer += iprot->readI32(ecast183); + this->type = static_cast(ecast183); isset_type = true; } else { xfer += iprot->skip(ftype); @@ -3250,9 +3316,9 @@ uint32_t PageEncodingStats::read(Protocol_* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast192; - xfer += iprot->readI32(ecast192); - this->page_type = static_cast(ecast192); + int32_t ecast196; + xfer += iprot->readI32(ecast196); + this->page_type = static_cast(ecast196); isset_page_type = true; } else { xfer += iprot->skip(ftype); @@ -3260,9 +3326,9 @@ uint32_t PageEncodingStats::read(Protocol_* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast193; - xfer += iprot->readI32(ecast193); - this->encoding = static_cast(ecast193); + int32_t ecast197; + xfer += iprot->readI32(ecast197); + this->encoding = static_cast(ecast197); isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -3349,9 +3415,9 @@ uint32_t ColumnMetaData::read(Protocol_* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast198; - xfer += iprot->readI32(ecast198); - this->type = static_cast(ecast198); + int32_t ecast202; + xfer += iprot->readI32(ecast202); + this->type = static_cast(ecast202); isset_type = true; } else { xfer += iprot->skip(ftype); @@ -3361,16 +3427,16 @@ uint32_t ColumnMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->encodings.clear(); - uint32_t _size199; - ::apache::thrift::protocol::TType _etype202; - xfer += iprot->readListBegin(_etype202, _size199); - this->encodings.resize(_size199); - uint32_t _i203; - for (_i203 = 0; _i203 < _size199; ++_i203) + uint32_t _size203; + ::apache::thrift::protocol::TType _etype206; + xfer += iprot->readListBegin(_etype206, _size203); + this->encodings.resize(_size203); + uint32_t _i207; + for (_i207 = 0; _i207 < _size203; ++_i207) { - int32_t ecast204; - xfer += iprot->readI32(ecast204); - this->encodings[_i203] = static_cast(ecast204); + int32_t ecast208; + xfer += iprot->readI32(ecast208); + this->encodings[_i207] = static_cast(ecast208); } xfer += iprot->readListEnd(); } @@ -3383,14 +3449,14 @@ uint32_t ColumnMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->path_in_schema.clear(); - uint32_t _size205; - ::apache::thrift::protocol::TType _etype208; - xfer += iprot->readListBegin(_etype208, _size205); - this->path_in_schema.resize(_size205); - uint32_t _i209; - for (_i209 = 0; _i209 < _size205; ++_i209) + uint32_t _size209; + ::apache::thrift::protocol::TType _etype212; + xfer += iprot->readListBegin(_etype212, _size209); + this->path_in_schema.resize(_size209); + uint32_t _i213; + for (_i213 = 0; _i213 < _size209; ++_i213) { - xfer += iprot->readString(this->path_in_schema[_i209]); + xfer += iprot->readString(this->path_in_schema[_i213]); } xfer += iprot->readListEnd(); } @@ -3401,9 +3467,9 @@ uint32_t ColumnMetaData::read(Protocol_* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast210; - xfer += iprot->readI32(ecast210); - this->codec = static_cast(ecast210); + int32_t ecast214; + xfer += iprot->readI32(ecast214); + this->codec = static_cast(ecast214); isset_codec = true; } else { xfer += iprot->skip(ftype); @@ -3437,14 +3503,14 @@ uint32_t ColumnMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->key_value_metadata.clear(); - uint32_t _size211; - ::apache::thrift::protocol::TType _etype214; - xfer += iprot->readListBegin(_etype214, _size211); - this->key_value_metadata.resize(_size211); - uint32_t _i215; - for (_i215 = 0; _i215 < _size211; ++_i215) + uint32_t _size215; + ::apache::thrift::protocol::TType _etype218; + xfer += iprot->readListBegin(_etype218, _size215); + this->key_value_metadata.resize(_size215); + uint32_t _i219; + for (_i219 = 0; _i219 < _size215; ++_i219) { - xfer += this->key_value_metadata[_i215].read(iprot); + xfer += this->key_value_metadata[_i219].read(iprot); } xfer += iprot->readListEnd(); } @@ -3489,14 +3555,14 @@ uint32_t ColumnMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->encoding_stats.clear(); - uint32_t _size216; - ::apache::thrift::protocol::TType _etype219; - xfer += iprot->readListBegin(_etype219, _size216); - this->encoding_stats.resize(_size216); - uint32_t _i220; - for (_i220 = 0; _i220 < _size216; ++_i220) + uint32_t _size220; + ::apache::thrift::protocol::TType _etype223; + xfer += iprot->readListBegin(_etype223, _size220); + this->encoding_stats.resize(_size220); + uint32_t _i224; + for (_i224 = 0; _i224 < _size220; ++_i224) { - xfer += this->encoding_stats[_i220].read(iprot); + xfer += this->encoding_stats[_i224].read(iprot); } xfer += iprot->readListEnd(); } @@ -3578,10 +3644,10 @@ uint32_t ColumnMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("encodings", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I32, static_cast(this->encodings.size())); - std::vector ::const_iterator _iter221; - for (_iter221 = this->encodings.begin(); _iter221 != this->encodings.end(); ++_iter221) + std::vector ::const_iterator _iter225; + for (_iter225 = this->encodings.begin(); _iter225 != this->encodings.end(); ++_iter225) { - xfer += oprot->writeI32(static_cast((*_iter221))); + xfer += oprot->writeI32(static_cast((*_iter225))); } xfer += oprot->writeListEnd(); } @@ -3590,10 +3656,10 @@ uint32_t ColumnMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("path_in_schema", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->path_in_schema.size())); - std::vector ::const_iterator _iter222; - for (_iter222 = this->path_in_schema.begin(); _iter222 != this->path_in_schema.end(); ++_iter222) + std::vector ::const_iterator _iter226; + for (_iter226 = this->path_in_schema.begin(); _iter226 != this->path_in_schema.end(); ++_iter226) { - xfer += oprot->writeString((*_iter222)); + xfer += oprot->writeString((*_iter226)); } xfer += oprot->writeListEnd(); } @@ -3619,10 +3685,10 @@ uint32_t ColumnMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("key_value_metadata", ::apache::thrift::protocol::T_LIST, 8); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->key_value_metadata.size())); - std::vector ::const_iterator _iter223; - for (_iter223 = this->key_value_metadata.begin(); _iter223 != this->key_value_metadata.end(); ++_iter223) + std::vector ::const_iterator _iter227; + for (_iter227 = this->key_value_metadata.begin(); _iter227 != this->key_value_metadata.end(); ++_iter227) { - xfer += (*_iter223).write(oprot); + xfer += (*_iter227).write(oprot); } xfer += oprot->writeListEnd(); } @@ -3651,10 +3717,10 @@ uint32_t ColumnMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("encoding_stats", ::apache::thrift::protocol::T_LIST, 13); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->encoding_stats.size())); - std::vector ::const_iterator _iter224; - for (_iter224 = this->encoding_stats.begin(); _iter224 != this->encoding_stats.end(); ++_iter224) + std::vector ::const_iterator _iter228; + for (_iter228 = this->encoding_stats.begin(); _iter228 != this->encoding_stats.end(); ++_iter228) { - xfer += (*_iter224).write(oprot); + xfer += (*_iter228).write(oprot); } xfer += oprot->writeListEnd(); } @@ -3752,14 +3818,14 @@ uint32_t EncryptionWithColumnKey::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->path_in_schema.clear(); - uint32_t _size233; - ::apache::thrift::protocol::TType _etype236; - xfer += iprot->readListBegin(_etype236, _size233); - this->path_in_schema.resize(_size233); - uint32_t _i237; - for (_i237 = 0; _i237 < _size233; ++_i237) + uint32_t _size237; + ::apache::thrift::protocol::TType _etype240; + xfer += iprot->readListBegin(_etype240, _size237); + this->path_in_schema.resize(_size237); + uint32_t _i241; + for (_i241 = 0; _i241 < _size237; ++_i241) { - xfer += iprot->readString(this->path_in_schema[_i237]); + xfer += iprot->readString(this->path_in_schema[_i241]); } xfer += iprot->readListEnd(); } @@ -3799,10 +3865,10 @@ uint32_t EncryptionWithColumnKey::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("path_in_schema", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->path_in_schema.size())); - std::vector ::const_iterator _iter238; - for (_iter238 = this->path_in_schema.begin(); _iter238 != this->path_in_schema.end(); ++_iter238) + std::vector ::const_iterator _iter242; + for (_iter242 = this->path_in_schema.begin(); _iter242 != this->path_in_schema.end(); ++_iter242) { - xfer += oprot->writeString((*_iter238)); + xfer += oprot->writeString((*_iter242)); } xfer += oprot->writeListEnd(); } @@ -4082,14 +4148,14 @@ uint32_t RowGroup::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->columns.clear(); - uint32_t _size251; - ::apache::thrift::protocol::TType _etype254; - xfer += iprot->readListBegin(_etype254, _size251); - this->columns.resize(_size251); - uint32_t _i255; - for (_i255 = 0; _i255 < _size251; ++_i255) + uint32_t _size255; + ::apache::thrift::protocol::TType _etype258; + xfer += iprot->readListBegin(_etype258, _size255); + this->columns.resize(_size255); + uint32_t _i259; + for (_i259 = 0; _i259 < _size255; ++_i259) { - xfer += this->columns[_i255].read(iprot); + xfer += this->columns[_i259].read(iprot); } xfer += iprot->readListEnd(); } @@ -4118,14 +4184,14 @@ uint32_t RowGroup::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->sorting_columns.clear(); - uint32_t _size256; - ::apache::thrift::protocol::TType _etype259; - xfer += iprot->readListBegin(_etype259, _size256); - this->sorting_columns.resize(_size256); - uint32_t _i260; - for (_i260 = 0; _i260 < _size256; ++_i260) + uint32_t _size260; + ::apache::thrift::protocol::TType _etype263; + xfer += iprot->readListBegin(_etype263, _size260); + this->sorting_columns.resize(_size260); + uint32_t _i264; + for (_i264 = 0; _i264 < _size260; ++_i264) { - xfer += this->sorting_columns[_i260].read(iprot); + xfer += this->sorting_columns[_i264].read(iprot); } xfer += iprot->readListEnd(); } @@ -4185,10 +4251,10 @@ uint32_t RowGroup::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("columns", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->columns.size())); - std::vector ::const_iterator _iter261; - for (_iter261 = this->columns.begin(); _iter261 != this->columns.end(); ++_iter261) + std::vector ::const_iterator _iter265; + for (_iter265 = this->columns.begin(); _iter265 != this->columns.end(); ++_iter265) { - xfer += (*_iter261).write(oprot); + xfer += (*_iter265).write(oprot); } xfer += oprot->writeListEnd(); } @@ -4206,10 +4272,10 @@ uint32_t RowGroup::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("sorting_columns", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->sorting_columns.size())); - std::vector ::const_iterator _iter262; - for (_iter262 = this->sorting_columns.begin(); _iter262 != this->sorting_columns.end(); ++_iter262) + std::vector ::const_iterator _iter266; + for (_iter266 = this->sorting_columns.begin(); _iter266 != this->sorting_columns.end(); ++_iter266) { - xfer += (*_iter262).write(oprot); + xfer += (*_iter266).write(oprot); } xfer += oprot->writeListEnd(); } @@ -4503,14 +4569,14 @@ uint32_t OffsetIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->page_locations.clear(); - uint32_t _size283; - ::apache::thrift::protocol::TType _etype286; - xfer += iprot->readListBegin(_etype286, _size283); - this->page_locations.resize(_size283); - uint32_t _i287; - for (_i287 = 0; _i287 < _size283; ++_i287) + uint32_t _size287; + ::apache::thrift::protocol::TType _etype290; + xfer += iprot->readListBegin(_etype290, _size287); + this->page_locations.resize(_size287); + uint32_t _i291; + for (_i291 = 0; _i291 < _size287; ++_i291) { - xfer += this->page_locations[_i287].read(iprot); + xfer += this->page_locations[_i291].read(iprot); } xfer += iprot->readListEnd(); } @@ -4523,14 +4589,14 @@ uint32_t OffsetIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->unencoded_byte_array_data_bytes.clear(); - uint32_t _size288; - ::apache::thrift::protocol::TType _etype291; - xfer += iprot->readListBegin(_etype291, _size288); - this->unencoded_byte_array_data_bytes.resize(_size288); - uint32_t _i292; - for (_i292 = 0; _i292 < _size288; ++_i292) + uint32_t _size292; + ::apache::thrift::protocol::TType _etype295; + xfer += iprot->readListBegin(_etype295, _size292); + this->unencoded_byte_array_data_bytes.resize(_size292); + uint32_t _i296; + for (_i296 = 0; _i296 < _size292; ++_i296) { - xfer += iprot->readI64(this->unencoded_byte_array_data_bytes[_i292]); + xfer += iprot->readI64(this->unencoded_byte_array_data_bytes[_i296]); } xfer += iprot->readListEnd(); } @@ -4562,10 +4628,10 @@ uint32_t OffsetIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("page_locations", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->page_locations.size())); - std::vector ::const_iterator _iter293; - for (_iter293 = this->page_locations.begin(); _iter293 != this->page_locations.end(); ++_iter293) + std::vector ::const_iterator _iter297; + for (_iter297 = this->page_locations.begin(); _iter297 != this->page_locations.end(); ++_iter297) { - xfer += (*_iter293).write(oprot); + xfer += (*_iter297).write(oprot); } xfer += oprot->writeListEnd(); } @@ -4575,10 +4641,10 @@ uint32_t OffsetIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("unencoded_byte_array_data_bytes", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->unencoded_byte_array_data_bytes.size())); - std::vector ::const_iterator _iter294; - for (_iter294 = this->unencoded_byte_array_data_bytes.begin(); _iter294 != this->unencoded_byte_array_data_bytes.end(); ++_iter294) + std::vector ::const_iterator _iter298; + for (_iter298 = this->unencoded_byte_array_data_bytes.begin(); _iter298 != this->unencoded_byte_array_data_bytes.end(); ++_iter298) { - xfer += oprot->writeI64((*_iter294)); + xfer += oprot->writeI64((*_iter298)); } xfer += oprot->writeListEnd(); } @@ -4619,14 +4685,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->null_pages.clear(); - uint32_t _size299; - ::apache::thrift::protocol::TType _etype302; - xfer += iprot->readListBegin(_etype302, _size299); - this->null_pages.resize(_size299); - uint32_t _i303; - for (_i303 = 0; _i303 < _size299; ++_i303) + uint32_t _size303; + ::apache::thrift::protocol::TType _etype306; + xfer += iprot->readListBegin(_etype306, _size303); + this->null_pages.resize(_size303); + uint32_t _i307; + for (_i307 = 0; _i307 < _size303; ++_i307) { - xfer += iprot->readBool(this->null_pages[_i303]); + xfer += iprot->readBool(this->null_pages[_i307]); } xfer += iprot->readListEnd(); } @@ -4639,14 +4705,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->min_values.clear(); - uint32_t _size304; - ::apache::thrift::protocol::TType _etype307; - xfer += iprot->readListBegin(_etype307, _size304); - this->min_values.resize(_size304); - uint32_t _i308; - for (_i308 = 0; _i308 < _size304; ++_i308) + uint32_t _size308; + ::apache::thrift::protocol::TType _etype311; + xfer += iprot->readListBegin(_etype311, _size308); + this->min_values.resize(_size308); + uint32_t _i312; + for (_i312 = 0; _i312 < _size308; ++_i312) { - xfer += iprot->readBinary(this->min_values[_i308]); + xfer += iprot->readBinary(this->min_values[_i312]); } xfer += iprot->readListEnd(); } @@ -4659,14 +4725,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->max_values.clear(); - uint32_t _size309; - ::apache::thrift::protocol::TType _etype312; - xfer += iprot->readListBegin(_etype312, _size309); - this->max_values.resize(_size309); - uint32_t _i313; - for (_i313 = 0; _i313 < _size309; ++_i313) + uint32_t _size313; + ::apache::thrift::protocol::TType _etype316; + xfer += iprot->readListBegin(_etype316, _size313); + this->max_values.resize(_size313); + uint32_t _i317; + for (_i317 = 0; _i317 < _size313; ++_i317) { - xfer += iprot->readBinary(this->max_values[_i313]); + xfer += iprot->readBinary(this->max_values[_i317]); } xfer += iprot->readListEnd(); } @@ -4677,9 +4743,9 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast314; - xfer += iprot->readI32(ecast314); - this->boundary_order = static_cast(ecast314); + int32_t ecast318; + xfer += iprot->readI32(ecast318); + this->boundary_order = static_cast(ecast318); isset_boundary_order = true; } else { xfer += iprot->skip(ftype); @@ -4689,14 +4755,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->null_counts.clear(); - uint32_t _size315; - ::apache::thrift::protocol::TType _etype318; - xfer += iprot->readListBegin(_etype318, _size315); - this->null_counts.resize(_size315); - uint32_t _i319; - for (_i319 = 0; _i319 < _size315; ++_i319) + uint32_t _size319; + ::apache::thrift::protocol::TType _etype322; + xfer += iprot->readListBegin(_etype322, _size319); + this->null_counts.resize(_size319); + uint32_t _i323; + for (_i323 = 0; _i323 < _size319; ++_i323) { - xfer += iprot->readI64(this->null_counts[_i319]); + xfer += iprot->readI64(this->null_counts[_i323]); } xfer += iprot->readListEnd(); } @@ -4709,14 +4775,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->repetition_level_histograms.clear(); - uint32_t _size320; - ::apache::thrift::protocol::TType _etype323; - xfer += iprot->readListBegin(_etype323, _size320); - this->repetition_level_histograms.resize(_size320); - uint32_t _i324; - for (_i324 = 0; _i324 < _size320; ++_i324) + uint32_t _size324; + ::apache::thrift::protocol::TType _etype327; + xfer += iprot->readListBegin(_etype327, _size324); + this->repetition_level_histograms.resize(_size324); + uint32_t _i328; + for (_i328 = 0; _i328 < _size324; ++_i328) { - xfer += iprot->readI64(this->repetition_level_histograms[_i324]); + xfer += iprot->readI64(this->repetition_level_histograms[_i328]); } xfer += iprot->readListEnd(); } @@ -4729,14 +4795,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->definition_level_histograms.clear(); - uint32_t _size325; - ::apache::thrift::protocol::TType _etype328; - xfer += iprot->readListBegin(_etype328, _size325); - this->definition_level_histograms.resize(_size325); - uint32_t _i329; - for (_i329 = 0; _i329 < _size325; ++_i329) + uint32_t _size329; + ::apache::thrift::protocol::TType _etype332; + xfer += iprot->readListBegin(_etype332, _size329); + this->definition_level_histograms.resize(_size329); + uint32_t _i333; + for (_i333 = 0; _i333 < _size329; ++_i333) { - xfer += iprot->readI64(this->definition_level_histograms[_i329]); + xfer += iprot->readI64(this->definition_level_histograms[_i333]); } xfer += iprot->readListEnd(); } @@ -4749,14 +4815,14 @@ uint32_t ColumnIndex::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->nan_counts.clear(); - uint32_t _size330; - ::apache::thrift::protocol::TType _etype333; - xfer += iprot->readListBegin(_etype333, _size330); - this->nan_counts.resize(_size330); - uint32_t _i334; - for (_i334 = 0; _i334 < _size330; ++_i334) + uint32_t _size334; + ::apache::thrift::protocol::TType _etype337; + xfer += iprot->readListBegin(_etype337, _size334); + this->nan_counts.resize(_size334); + uint32_t _i338; + for (_i338 = 0; _i338 < _size334; ++_i338) { - xfer += iprot->readI64(this->nan_counts[_i334]); + xfer += iprot->readI64(this->nan_counts[_i338]); } xfer += iprot->readListEnd(); } @@ -4794,10 +4860,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("null_pages", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_BOOL, static_cast(this->null_pages.size())); - std::vector ::const_iterator _iter335; - for (_iter335 = this->null_pages.begin(); _iter335 != this->null_pages.end(); ++_iter335) + std::vector ::const_iterator _iter339; + for (_iter339 = this->null_pages.begin(); _iter339 != this->null_pages.end(); ++_iter339) { - xfer += oprot->writeBool((*_iter335)); + xfer += oprot->writeBool((*_iter339)); } xfer += oprot->writeListEnd(); } @@ -4806,10 +4872,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("min_values", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->min_values.size())); - std::vector ::const_iterator _iter336; - for (_iter336 = this->min_values.begin(); _iter336 != this->min_values.end(); ++_iter336) + std::vector ::const_iterator _iter340; + for (_iter340 = this->min_values.begin(); _iter340 != this->min_values.end(); ++_iter340) { - xfer += oprot->writeBinary((*_iter336)); + xfer += oprot->writeBinary((*_iter340)); } xfer += oprot->writeListEnd(); } @@ -4818,10 +4884,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("max_values", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->max_values.size())); - std::vector ::const_iterator _iter337; - for (_iter337 = this->max_values.begin(); _iter337 != this->max_values.end(); ++_iter337) + std::vector ::const_iterator _iter341; + for (_iter341 = this->max_values.begin(); _iter341 != this->max_values.end(); ++_iter341) { - xfer += oprot->writeBinary((*_iter337)); + xfer += oprot->writeBinary((*_iter341)); } xfer += oprot->writeListEnd(); } @@ -4835,10 +4901,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("null_counts", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->null_counts.size())); - std::vector ::const_iterator _iter338; - for (_iter338 = this->null_counts.begin(); _iter338 != this->null_counts.end(); ++_iter338) + std::vector ::const_iterator _iter342; + for (_iter342 = this->null_counts.begin(); _iter342 != this->null_counts.end(); ++_iter342) { - xfer += oprot->writeI64((*_iter338)); + xfer += oprot->writeI64((*_iter342)); } xfer += oprot->writeListEnd(); } @@ -4848,10 +4914,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("repetition_level_histograms", ::apache::thrift::protocol::T_LIST, 6); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->repetition_level_histograms.size())); - std::vector ::const_iterator _iter339; - for (_iter339 = this->repetition_level_histograms.begin(); _iter339 != this->repetition_level_histograms.end(); ++_iter339) + std::vector ::const_iterator _iter343; + for (_iter343 = this->repetition_level_histograms.begin(); _iter343 != this->repetition_level_histograms.end(); ++_iter343) { - xfer += oprot->writeI64((*_iter339)); + xfer += oprot->writeI64((*_iter343)); } xfer += oprot->writeListEnd(); } @@ -4861,10 +4927,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("definition_level_histograms", ::apache::thrift::protocol::T_LIST, 7); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->definition_level_histograms.size())); - std::vector ::const_iterator _iter340; - for (_iter340 = this->definition_level_histograms.begin(); _iter340 != this->definition_level_histograms.end(); ++_iter340) + std::vector ::const_iterator _iter344; + for (_iter344 = this->definition_level_histograms.begin(); _iter344 != this->definition_level_histograms.end(); ++_iter344) { - xfer += oprot->writeI64((*_iter340)); + xfer += oprot->writeI64((*_iter344)); } xfer += oprot->writeListEnd(); } @@ -4874,10 +4940,10 @@ uint32_t ColumnIndex::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("nan_counts", ::apache::thrift::protocol::T_LIST, 8); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->nan_counts.size())); - std::vector ::const_iterator _iter341; - for (_iter341 = this->nan_counts.begin(); _iter341 != this->nan_counts.end(); ++_iter341) + std::vector ::const_iterator _iter345; + for (_iter345 = this->nan_counts.begin(); _iter345 != this->nan_counts.end(); ++_iter345) { - xfer += oprot->writeI64((*_iter341)); + xfer += oprot->writeI64((*_iter345)); } xfer += oprot->writeListEnd(); } @@ -5165,14 +5231,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->schema.clear(); - uint32_t _size358; - ::apache::thrift::protocol::TType _etype361; - xfer += iprot->readListBegin(_etype361, _size358); - this->schema.resize(_size358); - uint32_t _i362; - for (_i362 = 0; _i362 < _size358; ++_i362) + uint32_t _size362; + ::apache::thrift::protocol::TType _etype365; + xfer += iprot->readListBegin(_etype365, _size362); + this->schema.resize(_size362); + uint32_t _i366; + for (_i366 = 0; _i366 < _size362; ++_i366) { - xfer += this->schema[_i362].read(iprot); + xfer += this->schema[_i366].read(iprot); } xfer += iprot->readListEnd(); } @@ -5193,14 +5259,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->row_groups.clear(); - uint32_t _size363; - ::apache::thrift::protocol::TType _etype366; - xfer += iprot->readListBegin(_etype366, _size363); - this->row_groups.resize(_size363); - uint32_t _i367; - for (_i367 = 0; _i367 < _size363; ++_i367) + uint32_t _size367; + ::apache::thrift::protocol::TType _etype370; + xfer += iprot->readListBegin(_etype370, _size367); + this->row_groups.resize(_size367); + uint32_t _i371; + for (_i371 = 0; _i371 < _size367; ++_i371) { - xfer += this->row_groups[_i367].read(iprot); + xfer += this->row_groups[_i371].read(iprot); } xfer += iprot->readListEnd(); } @@ -5213,14 +5279,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->key_value_metadata.clear(); - uint32_t _size368; - ::apache::thrift::protocol::TType _etype371; - xfer += iprot->readListBegin(_etype371, _size368); - this->key_value_metadata.resize(_size368); - uint32_t _i372; - for (_i372 = 0; _i372 < _size368; ++_i372) + uint32_t _size372; + ::apache::thrift::protocol::TType _etype375; + xfer += iprot->readListBegin(_etype375, _size372); + this->key_value_metadata.resize(_size372); + uint32_t _i376; + for (_i376 = 0; _i376 < _size372; ++_i376) { - xfer += this->key_value_metadata[_i372].read(iprot); + xfer += this->key_value_metadata[_i376].read(iprot); } xfer += iprot->readListEnd(); } @@ -5241,14 +5307,14 @@ uint32_t FileMetaData::read(Protocol_* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->column_orders.clear(); - uint32_t _size373; - ::apache::thrift::protocol::TType _etype376; - xfer += iprot->readListBegin(_etype376, _size373); - this->column_orders.resize(_size373); - uint32_t _i377; - for (_i377 = 0; _i377 < _size373; ++_i377) + uint32_t _size377; + ::apache::thrift::protocol::TType _etype380; + xfer += iprot->readListBegin(_etype380, _size377); + this->column_orders.resize(_size377); + uint32_t _i381; + for (_i381 = 0; _i381 < _size377; ++_i381) { - xfer += this->column_orders[_i377].read(iprot); + xfer += this->column_orders[_i381].read(iprot); } xfer += iprot->readListEnd(); } @@ -5306,10 +5372,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("schema", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->schema.size())); - std::vector ::const_iterator _iter378; - for (_iter378 = this->schema.begin(); _iter378 != this->schema.end(); ++_iter378) + std::vector ::const_iterator _iter382; + for (_iter382 = this->schema.begin(); _iter382 != this->schema.end(); ++_iter382) { - xfer += (*_iter378).write(oprot); + xfer += (*_iter382).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5322,10 +5388,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("row_groups", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->row_groups.size())); - std::vector ::const_iterator _iter379; - for (_iter379 = this->row_groups.begin(); _iter379 != this->row_groups.end(); ++_iter379) + std::vector ::const_iterator _iter383; + for (_iter383 = this->row_groups.begin(); _iter383 != this->row_groups.end(); ++_iter383) { - xfer += (*_iter379).write(oprot); + xfer += (*_iter383).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5335,10 +5401,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("key_value_metadata", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->key_value_metadata.size())); - std::vector ::const_iterator _iter380; - for (_iter380 = this->key_value_metadata.begin(); _iter380 != this->key_value_metadata.end(); ++_iter380) + std::vector ::const_iterator _iter384; + for (_iter384 = this->key_value_metadata.begin(); _iter384 != this->key_value_metadata.end(); ++_iter384) { - xfer += (*_iter380).write(oprot); + xfer += (*_iter384).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5353,10 +5419,10 @@ uint32_t FileMetaData::write(Protocol_* oprot) const { xfer += oprot->writeFieldBegin("column_orders", ::apache::thrift::protocol::T_LIST, 7); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->column_orders.size())); - std::vector ::const_iterator _iter381; - for (_iter381 = this->column_orders.begin(); _iter381 != this->column_orders.end(); ++_iter381) + std::vector ::const_iterator _iter385; + for (_iter385 = this->column_orders.begin(); _iter385 != this->column_orders.end(); ++_iter385) { - xfer += (*_iter381).write(oprot); + xfer += (*_iter385).write(oprot); } xfer += oprot->writeListEnd(); } diff --git a/cpp/src/parquet/parquet.thrift b/cpp/src/parquet/parquet.thrift index 9603cefed388..d5f9b9608d73 100644 --- a/cpp/src/parquet/parquet.thrift +++ b/cpp/src/parquet/parquet.thrift @@ -190,6 +190,63 @@ enum FieldRepetitionType { /** The field is repeated and can contain 0 or more values */ REPEATED = 2; + + // EXPERIMENTAL VECTOR layout rules (kept out of the doc comment so they are + // not copied into generated code): + // + // A VECTOR field repeats exactly vector_length times for every parent value + // and, unlike REPEATED, does not increase the maximum definition or + // repetition level of its descendants: readers reconstruct vector values + // from the fixed multiplicity declared in the schema instead of decoding + // repetition levels. + // + // Vector fields MUST use the following 3-level structure, mirroring LIST: + // + // group (VECTOR) { + // vector group list [vector_length] { + // element; + // } + // } + // + // - The outer group MUST be REQUIRED or OPTIONAL and carries the + // nullability of the vector value itself. It MUST be annotated with the + // VECTOR logical type and MUST have exactly one child: the + // VECTOR-repeated group. + // - The VECTOR-repeated middle group carries vector_length and MUST have + // exactly one child: the element field. + // - The element field MUST be REQUIRED or OPTIONAL and MAY be a primitive + // type, a group of fields (a struct), or another vector group in the same + // form. For nested vectors a leaf stores the product of the + // vector_length values of its VECTOR ancestors physical values per parent + // record. + // + // Data layout rules: + // - Writers MUST emit exactly vector_length child slots for every parent + // record in which the vector value is present or null; null vector values + // emit vector_length slots carrying the definition level of the null + // vector. Ancestors that make the vector value absent altogether (for + // example an empty or null list) contribute a single level entry with + // their usual definition level and no slots. Column chunk num_values and + // null counts therefore include the padding slots of null vector values. + // - Every slot of an existing vector occupies one position in both level + // streams. The repetition level of the first slot encodes record + // structure as usual; the repetition levels of the remaining + // vector_length - 1 slots MUST be written as the column's maximum + // repetition level and MUST be ignored by readers. Stream consumption is + // definition-driven: after reading a position whose definition level + // indicates an existing vector, a reader consumes that position and the + // following vector_length - 1 positions as one vector value. + // - Writers MUST NOT split the slots of one vector value across data pages; + // pages begin and end on whole-vector boundaries. + // - vector_length MUST be positive: a zero-length vector value would + // contribute no slots at all, leaving row counts and row nullability + // unrepresentable in this layout. Writers MUST represent zero-length + // fixed-size lists with the LIST encoding instead. + // + // Readers that do not understand VECTOR are expected to reject the file. + + // EXPERIMENTAL: fixed-size repetition. + VECTOR = 3; } /** @@ -469,6 +526,17 @@ struct GeographyType { 2: optional EdgeInterpolationAlgorithm algorithm; } +// The VECTOR annotation marks the outer group of a fixed-size vector field; +// see FieldRepetitionType.VECTOR for the full structure and data layout +// rules. The annotated group MUST be REQUIRED or OPTIONAL and MUST have +// exactly one child: a VECTOR-repeated group whose +// SchemaElement.vector_length carries the fixed multiplicity. The annotation +// is required so that readers can distinguish a vector field from a +// struct-like group that happens to contain a VECTOR-repeated child. + +// EXPERIMENTAL: Embedded Vector logical type annotation. +struct VectorType {} // allowed for group nodes only + /** * LogicalType annotations to replace ConvertedType. * @@ -502,6 +570,7 @@ union LogicalType { 16: VariantType VARIANT // no compatible ConvertedType 17: GeometryType GEOMETRY // no compatible ConvertedType 18: GeographyType GEOGRAPHY // no compatible ConvertedType + 19: VectorType VECTOR // no compatible ConvertedType } /** @@ -564,6 +633,16 @@ struct SchemaElement { * for some logical types to ensure forward-compatibility in format v1. */ 10: optional LogicalType logicalType + + // vector_length MUST be set, and positive, when repetition_type is VECTOR + // and MUST NOT be set otherwise. Zero-length vectors are not representable + // as VECTOR and use the LIST encoding (see FieldRepetitionType.VECTOR). + // For nested VECTOR fields the number of physical leaf values per parent + // record is the product of vector_length over the leaf's VECTOR ancestors. + + // EXPERIMENTAL: The fixed number of times the field repeats per parent + // value when repetition_type is VECTOR. + 12: optional i32 vector_length; } /** diff --git a/cpp/src/parquet/types.cc b/cpp/src/parquet/types.cc index 9d7604faec30..827e001a5f71 100644 --- a/cpp/src/parquet/types.cc +++ b/cpp/src/parquet/types.cc @@ -614,6 +614,8 @@ std::shared_ptr LogicalType::FromThrift( } return VariantLogicalType::Make(spec_version); + } else if (type.__isset.VECTOR) { + return VectorLogicalType::Make(); } else { // Sentinel type for one we do not recognize return UndefinedLogicalType::Make(); @@ -685,6 +687,10 @@ std::shared_ptr LogicalType::Variant(int8_t spec_version) { return VariantLogicalType::Make(spec_version); } +std::shared_ptr LogicalType::Vector() { + return VectorLogicalType::Make(); +} + std::shared_ptr LogicalType::None() { return NoLogicalType::Make(); } /* @@ -770,6 +776,7 @@ class LogicalType::Impl { class Geometry; class Geography; class Variant; + class Vector; class No; class Undefined; @@ -851,6 +858,7 @@ bool LogicalType::is_geography() const { bool LogicalType::is_variant() const { return impl_->type() == LogicalType::Type::VARIANT; } +bool LogicalType::is_vector() const { return impl_->type() == LogicalType::Type::VECTOR; } bool LogicalType::is_none() const { return impl_->type() == LogicalType::Type::NONE; } bool LogicalType::is_valid() const { return impl_->type() != LogicalType::Type::UNDEFINED; @@ -859,7 +867,8 @@ bool LogicalType::is_invalid() const { return !is_valid(); } bool LogicalType::is_nested() const { return impl_->type() == LogicalType::Type::LIST || impl_->type() == LogicalType::Type::MAP || - impl_->type() == LogicalType::Type::VARIANT; + impl_->type() == LogicalType::Type::VARIANT || + impl_->type() == LogicalType::Type::VECTOR; } bool LogicalType::is_nonnested() const { return !is_nested(); } bool LogicalType::is_serialized() const { return impl_->is_serialized(); } @@ -2028,6 +2037,20 @@ std::shared_ptr VariantLogicalType::Make(const int8_t spec_ve return logical_type; } +class LogicalType::Impl::Vector final : public LogicalType::Impl::Incompatible, + public LogicalType::Impl::Inapplicable { + public: + friend class VectorLogicalType; + + OVERRIDE_TOSTRING(Vector) + OVERRIDE_TOTHRIFT(VectorType, VECTOR) + + private: + Vector() : LogicalType::Impl(LogicalType::Type::VECTOR, SortOrder::UNKNOWN) {} +}; + +GENERATE_MAKE(Vector) + class LogicalType::Impl::No final : public LogicalType::Impl::SimpleCompatible, public LogicalType::Impl::UniversalApplicable { public: diff --git a/cpp/src/parquet/types.h b/cpp/src/parquet/types.h index 687353aa9bcb..6fcb24943f80 100644 --- a/cpp/src/parquet/types.h +++ b/cpp/src/parquet/types.h @@ -112,7 +112,13 @@ class LogicalType; // Mirrors parquet::FieldRepetitionType struct Repetition { - enum type { REQUIRED = 0, OPTIONAL = 1, REPEATED = 2, /*Always last*/ UNDEFINED = 3 }; + enum type { + REQUIRED = 0, + OPTIONAL = 1, + REPEATED = 2, + VECTOR = 3, + /*Always last*/ UNDEFINED = 4 + }; }; // Reference: @@ -162,6 +168,7 @@ class PARQUET_EXPORT LogicalType { GEOMETRY, GEOGRAPHY, VARIANT, + VECTOR, NONE // Not a real logical type; should always be last element }; }; @@ -231,6 +238,10 @@ class PARQUET_EXPORT LogicalType { static std::shared_ptr Variant( int8_t specVersion = kVariantSpecVersion); + /// \brief Create an experimental Vector logical type, annotating a group + /// node whose single child has VECTOR repetition. + static std::shared_ptr Vector(); + static std::shared_ptr Geometry(std::string crs = ""); static std::shared_ptr Geography( @@ -293,6 +304,7 @@ class PARQUET_EXPORT LogicalType { bool is_geometry() const; bool is_geography() const; bool is_variant() const; + bool is_vector() const; bool is_none() const; /// \brief Return true if this logical type is of a known type. bool is_valid() const; @@ -342,6 +354,16 @@ class PARQUET_EXPORT ListLogicalType : public LogicalType { ListLogicalType() = default; }; +/// \brief Allowed for group nodes only. EXPERIMENTAL: annotates a group whose +/// single child has VECTOR repetition. +class PARQUET_EXPORT VectorLogicalType : public LogicalType { + public: + static std::shared_ptr Make(); + + private: + VectorLogicalType() = default; +}; + /// \brief Allowed for physical type BYTE_ARRAY, must be encoded as UTF-8. class PARQUET_EXPORT EnumLogicalType : public LogicalType { public: From 16a93bcae541b73ba99b99dbcdefc0f89cc39d8f Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Thu, 11 Jun 2026 12:44:22 +0200 Subject: [PATCH 2/8] Support VECTOR repetition in the Parquet schema and column writer Schema nodes carry vector_length (groups only; VECTOR on primitives is rejected), and ColumnDescriptor exposes the overflow-checked effective vector length product. The column writer accounts rows for vector columns, requires whole-vector batches, breaks pages on whole-vector boundaries, and aligns content-defined chunks to vector boundaries using definition-level slot counts. --- cpp/src/parquet/column_writer.cc | 123 ++++++++++++++++++++++-- cpp/src/parquet/schema.cc | 140 ++++++++++++++++++++++------ cpp/src/parquet/schema.h | 63 +++++++++---- cpp/src/parquet/schema_test.cc | 155 ++++++++++++++++++++++++++++++- 4 files changed, 425 insertions(+), 56 deletions(-) diff --git a/cpp/src/parquet/column_writer.cc b/cpp/src/parquet/column_writer.cc index 653f28f64bde..82cb90d62340 100644 --- a/cpp/src/parquet/column_writer.cc +++ b/cpp/src/parquet/column_writer.cc @@ -83,6 +83,43 @@ namespace parquet { namespace { +int64_t CountVectorSlots(const int16_t* def_levels, int64_t level_offset, + int64_t levels_to_write, int16_t repeated_ancestor_def_level) { + if (def_levels == nullptr || repeated_ancestor_def_level <= 0) { + return levels_to_write; + } + int64_t slots = 0; + for (int64_t i = level_offset; i < level_offset + levels_to_write; ++i) { + slots += def_levels[i] >= repeated_ancestor_def_level; + } + return slots; +} + +std::vector<::parquet::internal::Chunk> AlignCdcChunksToVectorBoundaries( + std::vector<::parquet::internal::Chunk> chunks, const int16_t* def_levels, + int32_t vector_length, int16_t repeated_ancestor_def_level) { + if (vector_length <= 1 || chunks.size() <= 1) { + return chunks; + } + + std::vector<::parquet::internal::Chunk> aligned; + aligned.reserve(chunks.size()); + auto pending = chunks.front(); + for (size_t i = 1; i < chunks.size(); ++i) { + const int64_t pending_slots = + CountVectorSlots(def_levels, pending.level_offset, pending.levels_to_write, + repeated_ancestor_def_level); + if (pending_slots % vector_length == 0) { + aligned.push_back(pending); + pending = chunks[i]; + } else { + pending.levels_to_write += chunks[i].levels_to_write; + } + } + aligned.push_back(pending); + return aligned; +} + // Visitor that extracts the value buffer from a FlatArray at a given offset. struct ValueBufferSlicer { template @@ -1185,6 +1222,38 @@ inline void DoInBatchesNonRepeated(int64_t num_levels, int64_t batch_size, } } +// DoInBatches for VECTOR columns without repeated ancestors. num_levels +// counts physical leaf slots and must be a whole number of vectors; batching +// happens on vector boundaries so a vector value is never split across pages. +template +inline void DoInBatchesVectorNonRepeated(int64_t num_levels, int64_t batch_size, + int64_t max_rows_per_page, int32_t vector_length, + Action&& action, + GetBufferedRows&& curr_page_buffered_rows) { + ARROW_DCHECK_GT(vector_length, 0); + if (num_levels % vector_length != 0) { + throw ParquetException("VECTOR columns must be written in whole-vector batches"); + } + const int64_t total_rows = num_levels / vector_length; + int64_t row_offset = 0; + while (row_offset < total_rows) { + int64_t page_buffered_rows = curr_page_buffered_rows(); + ARROW_DCHECK_LE(page_buffered_rows, max_rows_per_page); + + int64_t max_batch_rows = std::max(1, batch_size / vector_length); + max_batch_rows = std::min(max_batch_rows, total_rows - row_offset); + max_batch_rows = std::min(max_batch_rows, max_rows_per_page - page_buffered_rows); + if (max_batch_rows == 0) { + max_batch_rows = 1; + } + int64_t level_offset = row_offset * vector_length; + int64_t level_count = max_batch_rows * vector_length; + + action(level_offset, level_count, /*check_page_limit=*/true); + row_offset += max_batch_rows; + } +} + // DoInBatches for repeated columns template inline void DoInBatchesRepeated(const int16_t* def_levels, const int16_t* rep_levels, @@ -1240,12 +1309,20 @@ inline void DoInBatchesRepeated(const int16_t* def_levels, const int16_t* rep_le template inline void DoInBatches(const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels, int64_t batch_size, int64_t max_rows_per_page, - bool pages_change_on_record_boundaries, Action&& action, + bool pages_change_on_record_boundaries, bool is_vector, + int32_t vector_length, Action&& action, GetBufferedRows&& curr_page_buffered_rows) { if (!rep_levels) { - DoInBatchesNonRepeated(num_levels, batch_size, max_rows_per_page, - std::forward(action), - std::forward(curr_page_buffered_rows)); + if (is_vector) { + DoInBatchesVectorNonRepeated( + num_levels, batch_size, max_rows_per_page, vector_length, + std::forward(action), + std::forward(curr_page_buffered_rows)); + } else { + DoInBatchesNonRepeated(num_levels, batch_size, max_rows_per_page, + std::forward(action), + std::forward(curr_page_buffered_rows)); + } } else { DoInBatchesRepeated(def_levels, rep_levels, num_levels, batch_size, max_rows_per_page, pages_change_on_record_boundaries, std::forward(action), @@ -1323,7 +1400,11 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, } pages_change_on_record_boundaries_ = properties->data_page_version() == ParquetDataPageVersion::V2 || - properties->page_index_enabled(descr_->path()); + properties->page_index_enabled(descr_->path()) || + // A vector value must not be split across pages; for VECTOR columns + // below repeated ancestors a record contains whole vectors, so + // breaking pages on record boundaries preserves the invariant. + descr_->in_vector_column(); } int64_t Close() override { return ColumnWriterImpl::Close(); } @@ -1366,7 +1447,8 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, }; DoInBatches(def_levels, rep_levels, num_values, properties_->write_batch_size(), properties_->max_rows_per_page(), pages_change_on_record_boundaries(), - WriteChunk, [this]() { return num_buffered_rows_; }); + descr_->in_vector_column(), descr_->effective_vector_length(), WriteChunk, + [this]() { return num_buffered_rows_; }); return value_offset; } @@ -1417,16 +1499,20 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, }; DoInBatches(def_levels, rep_levels, num_values, properties_->write_batch_size(), properties_->max_rows_per_page(), pages_change_on_record_boundaries(), - WriteChunk, [this]() { return num_buffered_rows_; }); + descr_->in_vector_column(), descr_->effective_vector_length(), WriteChunk, + [this]() { return num_buffered_rows_; }); } Status WriteArrow(const int16_t* def_levels, const int16_t* rep_levels, int64_t num_levels, const ::arrow::Array& leaf_array, ArrowWriteContext* ctx, bool leaf_field_nullable) override { BEGIN_PARQUET_CATCH_EXCEPTIONS + const bool is_vector = descr_->in_vector_column(); // Leaf nulls are canonical when there is only a single null element after a list - // and it is at the leaf. + // and it is at the leaf. VECTOR parent nulls may also be materialized as spaced + // null slots in the leaf array, so do not treat those as single nullable elements. bool single_nullable_element = + !is_vector && (level_info_.def_level == level_info_.repeated_ancestor_def_level + 1) && leaf_field_nullable; if (!leaf_field_nullable && leaf_array.null_count() != 0) { @@ -1446,6 +1532,11 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, DCHECK(content_defined_chunker_.has_value()); auto chunks = content_defined_chunker_->GetChunks(def_levels, rep_levels, num_levels, leaf_array); + if (is_vector) { + chunks = AlignCdcChunksToVectorBoundaries( + std::move(chunks), def_levels, descr_->effective_vector_length(), + level_info_.repeated_ancestor_def_level); + } for (size_t i = 0; i < chunks.size(); i++) { auto chunk = chunks[i]; auto chunk_array = leaf_array.Slice(chunk.value_offset); @@ -1700,6 +1791,13 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, } WriteRepetitionLevels(num_levels, rep_levels); + } else if (descr_->in_vector_column()) { + const int32_t vector_length = descr_->effective_vector_length(); + if (vector_length <= 0 || num_levels % vector_length != 0) { + throw ParquetException("VECTOR columns must be written in whole-vector batches"); + } + rows_written_ += num_levels / vector_length; + num_buffered_rows_ += num_levels / vector_length; } else { // Each value is exactly one row rows_written_ += num_levels; @@ -1794,6 +1892,13 @@ class TypedColumnWriterImpl : public ColumnWriterImpl, } } WriteRepetitionLevels(num_levels, rep_levels); + } else if (descr_->in_vector_column()) { + const int32_t vector_length = descr_->effective_vector_length(); + if (vector_length <= 0 || num_levels % vector_length != 0) { + throw ParquetException("VECTOR columns must be written in whole-vector batches"); + } + rows_written_ += num_levels / vector_length; + num_buffered_rows_ += num_levels / vector_length; } else { // Each value is exactly one row rows_written_ += num_levels; @@ -2078,6 +2183,7 @@ Status TypedColumnWriterImpl::WriteArrowDictionary( PARQUET_CATCH_NOT_OK( DoInBatches(def_levels, rep_levels, num_levels, properties_->write_batch_size(), properties_->max_rows_per_page(), pages_change_on_record_boundaries(), + descr_->in_vector_column(), descr_->effective_vector_length(), WriteIndicesChunk, [this]() { return num_buffered_rows_; })); return Status::OK(); } @@ -2540,6 +2646,7 @@ Status TypedColumnWriterImpl::WriteArrowDense( PARQUET_CATCH_NOT_OK( DoInBatches(def_levels, rep_levels, num_levels, properties_->write_batch_size(), properties_->max_rows_per_page(), pages_change_on_record_boundaries(), + descr_->in_vector_column(), descr_->effective_vector_length(), WriteChunk, [this]() { return num_buffered_rows_; })); return Status::OK(); } diff --git a/cpp/src/parquet/schema.cc b/cpp/src/parquet/schema.cc index 0cfa49c21c16..f1573941c0ba 100644 --- a/cpp/src/parquet/schema.cc +++ b/cpp/src/parquet/schema.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -50,6 +51,16 @@ void CheckColumnBounds(int column_index, size_t max_columns) { } } +void ValidateVectorProperties(Repetition::type repetition, int32_t vector_length) { + if (repetition == Repetition::VECTOR) { + if (vector_length <= 0) { + throw ParquetException("VECTOR nodes must specify a positive vector_length"); + } + } else if (vector_length != -1) { + throw ParquetException("Only VECTOR nodes may specify vector_length"); + } +} + } // namespace namespace schema { @@ -117,7 +128,7 @@ const std::shared_ptr Node::path() const { bool Node::EqualsInternal(const Node* other) const { return type_ == other->type_ && name_ == other->name_ && repetition_ == other->repetition_ && converted_type_ == other->converted_type_ && - field_id_ == other->field_id() && + field_id_ == other->field_id() && vector_length_ == other->vector_length() && logical_type_->Equals(*(other->logical_type())); } @@ -128,10 +139,15 @@ void Node::SetParent(const Node* parent) { parent_ = parent; } PrimitiveNode::PrimitiveNode(const std::string& name, Repetition::type repetition, Type::type type, ConvertedType::type converted_type, - int length, int precision, int scale, int id) - : Node(Node::PRIMITIVE, name, repetition, converted_type, id), + int length, int precision, int scale, int id, + int32_t vector_length) + : Node(Node::PRIMITIVE, name, repetition, converted_type, id, vector_length), physical_type_(type), type_length_(length) { + if (repetition == Repetition::VECTOR) { + throw ParquetException("VECTOR repetition is only allowed on group nodes"); + } + ValidateVectorProperties(repetition, vector_length); std::stringstream ss; // PARQUET-842: In an earlier revision, decimal_metadata_.isset was being @@ -241,10 +257,15 @@ PrimitiveNode::PrimitiveNode(const std::string& name, Repetition::type repetitio PrimitiveNode::PrimitiveNode(const std::string& name, Repetition::type repetition, std::shared_ptr logical_type, - Type::type physical_type, int physical_length, int id) - : Node(Node::PRIMITIVE, name, repetition, std::move(logical_type), id), + Type::type physical_type, int physical_length, int id, + int32_t vector_length) + : Node(Node::PRIMITIVE, name, repetition, std::move(logical_type), id, vector_length), physical_type_(physical_type), type_length_(physical_length) { + if (repetition == Repetition::VECTOR) { + throw ParquetException("VECTOR repetition is only allowed on group nodes"); + } + ValidateVectorProperties(repetition, vector_length); std::stringstream error; if (logical_type_) { // Check for logical type <=> node type consistency @@ -315,8 +336,11 @@ void PrimitiveNode::VisitConst(Node::ConstVisitor* visitor) const { // Group node GroupNode::GroupNode(const std::string& name, Repetition::type repetition, - const NodeVector& fields, ConvertedType::type converted_type, int id) - : Node(Node::GROUP, name, repetition, converted_type, id), fields_(fields) { + const NodeVector& fields, ConvertedType::type converted_type, int id, + int32_t vector_length) + : Node(Node::GROUP, name, repetition, converted_type, id, vector_length), + fields_(fields) { + ValidateVectorProperties(repetition, vector_length); // For forward compatibility, create an equivalent logical type logical_type_ = LogicalType::FromConvertedType(converted_type_); if (!(logical_type_ && (logical_type_->is_nested() || logical_type_->is_none()) && @@ -334,8 +358,11 @@ GroupNode::GroupNode(const std::string& name, Repetition::type repetition, GroupNode::GroupNode(const std::string& name, Repetition::type repetition, const NodeVector& fields, - std::shared_ptr logical_type, int id) - : Node(Node::GROUP, name, repetition, std::move(logical_type), id), fields_(fields) { + std::shared_ptr logical_type, int id, + int32_t vector_length) + : Node(Node::GROUP, name, repetition, std::move(logical_type), id, vector_length), + fields_(fields) { + ValidateVectorProperties(repetition, vector_length); if (logical_type_) { // Check for logical type <=> node type consistency if (logical_type_->is_nested()) { @@ -423,19 +450,23 @@ std::unique_ptr GroupNode::FromParquet(const void* opaque_element, if (element->__isset.field_id) { field_id = element->field_id; } + int32_t vector_length = -1; + if (element->__isset.vector_length) { + vector_length = element->vector_length; + } std::unique_ptr group_node; if (element->__isset.logicalType) { // updated writer with logical type present - group_node = std::unique_ptr( - new GroupNode(element->name, LoadEnumSafe(&element->repetition_type), fields, - LogicalType::FromThrift(element->logicalType), field_id)); + group_node = std::unique_ptr(new GroupNode( + element->name, LoadEnumSafe(&element->repetition_type), fields, + LogicalType::FromThrift(element->logicalType), field_id, vector_length)); } else { group_node = std::unique_ptr(new GroupNode( element->name, LoadEnumSafe(&element->repetition_type), fields, (element->__isset.converted_type ? LoadEnumSafe(&element->converted_type) : ConvertedType::NONE), - field_id)); + field_id, vector_length)); } return std::unique_ptr(group_node.release()); @@ -449,25 +480,30 @@ std::unique_ptr PrimitiveNode::FromParquet(const void* opaque_element) { if (element->__isset.field_id) { field_id = element->field_id; } + int32_t vector_length = -1; + if (element->__isset.vector_length) { + vector_length = element->vector_length; + } std::unique_ptr primitive_node; if (element->__isset.logicalType) { // updated writer with logical type present - primitive_node = std::unique_ptr( - new PrimitiveNode(element->name, LoadEnumSafe(&element->repetition_type), - LogicalType::FromThrift(element->logicalType), - LoadEnumSafe(&element->type), element->type_length, field_id)); - } else if (element->__isset.converted_type) { - // legacy writer with converted type present primitive_node = std::unique_ptr(new PrimitiveNode( element->name, LoadEnumSafe(&element->repetition_type), - LoadEnumSafe(&element->type), LoadEnumSafe(&element->converted_type), - element->type_length, element->precision, element->scale, field_id)); + LogicalType::FromThrift(element->logicalType), LoadEnumSafe(&element->type), + element->type_length, field_id, vector_length)); + } else if (element->__isset.converted_type) { + // legacy writer with converted type present + primitive_node = std::unique_ptr( + new PrimitiveNode(element->name, LoadEnumSafe(&element->repetition_type), + LoadEnumSafe(&element->type), + LoadEnumSafe(&element->converted_type), element->type_length, + element->precision, element->scale, field_id, vector_length)); } else { // logical type not present primitive_node = std::unique_ptr(new PrimitiveNode( element->name, LoadEnumSafe(&element->repetition_type), NoLogicalType::Make(), - LoadEnumSafe(&element->type), element->type_length, field_id)); + LoadEnumSafe(&element->type), element->type_length, field_id, vector_length)); } // Return as unique_ptr to the base type @@ -499,6 +535,9 @@ void GroupNode::ToParquet(void* opaque_element) const { if (field_id_ >= 0) { element->__set_field_id(field_id_); } + if (is_vector()) { + element->__set_vector_length(vector_length_); + } if (logical_type_ && logical_type_->is_serialized()) { element->__set_logicalType(logical_type_->ToThrift()); } @@ -524,6 +563,9 @@ void PrimitiveNode::ToParquet(void* opaque_element) const { if (field_id_ >= 0) { element->__set_field_id(field_id_); } + if (is_vector()) { + element->__set_vector_length(vector_length_); + } if (logical_type_ && logical_type_->is_serialized() && // TODO(tpboudreau): remove the following conjunct to enable serialization // of IntervalTypes after parquet.thrift recognizes them @@ -634,6 +676,9 @@ static void PrintRepLevel(Repetition::type repetition, std::ostream& stream) { case Repetition::REPEATED: stream << "repeated"; break; + case Repetition::VECTOR: + stream << "vector"; + break; default: break; } @@ -710,14 +755,19 @@ struct SchemaPrinter : public Node::ConstVisitor { stream_ << " "; PrintType(node, stream_); stream_ << " field_id=" << node->field_id() << " " << node->name(); + if (node->is_vector()) { + stream_ << " [" << node->vector_length() << "]"; + } PrintConvertedType(node, stream_); stream_ << ";" << std::endl; } void Visit(const GroupNode* node) { PrintRepLevel(node->repetition(), stream_); - stream_ << " group " - << "field_id=" << node->field_id() << " " << node->name(); + stream_ << " group " << "field_id=" << node->field_id() << " " << node->name(); + if (node->is_vector()) { + stream_ << " [" << node->vector_length() << "]"; + } auto lt = node->converted_type(); const auto& la = node->logical_type(); if (la && la->is_valid() && !la->is_none()) { @@ -836,6 +886,9 @@ void SchemaDescriptor::BuildTree(const NodePtr& node, int16_t max_def_level, // between an empty list and a list with an item in it. ++max_rep_level; ++max_def_level; + } else if (node->is_vector()) { + // VECTOR fields repeat a fixed number of times per parent value without + // increasing the maximum definition or repetition level. } // Now, walk the schema and create a ColumnDescriptor for each leaf node @@ -848,8 +901,27 @@ void SchemaDescriptor::BuildTree(const NodePtr& node, int16_t max_def_level, node_to_leaf_index_[static_cast(node.get())] = static_cast(leaves_.size()); + // Determine the product of all VECTOR-repeated ancestors (including this leaf) + // so nested VECTOR shapes such as vector<3, vector<4, int32>> report the total + // number of physical leaf values contributed per parent record. + int32_t effective_vector_length = -1; + for (const Node* cursor = node.get(); cursor != nullptr; cursor = cursor->parent()) { + if (cursor->is_vector()) { + if (effective_vector_length < 0) { + effective_vector_length = cursor->vector_length(); + } else { + if (effective_vector_length > + std::numeric_limits::max() / cursor->vector_length()) { + throw ParquetException("Nested VECTOR effective vector_length overflow"); + } + effective_vector_length *= cursor->vector_length(); + } + } + } + // Primitive node, append to leaves - leaves_.push_back(ColumnDescriptor(node, max_def_level, max_rep_level, this)); + leaves_.push_back(ColumnDescriptor(node, max_def_level, max_rep_level, this, + effective_vector_length)); leaf_to_base_.emplace(static_cast(leaves_.size()) - 1, base); leaf_to_idx_.emplace(node->path()->ToDotString(), static_cast(leaves_.size()) - 1); @@ -866,10 +938,12 @@ int SchemaDescriptor::GetColumnIndex(const PrimitiveNode& node) const { ColumnDescriptor::ColumnDescriptor(schema::NodePtr node, int16_t max_definition_level, int16_t max_repetition_level, - const SchemaDescriptor* schema_descr) + const SchemaDescriptor* schema_descr, + int32_t effective_vector_length) : node_(std::move(node)), max_definition_level_(max_definition_level), - max_repetition_level_(max_repetition_level) { + max_repetition_level_(max_repetition_level), + effective_vector_length_(effective_vector_length) { if (!node_->is_primitive()) { throw ParquetException("Must be a primitive type"); } @@ -879,7 +953,8 @@ ColumnDescriptor::ColumnDescriptor(schema::NodePtr node, int16_t max_definition_ bool ColumnDescriptor::Equals(const ColumnDescriptor& other) const { return primitive_node_->Equals(other.primitive_node_) && max_repetition_level() == other.max_repetition_level() && - max_definition_level() == other.max_definition_level(); + max_definition_level() == other.max_definition_level() && + effective_vector_length() == other.effective_vector_length(); } const ColumnDescriptor* SchemaDescriptor::Column(int i) const { @@ -930,8 +1005,13 @@ std::string ColumnDescriptor::ToString() const { << " physical_type: " << TypeToString(physical_type()) << "," << std::endl << " converted_type: " << ConvertedTypeToString(converted_type()) << "," << std::endl - << " logical_type: " << logical_type()->ToString() << "," << std::endl - << " max_definition_level: " << max_definition_level() << "," << std::endl + << " logical_type: " << logical_type()->ToString() << "," << std::endl; + + if (schema_node()->is_vector()) { + ss << " vector_length: " << schema_node()->vector_length() << "," << std::endl; + } + + ss << " max_definition_level: " << max_definition_level() << "," << std::endl << " max_repetition_level: " << max_repetition_level() << "," << std::endl; if (physical_type() == ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { diff --git a/cpp/src/parquet/schema.h b/cpp/src/parquet/schema.h index 65732603ea1d..6c0d34e90c8d 100644 --- a/cpp/src/parquet/schema.h +++ b/cpp/src/parquet/schema.h @@ -109,6 +109,11 @@ class PARQUET_EXPORT Node { bool is_repeated() const { return repetition_ == Repetition::REPEATED; } + /// \brief True when this node specifically has VECTOR repetition; use + /// ColumnDescriptor::in_vector_column() to ask whether a column is part of + /// a VECTOR subtree. + bool is_vector() const { return repetition_ == Repetition::VECTOR; } + bool is_required() const { return repetition_ == Repetition::REQUIRED; } virtual bool Equals(const Node* other) const = 0; @@ -128,6 +133,10 @@ class PARQUET_EXPORT Node { /// Thrift. int field_id() const { return field_id_; } + /// \brief The fixed number of values per parent when repetition == VECTOR. + /// Returns -1 when this node is not VECTOR-repeated. + int32_t vector_length() const { return vector_length_; } + const Node* parent() const { return parent_; } const std::shared_ptr path() const; @@ -155,21 +164,25 @@ class PARQUET_EXPORT Node { friend class GroupNode; Node(Node::type type, const std::string& name, Repetition::type repetition, - ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1) + ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1, + int32_t vector_length = -1) : type_(type), name_(name), repetition_(repetition), converted_type_(converted_type), field_id_(field_id), + vector_length_(vector_length), parent_(NULLPTR) {} Node(Node::type type, const std::string& name, Repetition::type repetition, - std::shared_ptr logical_type, int field_id = -1) + std::shared_ptr logical_type, int field_id = -1, + int32_t vector_length = -1) : type_(type), name_(name), repetition_(repetition), logical_type_(std::move(logical_type)), field_id_(field_id), + vector_length_(vector_length), parent_(NULLPTR) {} Node::type type_; @@ -178,6 +191,7 @@ class PARQUET_EXPORT Node { ConvertedType::type converted_type_{ConvertedType::NONE}; std::shared_ptr logical_type_; int field_id_; + int32_t vector_length_; // Nodes should not be shared, they have a single parent. const Node* parent_; @@ -205,9 +219,9 @@ class PARQUET_EXPORT PrimitiveNode : public Node { Type::type type, ConvertedType::type converted_type = ConvertedType::NONE, int length = -1, int precision = -1, int scale = -1, - int field_id = -1) { + int field_id = -1, int32_t vector_length = -1) { return NodePtr(new PrimitiveNode(name, repetition, type, converted_type, length, - precision, scale, field_id)); + precision, scale, field_id, vector_length)); } // If no logical type, pass LogicalType::None() or nullptr @@ -215,9 +229,10 @@ class PARQUET_EXPORT PrimitiveNode : public Node { static inline NodePtr Make(const std::string& name, Repetition::type repetition, std::shared_ptr logical_type, Type::type primitive_type, int primitive_length = -1, - int field_id = -1) { + int field_id = -1, int32_t vector_length = -1) { return NodePtr(new PrimitiveNode(name, repetition, std::move(logical_type), - primitive_type, primitive_length, field_id)); + primitive_type, primitive_length, field_id, + vector_length)); } bool Equals(const Node* other) const override; @@ -239,11 +254,13 @@ class PARQUET_EXPORT PrimitiveNode : public Node { private: PrimitiveNode(const std::string& name, Repetition::type repetition, Type::type type, ConvertedType::type converted_type = ConvertedType::NONE, int length = -1, - int precision = -1, int scale = -1, int field_id = -1); + int precision = -1, int scale = -1, int field_id = -1, + int32_t vector_length = -1); PrimitiveNode(const std::string& name, Repetition::type repetition, std::shared_ptr logical_type, - Type::type primitive_type, int primitive_length = -1, int field_id = -1); + Type::type primitive_type, int primitive_length = -1, int field_id = -1, + int32_t vector_length = -1); Type::type physical_type_; int32_t type_length_; @@ -270,8 +287,9 @@ class PARQUET_EXPORT GroupNode : public Node { static inline NodePtr Make(const std::string& name, Repetition::type repetition, const NodeVector& fields, ConvertedType::type converted_type = ConvertedType::NONE, - int field_id = -1) { - return NodePtr(new GroupNode(name, repetition, fields, converted_type, field_id)); + int field_id = -1, int32_t vector_length = -1) { + return NodePtr( + new GroupNode(name, repetition, fields, converted_type, field_id, vector_length)); } // If no logical type, pass nullptr @@ -279,9 +297,9 @@ class PARQUET_EXPORT GroupNode : public Node { static inline NodePtr Make(const std::string& name, Repetition::type repetition, const NodeVector& fields, std::shared_ptr logical_type, - int field_id = -1) { - return NodePtr( - new GroupNode(name, repetition, fields, std::move(logical_type), field_id)); + int field_id = -1, int32_t vector_length = -1) { + return NodePtr(new GroupNode(name, repetition, fields, std::move(logical_type), + field_id, vector_length)); } bool Equals(const Node* other) const override; @@ -307,11 +325,12 @@ class PARQUET_EXPORT GroupNode : public Node { private: GroupNode(const std::string& name, Repetition::type repetition, const NodeVector& fields, - ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1); + ConvertedType::type converted_type = ConvertedType::NONE, int field_id = -1, + int32_t vector_length = -1); GroupNode(const std::string& name, Repetition::type repetition, const NodeVector& fields, std::shared_ptr logical_type, - int field_id = -1); + int field_id = -1, int32_t vector_length = -1); NodeVector fields_; bool EqualsInternal(const GroupNode* other) const; @@ -358,7 +377,8 @@ class PARQUET_EXPORT ColumnDescriptor { public: ColumnDescriptor(schema::NodePtr node, int16_t max_definition_level, int16_t max_repetition_level, - const SchemaDescriptor* schema_descr = NULLPTR); + const SchemaDescriptor* schema_descr = NULLPTR, + int32_t effective_vector_length = -1); bool Equals(const ColumnDescriptor& other) const; @@ -366,6 +386,16 @@ class PARQUET_EXPORT ColumnDescriptor { int16_t max_repetition_level() const { return max_repetition_level_; } + /// \brief The fixed number of leaf values per parent record contributed by + /// this column, computed as the product of all VECTOR-repeated ancestors + /// (including the leaf itself if it is VECTOR-repeated). Returns -1 when the + /// column is not part of a VECTOR-repeated subtree. + int32_t effective_vector_length() const { return effective_vector_length_; } + + /// \brief True when this column belongs to a VECTOR-repeated subtree + /// (the leaf or some ancestor has Repetition::VECTOR). + bool in_vector_column() const { return effective_vector_length_ > 0; } + Type::type physical_type() const { return primitive_node_->physical_type(); } ConvertedType::type converted_type() const { return primitive_node_->converted_type(); } @@ -418,6 +448,7 @@ class PARQUET_EXPORT ColumnDescriptor { int16_t max_definition_level_; int16_t max_repetition_level_; + int32_t effective_vector_length_; }; // Container for the converted Parquet schema with a computed information from diff --git a/cpp/src/parquet/schema_test.cc b/cpp/src/parquet/schema_test.cc index 859f14a34d91..17455e7e4e09 100644 --- a/cpp/src/parquet/schema_test.cc +++ b/cpp/src/parquet/schema_test.cc @@ -43,7 +43,8 @@ namespace schema { static inline SchemaElement NewPrimitive(const std::string& name, FieldRepetitionType::type repetition, - Type::type type, int field_id = -1) { + Type::type type, int field_id = -1, + int32_t vector_length = -1) { SchemaElement result; result.__set_name(name); result.__set_repetition_type(repetition); @@ -51,12 +52,16 @@ static inline SchemaElement NewPrimitive(const std::string& name, if (field_id >= 0) { result.__set_field_id(field_id); } + if (vector_length >= 0) { + result.__set_vector_length(vector_length); + } return result; } static inline SchemaElement NewGroup(const std::string& name, FieldRepetitionType::type repetition, - int num_children, int field_id = -1) { + int num_children, int field_id = -1, + int32_t vector_length = -1) { SchemaElement result; result.__set_name(name); result.__set_repetition_type(repetition); @@ -65,6 +70,9 @@ static inline SchemaElement NewGroup(const std::string& name, if (field_id >= 0) { result.__set_field_id(field_id); } + if (vector_length >= 0) { + result.__set_vector_length(vector_length); + } return result; } @@ -217,6 +225,24 @@ TEST_F(TestPrimitiveNode, FromParquet) { ASSERT_EQ(12, prim_node_->decimal_metadata().precision); } +TEST_F(TestPrimitiveNode, VectorFromParquetRejected) { + SchemaElement elt = + NewPrimitive(name_, FieldRepetitionType::VECTOR, Type::FLOAT, field_id_, 8); + + ASSERT_THROW(Convert(&elt), ParquetException); +} + +TEST_F(TestPrimitiveNode, VectorValidation) { + ASSERT_THROW(PrimitiveNode::Make("vec", Repetition::VECTOR, Type::FLOAT), + ParquetException); + ASSERT_THROW(PrimitiveNode::Make("scalar", Repetition::REQUIRED, Type::FLOAT, + ConvertedType::NONE, -1, -1, -1, -1, 4), + ParquetException); + ASSERT_THROW(PrimitiveNode::Make("empty", Repetition::VECTOR, Type::FLOAT, + ConvertedType::NONE, -1, -1, -1, -1, 0), + ParquetException); +} + TEST_F(TestPrimitiveNode, Equals) { PrimitiveNode node1("foo", Repetition::REQUIRED, Type::INT32); PrimitiveNode node2("foo", Repetition::REQUIRED, Type::INT64); @@ -365,6 +391,59 @@ TEST_F(TestGroupNode, Attrs) { ASSERT_EQ(ConvertedType::LIST, node2.converted_type()); } +TEST_F(TestGroupNode, VectorAttrs) { + auto node = GroupNode::Make("vec", Repetition::VECTOR, Fields1(), ConvertedType::NONE, + /*field_id=*/-1, + /*vector_length=*/4); + + ASSERT_TRUE(node->is_vector()); + ASSERT_EQ(Repetition::VECTOR, node->repetition()); + ASSERT_EQ(4, node->vector_length()); +} + +TEST_F(TestGroupNode, VectorValidation) { + ASSERT_THROW(GroupNode::Make("vec", Repetition::VECTOR, Fields1()), ParquetException); + ASSERT_THROW(GroupNode::Make("empty", Repetition::VECTOR, Fields1(), + ConvertedType::NONE, /*field_id=*/-1, + /*vector_length=*/0), + ParquetException); + ASSERT_THROW(GroupNode::Make("scalar", Repetition::REQUIRED, Fields1(), + ConvertedType::NONE, /*field_id=*/-1, + /*vector_length=*/4), + ParquetException); +} + +TEST_F(TestGroupNode, VectorLogicalTypeRoundTrip) { + // A canonical 3-level vector group annotated with the Vector logical type + // round-trips through the Thrift intermediary. + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::FLOAT); + auto list = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, /*field_id=*/-1, + /*vector_length=*/3); + auto group = + GroupNode::Make("embedding", Repetition::OPTIONAL, {list}, LogicalType::Vector()); + + ASSERT_TRUE(group->logical_type()->is_vector()); + ASSERT_TRUE(group->logical_type()->is_nested()); + ASSERT_EQ(ConvertedType::NONE, group->converted_type()); + + format::SchemaElement group_element; + group->ToParquet(&group_element); + ASSERT_TRUE(group_element.__isset.logicalType); + ASSERT_TRUE(group_element.logicalType.__isset.VECTOR); + + format::SchemaElement list_element; + list->ToParquet(&list_element); + format::SchemaElement element_element; + element->ToParquet(&element_element); + std::unique_ptr roundtripped = GroupNode::FromParquet( + &group_element, + {GroupNode::FromParquet(&list_element, + {PrimitiveNode::FromParquet(&element_element)})}); + ASSERT_TRUE(roundtripped->logical_type()->is_vector()); + ASSERT_TRUE(group->Equals(roundtripped.get())); +} + TEST_F(TestGroupNode, Equals) { NodeVector f1 = Fields1(); NodeVector f2 = Fields1(); @@ -825,6 +904,56 @@ TEST_F(TestSchemaDescriptor, BuildTree) { ASSERT_EQ(nleaves, descr_.num_columns()); } +TEST_F(TestSchemaDescriptor, BuildTreeVector) { + NodePtr element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::FLOAT); + NodePtr vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/3); + NodePtr embedding = GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}); + + descr_.Init(GroupNode::Make("schema", Repetition::REPEATED, {embedding})); + + ASSERT_EQ(1, descr_.num_columns()); + const ColumnDescriptor* col = descr_.Column(0); + EXPECT_EQ(1, col->max_definition_level()); + EXPECT_EQ(0, col->max_repetition_level()); + EXPECT_FALSE(col->schema_node()->is_vector()); + EXPECT_EQ(3, col->effective_vector_length()); +} + +TEST_F(TestSchemaDescriptor, BuildTreeNestedVector) { + NodePtr element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::FLOAT); + NodePtr inner = GroupNode::Make("inner", Repetition::VECTOR, {element}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/4); + NodePtr outer = GroupNode::Make("outer", Repetition::VECTOR, {inner}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/3); + NodePtr embedding = GroupNode::Make("embedding", Repetition::OPTIONAL, {outer}); + + descr_.Init(GroupNode::Make("schema", Repetition::REPEATED, {embedding})); + + ASSERT_EQ(1, descr_.num_columns()); + const ColumnDescriptor* col = descr_.Column(0); + EXPECT_EQ(1, col->max_definition_level()); + EXPECT_EQ(0, col->max_repetition_level()); + EXPECT_EQ(12, col->effective_vector_length()); +} + +TEST_F(TestSchemaDescriptor, BuildTreeNestedVectorLengthOverflow) { + NodePtr element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::FLOAT); + NodePtr inner = GroupNode::Make("inner", Repetition::VECTOR, {element}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/46341); + NodePtr outer = GroupNode::Make("outer", Repetition::VECTOR, {inner}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/46341); + NodePtr embedding = GroupNode::Make("embedding", Repetition::OPTIONAL, {outer}); + + ASSERT_THROW(descr_.Init(GroupNode::Make("schema", Repetition::REPEATED, {embedding})), + ParquetException); +} + TEST_F(TestSchemaDescriptor, HasRepeatedFields) { NodeVector fields; NodePtr schema; @@ -912,6 +1041,28 @@ TEST(TestSchemaPrinter, Examples) { ASSERT_EQ(expected, result); } +TEST(TestSchemaPrinter, Vector) { + NodePtr schema = GroupNode::Make( + "schema", Repetition::REPEATED, + {GroupNode::Make( + "embedding", Repetition::OPTIONAL, + {GroupNode::Make( + "vector", Repetition::VECTOR, + {PrimitiveNode::Make("element", Repetition::REQUIRED, Type::FLOAT)}, + /*logical_type=*/nullptr, -1, 3)}, + LogicalType::Vector())}); + + std::string expected = R"(repeated group field_id=-1 schema { + optional group field_id=-1 embedding (Vector) { + vector group field_id=-1 vector [3] { + required float field_id=-1 element; + } + } +} +)"; + ASSERT_EQ(expected, Print(schema)); +} + static void ConfirmFactoryEquivalence( ConvertedType::type converted_type, const std::shared_ptr& from_make, From 76287b55f10e346f51dcd633099c15bc3466b2aa Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Thu, 11 Jun 2026 12:44:22 +0200 Subject: [PATCH 3/8] Write and read Arrow FixedSizeList as Parquet VECTOR Opt-in via ArrowWriterProperties::enable_experimental_vector_encoding. The writer emits the canonical 3-level VECTOR structure for supported FixedSizeList fields and falls back to the standard LIST encoding for zero-length lists, variable-width, dictionary or extension elements, struct elements containing fixed-size lists, and fields below nullable groups. Schema conversion and write-path level generation share their gating predicates so they cannot disagree about the fallback. The reader requires the Vector annotation, accepts more than the writer produces (including vectors below nullable struct ancestors in foreign files), collapses definition levels in runs, and reconstructs FixedSizeList without a stored Arrow schema. Benchmarks (fixed_size_list, 10M values): dense reads 29.7 GiB/s vs 2.2 GiB/s with LIST encoding, writes 2.2 GiB/s vs 0.8 GiB/s; nullable vectors read at 4.5 GiB/s and have no LIST baseline because writing null FixedSizeList rows through the LIST encoding is not implemented upstream. --- .../parquet/arrow/arrow_reader_writer_test.cc | 1237 ++++++++++++++++- cpp/src/parquet/arrow/arrow_schema_test.cc | 439 +++++- cpp/src/parquet/arrow/path_internal.cc | 348 ++++- cpp/src/parquet/arrow/path_internal.h | 29 +- cpp/src/parquet/arrow/path_internal_test.cc | 82 ++ cpp/src/parquet/arrow/reader.cc | 392 +++++- .../parquet/arrow/reader_writer_benchmark.cc | 79 ++ cpp/src/parquet/arrow/schema.cc | 259 +++- cpp/src/parquet/arrow/schema.h | 8 + cpp/src/parquet/arrow/schema_internal.cc | 75 + cpp/src/parquet/arrow/schema_internal.h | 4 + cpp/src/parquet/arrow/writer.cc | 129 +- cpp/src/parquet/properties.h | 22 +- 13 files changed, 3047 insertions(+), 56 deletions(-) diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc index 8735aea731ce..f4ba0571444c 100644 --- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc +++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc @@ -24,10 +24,13 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#include #include #include #include #include +#include +#include #include #include #include @@ -66,6 +69,8 @@ #include "parquet/api/reader.h" #include "parquet/api/writer.h" +#include "parquet/bloom_filter.h" +#include "parquet/bloom_filter_reader.h" #include "parquet/arrow/fuzz_internal.h" #include "parquet/arrow/reader.h" @@ -75,6 +80,7 @@ #include "parquet/arrow/writer.h" #include "parquet/column_writer.h" #include "parquet/file_writer.h" +#include "parquet/page_index.h" #include "parquet/properties.h" #include "parquet/test_util.h" #include "parquet/types.h" @@ -423,6 +429,19 @@ void WriteTableToBuffer(const std::shared_ptr& table, int64_t row_group_s *out, WriteTableToBuffer(table, row_group_size, write_props, arrow_properties)); } +std::shared_ptr VectorWriterProperties() { + auto builder = WriterProperties::Builder(); + return builder.enable_write_page_index()->encoding(Encoding::PLAIN)->build(); +} + +std::shared_ptr VectorByteStreamSplitWriterProperties() { + auto builder = WriterProperties::Builder(); + return builder.disable_dictionary() + ->enable_write_page_index() + ->encoding(Encoding::BYTE_STREAM_SPLIT) + ->build(); +} + void DoRoundtrip(const std::shared_ptr
& table, int64_t row_group_size, std::shared_ptr
* out, const std::shared_ptr<::parquet::WriterProperties>& writer_properties = @@ -444,6 +463,62 @@ void DoRoundtrip(const std::shared_ptr
& table, int64_t row_group_size, ASSERT_OK_AND_ASSIGN(*out, reader->ReadTable()); } +std::shared_ptr<::arrow::DataType> MakeVectorFixedSizeListType( + const std::shared_ptr<::arrow::DataType>& item_type, bool element_nullable = false) { + return ::arrow::fixed_size_list(::arrow::field("item", item_type, element_nullable), + /*size=*/3); +} + +std::shared_ptr<::arrow::DataType> VectorFixedSizeListType( + bool element_nullable = false) { + return MakeVectorFixedSizeListType(::arrow::int16(), element_nullable); +} + +std::shared_ptr<::arrow::DataType> VectorFloatFixedSizeListType( + bool element_nullable = false) { + return MakeVectorFixedSizeListType(::arrow::float32(), element_nullable); +} + +std::shared_ptr
MakeVectorFixedSizeListTable( + const std::vector>& chunks, bool nullable = true, + bool element_nullable = false) { + auto type = VectorFixedSizeListType(element_nullable); + auto field = ::arrow::field("root", type, nullable); + auto column = std::make_shared(chunks, type); + return ::arrow::Table::Make(::arrow::schema({field}), {column}); +} + +std::shared_ptr
MakeVectorFixedSizeListTable(std::string_view json, + bool nullable = true, + bool element_nullable = false) { + auto type = VectorFixedSizeListType(element_nullable); + return MakeVectorFixedSizeListTable({::arrow::ArrayFromJSON(type, std::string(json))}, + nullable, element_nullable); +} + +std::shared_ptr
MakeVectorFixedSizeListTable( + const std::shared_ptr<::arrow::DataType>& item_type, std::string_view json, + bool nullable = true, bool element_nullable = false) { + auto type = MakeVectorFixedSizeListType(item_type, element_nullable); + auto field = ::arrow::field("root", type, nullable); + auto array = ::arrow::ArrayFromJSON(type, std::string(json)); + return ::arrow::Table::Make(::arrow::schema({field}), {array}); +} + +void CheckVectorFixedSizeListRoundtrip( + const std::shared_ptr
& table, int64_t row_group_size, + const ArrowReaderProperties& arrow_reader_properties = + default_arrow_reader_properties()) { + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE(DoRoundtrip(table, row_group_size, &result, + VectorWriterProperties(), builder.build(), + arrow_reader_properties)); + ::arrow::AssertSchemaEqual(*table->schema(), *result->schema(), false); + ::arrow::AssertTablesEqual(*table, *result, false); +} + void CheckConfiguredRoundtrip( const std::shared_ptr
& input_table, const std::shared_ptr
& expected_table = nullptr, @@ -553,6 +628,18 @@ void CheckSimpleRoundtrip(const std::shared_ptr
& table, int64_t row_group ::arrow::AssertTablesEqual(*table, *result, false); } +// Roundtrip for tables whose FixedSizeList columns fall back to the standard +// LIST encoding even though vector encoding is enabled. store_schema() is +// needed to restore the FixedSizeList type from the LIST encoding on read. +void CheckVectorFallsBackToListRoundtrip(const std::shared_ptr
& table, + int64_t row_group_size) { + auto arrow_writer_properties = ArrowWriterProperties::Builder() + .store_schema() + ->enable_experimental_vector_encoding() + ->build(); + CheckSimpleRoundtrip(table, row_group_size, arrow_writer_properties); +} + static std::shared_ptr MakeSimpleSchema(const DataType& type, Repetition::type repetition) { int32_t byte_width = -1; @@ -3396,22 +3483,987 @@ TEST(ArrowReadWrite, EmptyListView) { } TEST(ArrowReadWrite, FixedSizeList) { - using ::arrow::field; - using ::arrow::fixed_size_list; - using ::arrow::struct_; - - auto type = fixed_size_list(::arrow::int16(), /*size=*/3); - - const char* json = R"([ + auto table = MakeVectorFixedSizeListTable(R"([ [1, 2, 3], [4, 5, 6], - [7, 8, 9]])"; - auto array = ::arrow::ArrayFromJSON(type, json); - auto table = ::arrow::Table::Make(::arrow::schema({field("root", type)}), {array}); + [7, 8, 9]])", + /*nullable=*/true, + /*element_nullable=*/true); auto props_store_schema = ArrowWriterProperties::Builder().store_schema()->build(); CheckSimpleRoundtrip(table, 2, props_store_schema); } +TEST(ArrowWriteOnly, FixedSizeListVectorSchemaRequired) { + auto table = MakeVectorFixedSizeListTable(R"([ + [1, 2, 3], + [4, 5, 6], + [7, 8, 9]])", + /*nullable=*/false); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, + VectorWriterProperties(), builder.build())); + + auto reader = ParquetFileReader::Open(std::make_shared(buffer)); + const auto* schema = reader->metadata()->schema(); + const auto* root = schema->group_node()->field(0).get(); + ASSERT_TRUE(root->is_group()); + const auto* root_group = static_cast(root); + ASSERT_EQ(root_group->repetition(), Repetition::REQUIRED); + ASSERT_TRUE(root_group->logical_type() != nullptr && + root_group->logical_type()->is_vector()); + ASSERT_EQ(root_group->field_count(), 1); + const auto* vector_node = root_group->field(0).get(); + ASSERT_TRUE(vector_node->is_vector()); + ASSERT_TRUE(vector_node->is_group()); + ASSERT_EQ(vector_node->vector_length(), 3); + const auto* vector_group = static_cast(vector_node); + ASSERT_EQ(vector_group->field_count(), 1); + ASSERT_TRUE(vector_group->field(0)->is_primitive()); + ASSERT_TRUE(vector_group->field(0)->is_required()); + ASSERT_EQ(reader->metadata()->RowGroup(0)->ColumnChunk(0)->num_values(), 6); + ASSERT_EQ(reader->metadata()->RowGroup(1)->ColumnChunk(0)->num_values(), 3); +} + +TEST(ArrowWriteOnly, FixedSizeListVectorSchemaNullable) { + auto table = MakeVectorFixedSizeListTable(R"([ + [1, 2, 3], + null, + [7, 8, 9]])"); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/3, + VectorWriterProperties(), builder.build())); + + auto reader = ParquetFileReader::Open(std::make_shared(buffer)); + const auto* schema = reader->metadata()->schema(); + const auto* root = schema->group_node()->field(0).get(); + ASSERT_TRUE(root->is_group()); + const auto* root_group = static_cast(root); + ASSERT_EQ(root_group->repetition(), Repetition::OPTIONAL); + ASSERT_TRUE(root_group->logical_type() != nullptr && + root_group->logical_type()->is_vector()); + ASSERT_EQ(root_group->field_count(), 1); + const auto* vector_node = root_group->field(0).get(); + ASSERT_TRUE(vector_node->is_vector()); + ASSERT_TRUE(vector_node->is_group()); + ASSERT_EQ(vector_node->vector_length(), 3); + const auto* vector_group = static_cast(vector_node); + ASSERT_EQ(vector_group->field_count(), 1); + ASSERT_TRUE(vector_group->field(0)->is_primitive()); + ASSERT_TRUE(vector_group->field(0)->is_required()); + // Option B uses fixed-stride nullable parent layout: every parent row, + // including a null vector row, contributes vector_length child slots. + ASSERT_EQ(reader->metadata()->RowGroup(0)->ColumnChunk(0)->num_values(), 9); +} + +TEST(ArrowWriteOnly, FixedSizeListVectorNullRowsEmitFixedWidthNullSlots) { + auto table = MakeVectorFixedSizeListTable(R"([ + null, + null, + null])"); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/3, + VectorWriterProperties(), builder.build())); + + auto reader = ParquetFileReader::Open(std::make_shared(buffer)); + ASSERT_EQ(reader->metadata()->RowGroup(0)->num_rows(), 3); + ASSERT_EQ(reader->metadata()->RowGroup(0)->ColumnChunk(0)->num_values(), 9); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableRoundTripNullPatterns) { + const std::vector cases = { + R"([[1, 2, 3], [4, 5, 6], [7, 8, 9]])", + R"([[1, 2, 3], null, [7, 8, 9]])", + R"([null, [4, 5, 6], [7, 8, 9]])", + R"([[1, 2, 3], [4, 5, 6], null])", + R"([[1, 2, 3], null, null, [10, 11, 12], null])", + R"([null, null, null])", + R"([])"}; + + for (const auto& json : cases) { + SCOPED_TRACE(json); + auto table = MakeVectorFixedSizeListTable(json); + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip( + table, std::max(1, table->num_rows()))); + } +} + +TEST(ArrowReadWrite, FixedSizeListVectorStructOfVectorsRoundTrip) { + auto ids_type = ::arrow::fixed_size_list( + ::arrow::field("value", ::arrow::int32(), /*nullable=*/true), 2); + auto scores_type = ::arrow::fixed_size_list(::arrow::float32(), 3); + auto type = ::arrow::struct_({::arrow::field("ids", ids_type, /*nullable=*/true), + ::arrow::field("scores", scores_type, + /*nullable=*/false)}); + auto array = ::arrow::ArrayFromJSON(type, R"([ + {"ids": [10, null], "scores": [0.1, 0.2, 0.3]}, + {"ids": null, "scores": [1.1, 1.2, 1.3]}, + {"ids": [30, 40], "scores": [2.1, 2.2, 2.3]}])"); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("root", type, /*nullable=*/false)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableStructAboveVectorFallsBackToList) { + // The writer encodes vectors below nullable groups as LIST so it does not + // have to expand one null-struct definition level to vector_length leaf slots. + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto struct_type = + ::arrow::struct_({::arrow::field("v", vector_type, /*nullable=*/false)}); + // No null struct rows: writing those through the LIST encoding is an + // upstream NotImplemented ("Lists with non-zero length null components") + // independent of VECTOR. + auto array = ::arrow::ArrayFromJSON(struct_type, R"([ + {"v": [1, 2]}, + {"v": [3, 4]}, + {"v": [5, 6]}])"); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("root", struct_type, /*nullable=*/true)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFallsBackToListRoundtrip(table, + /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableStructUnderListFallsBackToList) { + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::float32(), /*nullable=*/false), 3); + auto struct_type = + ::arrow::struct_({::arrow::field("embedding", vector_type, /*nullable=*/false)}); + auto type = ::arrow::list(::arrow::field("item", struct_type, /*nullable=*/true)); + // No null struct items: writing those through the LIST encoding is an + // upstream NotImplemented ("Lists with non-zero length null components") + // independent of VECTOR. + auto array = ::arrow::ArrayFromJSON(type, R"([ + [{"embedding": [1.0, 2.0, 3.0]}, {"embedding": [4.0, 5.0, 6.0]}], + [{"embedding": [7.0, 8.0, 9.0]}]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFallsBackToListRoundtrip(table, + /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorStructWithListFallsBackToListRoundTrip) { + // The struct element contains a variable-size list, so the column cannot use + // VECTOR encoding and must fall back to the standard LIST encoding. The + // schema conversion and the write-path level generation must agree on that + // fallback, otherwise the writer emits VECTOR-style levels for a LIST + // schema. + auto items_type = ::arrow::list(::arrow::field("item", ::arrow::int32(), + /*nullable=*/false)); + auto struct_type = + ::arrow::struct_({::arrow::field("items", items_type, /*nullable=*/false)}); + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", struct_type, /*nullable=*/false), 2); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [{"items": [1, 2]}, {"items": [3]}], + [{"items": []}, {"items": [4, 5, 6]}]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFallsBackToListRoundtrip(table, + /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorFloat16RoundTrip) { + // The element's logical type (FLOAT16 on FIXED_LEN_BYTE_ARRAY) survives the + // round trip without a stored Arrow schema. + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::float16(), /*nullable=*/false), 3); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [1.5, 2.5, 3.5], + null, + [4.5, 5.5, 6.5]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorDecimalRoundTrip) { + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::decimal128(10, 2), /*nullable=*/false), 2); + auto array = ::arrow::ArrayFromJSON(type, R"([ + ["123.45", "-67.89"], + null, + ["0.01", "99999999.99"]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorRandomNullPatternsRoundTrip) { + // Randomized null patterns across several row groups, read both as a table + // and through the record batch reader with a small batch size. + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 3); + auto type = ::arrow::list(::arrow::field("item", vector_type, /*nullable=*/true)); + std::stringstream json; + json << "["; + unsigned seed = 42; + auto rnd = [&seed]() { + seed = seed * 1103515245 + 12345; + return (seed >> 16) % 8; + }; + int value = 0; + for (int row = 0; row < 200; ++row) { + if (row != 0) json << ","; + unsigned r = rnd(); + if (r == 0) { + json << "null"; + } else if (r == 1) { + json << "[]"; + } else { + json << "["; + int items = r - 1; + for (int i = 0; i < items; ++i) { + if (i != 0) json << ","; + if (rnd() == 2) { + json << "null"; + } else { + json << "[" << value << "," << (value + 1) << "," << (value + 2) << "]"; + value += 3; + } + } + json << "]"; + } + } + json << "]"; + auto array = ::arrow::ArrayFromJSON(type, json.str()); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE( + CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/17)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/17, + VectorWriterProperties(), builder.build())); + ArrowReaderProperties props; + props.set_batch_size(7); + std::unique_ptr reader; + FileReaderBuilder reader_builder; + ASSERT_OK_NO_THROW(reader_builder.Open(std::make_shared(buffer))); + ASSERT_OK(reader_builder.properties(props)->Build(&reader)); + std::unique_ptr<::arrow::RecordBatchReader> batch_reader; + ASSERT_OK_AND_ASSIGN(batch_reader, reader->GetRecordBatchReader()); + ASSERT_OK_AND_ASSIGN(auto result, Table::FromRecordBatchReader(batch_reader.get())); + ::arrow::AssertTablesEqual(*table, *result->CombineChunks().ValueOrDie(), false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNestedRandomNullPatternsRoundTrip) { + auto inner_type = ::arrow::fixed_size_list( + ::arrow::field("value", ::arrow::int32(), /*nullable=*/false), 2); + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", inner_type, /*nullable=*/false), 2); + auto type = ::arrow::list(::arrow::field("item", vector_type, /*nullable=*/true)); + std::stringstream json; + json << "["; + unsigned seed = 1234; + auto rnd = [&seed]() { + seed = seed * 1103515245 + 12345; + return (seed >> 16) % 8; + }; + int value = 0; + for (int row = 0; row < 150; ++row) { + if (row != 0) json << ","; + unsigned r = rnd(); + if (r == 0) { + json << "null"; + } else if (r == 1) { + json << "[]"; + } else { + json << "["; + int items = r - 1; + for (int i = 0; i < items; ++i) { + if (i != 0) json << ","; + if (rnd() == 2) { + json << "null"; + } else { + json << "[[" << value << "," << (value + 1) << "],[" << (value + 2) << "," + << (value + 3) << "]]"; + value += 4; + } + } + json << "]"; + } + } + json << "]"; + auto array = ::arrow::ArrayFromJSON(type, json.str()); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE( + CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/13)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/13, + VectorWriterProperties(), builder.build())); + ArrowReaderProperties props; + props.set_batch_size(5); + std::unique_ptr reader; + FileReaderBuilder reader_builder; + ASSERT_OK_NO_THROW(reader_builder.Open(std::make_shared(buffer))); + ASSERT_OK(reader_builder.properties(props)->Build(&reader)); + std::unique_ptr<::arrow::RecordBatchReader> batch_reader; + ASSERT_OK_AND_ASSIGN(batch_reader, reader->GetRecordBatchReader()); + ASSERT_OK_AND_ASSIGN(auto result, Table::FromRecordBatchReader(batch_reader.get())); + ::arrow::AssertTablesEqual(*table, *result->CombineChunks().ValueOrDie(), false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorCdcAndByteStreamSplitUnderList) { + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::float32(), /*nullable=*/false), 3); + auto type = ::arrow::list(::arrow::field("item", vector_type, /*nullable=*/false)); + std::stringstream json; + json << "["; + for (int row = 0; row < 50; ++row) { + if (row != 0) json << ","; + if (row % 7 == 0) { + json << "[]"; + } else { + json << "[[" << row << ".0," << row << ".5," << row << ".25]]"; + } + } + json << "]"; + auto array = ::arrow::ArrayFromJSON(type, json.str()); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + auto cdc_props = WriterProperties::Builder() + .enable_content_defined_chunking() + ->enable_write_page_index() + ->build(); + ArrowWriterProperties::Builder cdc_arrow_builder; + cdc_arrow_builder.enable_experimental_vector_encoding(); + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE( + DoRoundtrip(table, 13, &result, cdc_props, cdc_arrow_builder.build())); + ::arrow::AssertTablesEqual(*table, *result, false); + + auto bss_props = WriterProperties::Builder() + .disable_dictionary() + ->encoding(Encoding::BYTE_STREAM_SPLIT) + ->build(); + ArrowWriterProperties::Builder bss_arrow_builder; + bss_arrow_builder.enable_experimental_vector_encoding(); + ASSERT_NO_FATAL_FAILURE( + DoRoundtrip(table, 13, &result, bss_props, bss_arrow_builder.build())); + ::arrow::AssertTablesEqual(*table, *result, false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorListOfNullableVectorsRoundTrip) { + // Null and present vector items interleaved within and across lists. + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto type = ::arrow::list(::arrow::field("item", vector_type, /*nullable=*/true)); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [[1, 2], null, [3, 4]], + [null], + [], + null, + [[5, 6], [7, 8], null, [9, 10]]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, + /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNestedVectorsUnderListRoundTrip) { + auto inner = ::arrow::fixed_size_list( + ::arrow::field("value", ::arrow::int32(), /*nullable=*/false), 2); + auto outer = + ::arrow::fixed_size_list(::arrow::field("element", inner, /*nullable=*/false), 2); + auto type = ::arrow::list(::arrow::field("item", outer, /*nullable=*/true)); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [[[1, 2], [3, 4]], null], + [], + null, + [[[5, 6], [7, 8]]]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, + /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorListOfVectorsRoundTrip) { + // FixedSizeList values below a repeated field can use VECTOR encoding; + // empty and null list entries must roundtrip. + auto vector_type = ::arrow::fixed_size_list(::arrow::int32(), 2); + auto type = ::arrow::list(::arrow::field("item", vector_type, /*nullable=*/false)); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [[1, 2], [3, 4]], + [], + null, + [[5, 6]], + [[7, 8], [9, 10], [11, 12]]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, + /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorListOfStructsWithVectorsRoundTrip) { + auto vector_type = ::arrow::fixed_size_list(::arrow::float32(), 3); + auto struct_type = + ::arrow::struct_({::arrow::field("embedding", vector_type, /*nullable=*/false)}); + auto type = ::arrow::list(::arrow::field("item", struct_type, /*nullable=*/false)); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [{"embedding": [1.0, 2.0, 3.0]}, {"embedding": [4.0, 5.0, 6.0]}], + [], + [{"embedding": [7.0, 8.0, 9.0]}], + [{"embedding": [10.0, 11.0, 12.0]}, {"embedding": [13.0, 14.0, 15.0]}]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, + /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorMapOfVectorsRoundTrip) { + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto type = ::arrow::map(::arrow::utf8(), ::arrow::field("value", vector_type, + /*nullable=*/false)); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [["a", [1, 2]], ["b", [3, 4]]], + [], + [["c", [5, 6]]]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, + /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorStructWithNestedVectorFallsBackToList) { + // A fixed-size list nested inside a struct vector element would give the + // struct's leaves different per-row multiplicities, which the write path + // does not support; the column falls back to the LIST encoding. + auto inner = ::arrow::fixed_size_list( + ::arrow::field("value", ::arrow::int32(), /*nullable=*/false), 2); + auto struct_type = + ::arrow::struct_({::arrow::field("v", inner, /*nullable=*/false), + ::arrow::field("x", ::arrow::int32(), /*nullable=*/false)}); + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", struct_type, /*nullable=*/false), 3); + // No null rows: writing null FixedSizeList rows with nested children via + // the LIST encoding is an upstream NotImplemented independent of VECTOR. + auto array = ::arrow::ArrayFromJSON(type, R"([ + [{"v": [1, 2], "x": 1}, {"v": [3, 4], "x": 2}, {"v": [5, 6], "x": 3}], + [{"v": [7, 8], "x": 4}, {"v": [9, 10], "x": 5}, {"v": [11, 12], "x": 6}]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFallsBackToListRoundtrip(table, + /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListZeroLength) { + // Regression test: zero-length fixed-size lists used to be written with the + // definition level of a present element instead of an empty list, producing + // one garbage element per row on read. + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 0); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [], + null, + []])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + auto props_store_schema = ArrowWriterProperties::Builder().store_schema()->build(); + CheckSimpleRoundtrip(table, 3, props_store_schema); +} + +TEST(ArrowReadWrite, FixedSizeListVectorZeroLengthFallsBackToList) { + // A zero-length vector value would contribute no physical leaf slots, so + // neither row counts nor row nullability are representable in the VECTOR + // encoding; zero-length fixed-size lists use the LIST encoding instead. + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 0); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [], + null, + []])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFallsBackToListRoundtrip(table, + /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorBoolRoundTrip) { + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::boolean(), /*nullable=*/false), 3); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [true, false, true], + null, + [false, false, true]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/3)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorRecordBatchReaderSmallBatches) { + auto table = MakeVectorFixedSizeListTable(R"([ + [1, 2, 3], + null, + [7, 8, 9], + [10, 11, 12]])"); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/4, + VectorWriterProperties(), builder.build())); + + ArrowReaderProperties reader_properties; + reader_properties.set_batch_size(1); + std::unique_ptr reader; + FileReaderBuilder reader_builder; + ASSERT_OK_NO_THROW(reader_builder.Open(std::make_shared(buffer))); + ASSERT_OK(reader_builder.properties(reader_properties)->Build(&reader)); + std::unique_ptr<::arrow::RecordBatchReader> batch_reader; + ASSERT_OK_AND_ASSIGN(batch_reader, reader->GetRecordBatchReader()); + ASSERT_OK_AND_ASSIGN(auto result, Table::FromRecordBatchReader(batch_reader.get())); + ::arrow::AssertTablesEqual(*table, *result, false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNestedVectorsRoundTrip) { + auto inner_type = ::arrow::fixed_size_list(::arrow::uint8(), 2); + auto type = + ::arrow::fixed_size_list(::arrow::field("item", inner_type, /*nullable=*/false), 2); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [[1, 2], [3, 4]], + null, + [[5, 6], [7, 8]]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableChunkedReadAcrossRowGroups) { + auto table = MakeVectorFixedSizeListTable(R"([ + [1, 2, 3], + null, + [7, 8, 9], + null, + [13, 14, 15]])"); + + ArrowWriterProperties::Builder writer_builder; + writer_builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN( + auto buffer, WriteTableToBuffer(table, /*row_group_size=*/2, + VectorWriterProperties(), writer_builder.build())); + + ArrowReaderProperties reader_properties; + reader_properties.set_batch_size(2); + FileReaderBuilder reader_builder; + ASSERT_OK(reader_builder.Open(std::make_shared(buffer))); + reader_builder.properties(reader_properties); + std::unique_ptr reader; + ASSERT_OK(reader_builder.Build(&reader)); + ASSERT_OK_AND_ASSIGN(auto rb_reader, reader->GetRecordBatchReader()); + ASSERT_OK_AND_ASSIGN(auto out, Table::FromRecordBatchReader(rb_reader.get())); + + ASSERT_EQ(out->column(0)->num_chunks(), 3); + ASSERT_EQ(out->column(0)->chunk(0)->length(), 2); + ASSERT_EQ(out->column(0)->chunk(1)->length(), 2); + ASSERT_EQ(out->column(0)->chunk(2)->length(), 1); + ::arrow::AssertTablesEqual(*table, *out, false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableRoundTripSlicedInput) { + auto base = ::arrow::ArrayFromJSON(VectorFixedSizeListType(), R"([ + [100, 101, 102], + [1, 2, 3], + null, + [7, 8, 9], + null, + [200, 201, 202]])"); + auto table = MakeVectorFixedSizeListTable({base->Slice(/*offset=*/1, /*length=*/4)}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableRowsAndElementsRoundTrip) { + auto table = MakeVectorFixedSizeListTable(R"([ + [1, null, 3], + null, + [null, null, null], + [7, 8, null]])", + /*nullable=*/true, + /*element_nullable=*/true); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/4)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNestedRoundTrip) { + auto inner_type = ::arrow::fixed_size_list( + ::arrow::field("item", ::arrow::int16(), /*nullable=*/true), /*list_size=*/2); + auto outer_type = ::arrow::fixed_size_list( + ::arrow::field("item", inner_type, /*nullable=*/true), /*list_size=*/3); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("root", outer_type, /*nullable=*/true)}), + {::arrow::ArrayFromJSON(outer_type, R"([ + [[1, 2], null, [5, null]], + null, + [[7, 8], [9, 10], [null, 12]]])")}); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorMixedColumnsRoundTripAcrossRowGroups) { + auto vector_type = + ::arrow::fixed_size_list(::arrow::field("item", ::arrow::int16(), true), + /*list_size=*/3); + auto list_type = ::arrow::list(::arrow::field("item", ::arrow::int32(), true)); + + auto vector_chunks = std::vector>{ + ::arrow::ArrayFromJSON(vector_type, + R"([[1, null, 3], null, [7, 8, null], [10, 11, 12]])"), + ::arrow::ArrayFromJSON(vector_type, + R"([null, [16, null, 18], [19, 20, 21], [22, null, 24]])")}; + auto table = ::arrow::Table::Make( + ::arrow::schema({ + ::arrow::field("id", ::arrow::int32(), false), + ::arrow::field("embedding", vector_type, true), + ::arrow::field("label", ::arrow::utf8(), true), + ::arrow::field("tags", list_type, true), + }), + { + std::make_shared( + ::arrow::ArrayFromJSON(::arrow::int32(), "[0, 1, 2, 3, 4, 5, 6, 7]")), + std::make_shared(std::move(vector_chunks), vector_type), + std::make_shared(::arrow::ArrayFromJSON( + ::arrow::utf8(), R"(["a", null, "c", "d", null, "f", "g", "h"])")), + std::make_shared(::arrow::ArrayFromJSON( + list_type, R"([[1, 2], [], null, [3, null], [4], [5, 6], null, []])")), + }); + + ArrowWriterProperties::Builder writer_builder; + writer_builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN( + auto buffer, WriteTableToBuffer(table, /*row_group_size=*/3, + VectorWriterProperties(), writer_builder.build())); + + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + ASSERT_EQ(parquet_reader->metadata()->num_row_groups(), 3); + for (int i = 0; i < 3; ++i) { + const int64_t expected_rows = i == 2 ? 2 : 3; + EXPECT_EQ(parquet_reader->metadata()->RowGroup(i)->num_rows(), expected_rows); + EXPECT_EQ(parquet_reader->metadata()->RowGroup(i)->ColumnChunk(1)->num_values(), + expected_rows * 3); + } + + ArrowReaderProperties reader_properties; + reader_properties.set_batch_size(2); + FileReaderBuilder reader_builder; + ASSERT_OK(reader_builder.Open(std::make_shared(buffer))); + reader_builder.properties(reader_properties); + std::unique_ptr reader; + ASSERT_OK(reader_builder.Build(&reader)); + ASSERT_OK_AND_ASSIGN(auto rb_reader, reader->GetRecordBatchReader()); + ASSERT_OK_AND_ASSIGN(auto out, Table::FromRecordBatchReader(rb_reader.get())); + + ::arrow::AssertSchemaEqual(*table->schema(), *out->schema(), false); + ::arrow::AssertTablesEqual(*table, *out, false); +} + +std::shared_ptr<::arrow::DataType> VectorFixedSizeListStructType( + bool element_nullable = false, bool field_nullable = false) { + return ::arrow::fixed_size_list( + ::arrow::field( + "item", + ::arrow::struct_({::arrow::field("x", ::arrow::float32(), false), + ::arrow::field("y", ::arrow::int32(), field_nullable)}), + element_nullable), + /*size=*/2); +} + +std::shared_ptr
MakeVectorFixedSizeListStructTable(std::string_view json, + bool nullable = true, + bool element_nullable = false, + bool field_nullable = false) { + auto type = VectorFixedSizeListStructType(element_nullable, field_nullable); + auto field = ::arrow::field("root", type, nullable); + auto array = ::arrow::ArrayFromJSON(type, std::string(json)); + return ::arrow::Table::Make(::arrow::schema({field}), {array}); +} + +std::shared_ptr<::arrow::DataType> VectorFixedSizeListNestedStructType( + bool element_nullable = false) { + return ::arrow::fixed_size_list( + ::arrow::field( + "item", + ::arrow::struct_( + {::arrow::field( + "point", + ::arrow::struct_({::arrow::field("x", ::arrow::float32(), false), + ::arrow::field("y", ::arrow::int32(), true)}), + false), + ::arrow::field("z", ::arrow::int16(), true)}), + element_nullable), + /*size=*/2); +} + +std::shared_ptr
MakeVectorFixedSizeListNestedStructTable( + std::string_view json, bool nullable = true, bool element_nullable = false) { + auto type = VectorFixedSizeListNestedStructType(element_nullable); + auto field = ::arrow::field("root", type, nullable); + auto array = ::arrow::ArrayFromJSON(type, std::string(json)); + return ::arrow::Table::Make(::arrow::schema({field}), {array}); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableStructElementsRoundTrip) { + auto table = MakeVectorFixedSizeListStructTable(R"([ + [{"x": 1.0, "y": 1}, null], + null, + [null, {"x": 6.0, "y": null}], + [{"x": 7.0, "y": 7}, {"x": 8.0, "y": 8}]])", + /*nullable=*/true, + /*element_nullable=*/true, + /*field_nullable=*/true); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/2)); +} + +TEST(ArrowReadWrite, FixedSizeListVectorNullableNestedStructElementsRoundTrip) { + auto table = MakeVectorFixedSizeListNestedStructTable(R"([ + [{"point": {"x": 1.0, "y": 1}, "z": 10}, null], + null, + [null, {"point": {"x": 4.0, "y": 4}, "z": 40}], + [{"point": {"x": 5.0, "y": null}, "z": null}, + {"point": {"x": 6.0, "y": 6}, "z": 60}]])", + /*nullable=*/true, + /*element_nullable=*/true); + + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, /*row_group_size=*/2)); +} + +TEST(ArrowWriteOnly, FixedSizeListVectorDefaultWriterProperties) { + auto table = MakeVectorFixedSizeListTable(R"([ + [1, 2, 3], + [4, 5, 6]])", + /*nullable=*/false); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, + default_writer_properties(), builder.build())); + + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + const auto encodings = + parquet_reader->metadata()->RowGroup(0)->ColumnChunk(0)->encodings(); + EXPECT_NE(std::find(encodings.begin(), encodings.end(), Encoding::RLE_DICTIONARY), + encodings.end()); +} + +TEST(ArrowWriteOnly, FixedSizeListVectorNullableElementStatistics) { + auto table = MakeVectorFixedSizeListTable(::arrow::int32(), R"([ + [1, 2, 3], + null, + [4, null, 6]])", + /*nullable=*/true, + /*element_nullable=*/true); + + auto writer_properties = WriterProperties::Builder() + .disable_dictionary() + ->encoding(Encoding::PLAIN) + ->build(); + ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/3, writer_properties, + arrow_builder.build())); + + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + auto column = parquet_reader->metadata()->RowGroup(0)->ColumnChunk(0); + ASSERT_TRUE(column->is_stats_set()); + auto statistics = column->statistics(); + ASSERT_EQ(5, statistics->num_values()); + ASSERT_EQ(4, statistics->null_count()); + + std::shared_ptr min, max; + ASSERT_OK(StatisticsAsScalars(*statistics, &min, &max)); + ASSERT_OK_AND_ASSIGN(auto expected_min, ::arrow::MakeScalar(::arrow::int32(), 1)); + ASSERT_OK_AND_ASSIGN(auto expected_max, ::arrow::MakeScalar(::arrow::int32(), 6)); + ::arrow::AssertScalarsEqual(*expected_min, *min, /*verbose=*/true); + ::arrow::AssertScalarsEqual(*expected_max, *max, /*verbose=*/true); +} + +TEST(ArrowWriteOnly, FixedSizeListVectorPageIndex) { + auto table = MakeVectorFixedSizeListTable(::arrow::int32(), R"([ + [1, 2, 3], + [4, 5, 6], + [7, 8, 9]])", + /*nullable=*/false); + + auto writer_properties = WriterProperties::Builder() + .disable_dictionary() + ->encoding(Encoding::PLAIN) + ->enable_write_page_index() + ->build(); + ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, writer_properties, + arrow_builder.build())); + + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + auto page_index_reader = parquet_reader->GetPageIndexReader(); + ASSERT_NE(nullptr, page_index_reader); + auto row_group_index_reader = page_index_reader->RowGroup(0); + ASSERT_NE(nullptr, row_group_index_reader); + ASSERT_NE(nullptr, row_group_index_reader->GetColumnIndex(0)); + ASSERT_NE(nullptr, row_group_index_reader->GetOffsetIndex(0)); +} + +TEST(ArrowWriteOnly, FixedSizeListVectorBloomFilter) { + auto table = MakeVectorFixedSizeListTable(::arrow::int32(), R"([ + [1, 2, 3], + [4, 5, 6]])", + /*nullable=*/false); + + BloomFilterOptions bloom_filter_options{10, 0.05}; + auto writer_properties = + WriterProperties::Builder() + .disable_dictionary() + ->encoding(Encoding::PLAIN) + ->enable_bloom_filter("root.list.element", bloom_filter_options) + ->build(); + ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, writer_properties, + arrow_builder.build())); + + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + auto& bloom_filter_reader = parquet_reader->GetBloomFilterReader(); + auto row_group_bloom_filter = bloom_filter_reader.RowGroup(0); + ASSERT_NE(nullptr, row_group_bloom_filter); + auto bloom_filter = row_group_bloom_filter->GetColumnBloomFilter(0); + ASSERT_NE(nullptr, bloom_filter); + + int32_t present = 4; + EXPECT_TRUE(bloom_filter->FindHash(bloom_filter->Hash(present))); +} + +TEST(ArrowReadWrite, FixedSizeListVectorContentDefinedChunkingRoundTrip) { + std::stringstream json; + json << "["; + for (int i = 0; i < 40; ++i) { + if (i != 0) { + json << ","; + } + json << "[" << (3 * i + 1) << "," << (3 * i + 2) << "," << (3 * i + 3) << "]"; + } + json << "]"; + auto table = MakeVectorFixedSizeListTable(::arrow::int32(), json.str(), + /*nullable=*/false); + + CdcOptions cdc_options; + cdc_options.min_chunk_size = 64; + cdc_options.max_chunk_size = 128; + auto writer_properties = WriterProperties::Builder() + .enable_content_defined_chunking() + ->content_defined_chunking_options(cdc_options) + ->build(); + ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_experimental_vector_encoding(); + + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE(DoRoundtrip(table, /*row_group_size=*/40, &result, + writer_properties, arrow_builder.build())); + ::arrow::AssertSchemaEqual(*table->schema(), *result->schema(), false); + ::arrow::AssertTablesEqual(*table, *result, false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorContentDefinedChunkingNullableRoundTrip) { + auto table = MakeVectorFixedSizeListTable(::arrow::int32(), R"([ + [1, 2, 3], + null, + [4, null, 6], + [7, 8, 9]])", + /*nullable=*/true, + /*element_nullable=*/true); + + CdcOptions cdc_options; + cdc_options.min_chunk_size = 64; + cdc_options.max_chunk_size = 128; + auto writer_properties = WriterProperties::Builder() + .disable_dictionary() + ->enable_content_defined_chunking() + ->content_defined_chunking_options(cdc_options) + ->build(); + ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_experimental_vector_encoding(); + + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE(DoRoundtrip(table, /*row_group_size=*/4, &result, + writer_properties, arrow_builder.build())); + ::arrow::AssertSchemaEqual(*table->schema(), *result->schema(), false); + ::arrow::AssertTablesEqual(*table, *result, false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorByteStreamSplitRoundTrip) { + auto table = MakeVectorFixedSizeListTable(::arrow::float32(), R"([ + [1.0, 2.0, 3.0], + [4.0, 5.0, 6.0], + [7.0, 8.0, 9.0]])", + /*nullable=*/false); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN( + auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, + VectorByteStreamSplitWriterProperties(), builder.build())); + + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + const auto encodings = + parquet_reader->metadata()->RowGroup(0)->ColumnChunk(0)->encodings(); + ASSERT_NE(std::find(encodings.begin(), encodings.end(), Encoding::BYTE_STREAM_SPLIT), + encodings.end()); + + FileReaderBuilder reader_builder; + ASSERT_OK(reader_builder.Open(std::make_shared(buffer))); + std::unique_ptr reader; + ASSERT_OK(reader_builder.Build(&reader)); + std::shared_ptr
result; + ASSERT_OK(reader->ReadTable(&result)); + ::arrow::AssertSchemaEqual(*table->schema(), *result->schema(), false); + ::arrow::AssertTablesEqual(*table, *result, false); +} + +TEST(ArrowReadWrite, FixedSizeListVectorByteStreamSplitNullableRoundTrip) { + auto table = MakeVectorFixedSizeListTable(::arrow::float32(), R"([ + [1.0, 2.0, 3.0], + null, + [7.0, 8.0, 9.0]])"); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE(DoRoundtrip( + table, 3, &result, VectorByteStreamSplitWriterProperties(), builder.build())); + ::arrow::AssertSchemaEqual(*table->schema(), *result->schema(), false); + ::arrow::AssertTablesEqual(*table, *result, false); +} + TEST(ArrowReadWrite, ListOfStructOfList2) { using ::arrow::field; using ::arrow::list; @@ -3864,6 +4916,168 @@ TEST(ArrowReadWrite, NestedRequiredOuterOptionalDecimal) { } } +TEST(TestArrowReaderAdHoc, ReadVectorUnderNullableStructAncestor) { + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::INT32); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + ConvertedType::NONE, /*field_id=*/-1, + /*vector_length=*/2); + auto embedding = + GroupNode::Make("embedding", Repetition::REQUIRED, {vector}, LogicalType::Vector()); + auto parent = GroupNode::Make("parent", Repetition::OPTIONAL, {embedding}); + auto list = GroupNode::Make("list", Repetition::REPEATED, {parent}); + auto rows = GroupNode::Make("rows", Repetition::OPTIONAL, {list}, LogicalType::List()); + auto schema = GroupNode::Make("schema", Repetition::REQUIRED, {rows}); + + auto sink = CreateOutputStream(); + auto file_writer = + ParquetFileWriter::Open(sink, checked_pointer_cast(schema)); + auto column_writer = + checked_cast(file_writer->AppendRowGroup()->NextColumn()); + + // One top-level list row: [{"embedding": [1, 2]}, null, {"embedding": [3, 4]}]. + // The null struct element is represented by a single level with def_level=2; + // each present VECTOR value contributes two physical leaf slots at def_level=3. + const std::vector def_levels = {3, 3, 2, 3, 3}; + const std::vector rep_levels = {0, 1, 1, 1, 1}; + const std::vector values = {1, 2, 3, 4}; + column_writer->WriteBatch(static_cast(def_levels.size()), def_levels.data(), + rep_levels.data(), values.data()); + column_writer->Close(); + file_writer->Close(); + + ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish()); + + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto parent_type = ::arrow::struct_({::arrow::field("embedding", vector_type, + /*nullable=*/false)}); + auto rows_type = ::arrow::list(::arrow::field("parent", parent_type, + /*nullable=*/true)); + auto expected = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("rows", rows_type, /*nullable=*/true)}), + {::arrow::ArrayFromJSON( + rows_type, R"([[{"embedding": [1, 2]}, null, {"embedding": [3, 4]}]])")}); + + FileReaderBuilder reader_builder; + ASSERT_OK(reader_builder.Open(std::make_shared(buffer))); + std::unique_ptr reader; + ASSERT_OK(reader_builder.Build(&reader)); + ASSERT_OK_AND_ASSIGN(auto result, reader->ReadTable()); + ASSERT_OK(result->ValidateFull()); + ::arrow::AssertTablesEqual(*expected, *result, /*same_chunk_layout=*/false); +} + +TEST(TestArrowReaderAdHoc, ReadVectorUnderNullableStructAncestorWithEmptyLists) { + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::INT32); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + ConvertedType::NONE, /*field_id=*/-1, + /*vector_length=*/2); + auto embedding = + GroupNode::Make("embedding", Repetition::REQUIRED, {vector}, LogicalType::Vector()); + auto parent = GroupNode::Make("parent", Repetition::OPTIONAL, {embedding}); + auto list = GroupNode::Make("list", Repetition::REPEATED, {parent}); + auto rows = GroupNode::Make("rows", Repetition::OPTIONAL, {list}, LogicalType::List()); + auto schema = GroupNode::Make("schema", Repetition::REQUIRED, {rows}); + + auto sink = CreateOutputStream(); + auto file_writer = + ParquetFileWriter::Open(sink, checked_pointer_cast(schema)); + auto column_writer = + checked_cast(file_writer->AppendRowGroup()->NextColumn()); + + // Four top-level rows: null, [], [null], [{"embedding": [1, 2]}]. + // The null and empty lists are compact absent-list markers and must not be + // expanded into FixedSizeList child rows; only the null struct item expands. + const std::vector def_levels = {0, 1, 2, 3, 3}; + const std::vector rep_levels = {0, 0, 0, 0, 1}; + const std::vector values = {1, 2}; + column_writer->WriteBatch(static_cast(def_levels.size()), def_levels.data(), + rep_levels.data(), values.data()); + column_writer->Close(); + file_writer->Close(); + + ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish()); + + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto parent_type = ::arrow::struct_({::arrow::field("embedding", vector_type, + /*nullable=*/false)}); + auto rows_type = ::arrow::list(::arrow::field("parent", parent_type, + /*nullable=*/true)); + auto expected = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("rows", rows_type, /*nullable=*/true)}), + {::arrow::ArrayFromJSON(rows_type, + R"([null, [], [null], [{"embedding": [1, 2]}]])")}); + + FileReaderBuilder reader_builder; + ASSERT_OK(reader_builder.Open(std::make_shared(buffer))); + std::unique_ptr reader; + ASSERT_OK(reader_builder.Build(&reader)); + ASSERT_OK_AND_ASSIGN(auto result, reader->ReadTable()); + ASSERT_OK(result->ValidateFull()); + ::arrow::AssertTablesEqual(*expected, *result, /*same_chunk_layout=*/false); +} + +TEST(TestArrowReaderAdHoc, ReadNestedVectorUnderNullableStructAncestor) { + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::INT32); + auto inner_vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + ConvertedType::NONE, /*field_id=*/-1, + /*vector_length=*/3); + auto inner = GroupNode::Make("inner", Repetition::REQUIRED, {inner_vector}, + LogicalType::Vector()); + auto outer_vector = GroupNode::Make("list", Repetition::VECTOR, {inner}, + ConvertedType::NONE, /*field_id=*/-1, + /*vector_length=*/2); + auto embedding = GroupNode::Make("embedding", Repetition::REQUIRED, {outer_vector}, + LogicalType::Vector()); + auto parent = GroupNode::Make("parent", Repetition::OPTIONAL, {embedding}); + auto list = GroupNode::Make("list", Repetition::REPEATED, {parent}); + auto rows = GroupNode::Make("rows", Repetition::OPTIONAL, {list}, LogicalType::List()); + auto schema = GroupNode::Make("schema", Repetition::REQUIRED, {rows}); + + auto sink = CreateOutputStream(); + auto file_writer = + ParquetFileWriter::Open(sink, checked_pointer_cast(schema)); + auto column_writer = + checked_cast(file_writer->AppendRowGroup()->NextColumn()); + + // One top-level list row: + // [{"embedding": [[1, 2, 3], [4, 5, 6]]}, null, + // {"embedding": [[7, 8, 9], [10, 11, 12]]}]. + const std::vector def_levels = {3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3}; + std::vector rep_levels(def_levels.size(), 1); + rep_levels[0] = 0; + const std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; + column_writer->WriteBatch(static_cast(def_levels.size()), def_levels.data(), + rep_levels.data(), values.data()); + column_writer->Close(); + file_writer->Close(); + + ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish()); + + auto inner_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 3); + auto outer_type = ::arrow::fixed_size_list( + ::arrow::field("inner", inner_type, /*nullable=*/false), 2); + auto parent_type = ::arrow::struct_({::arrow::field("embedding", outer_type, + /*nullable=*/false)}); + auto rows_type = ::arrow::list(::arrow::field("parent", parent_type, + /*nullable=*/true)); + auto expected = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("rows", rows_type, /*nullable=*/true)}), + {::arrow::ArrayFromJSON(rows_type, + R"([[{"embedding": [[1, 2, 3], [4, 5, 6]]}, null, + {"embedding": [[7, 8, 9], [10, 11, 12]]}]])")}); + + FileReaderBuilder reader_builder; + ASSERT_OK(reader_builder.Open(std::make_shared(buffer))); + std::unique_ptr reader; + ASSERT_OK(reader_builder.Build(&reader)); + ASSERT_OK_AND_ASSIGN(auto result, reader->ReadTable()); + ASSERT_OK(result->ValidateFull()); + ::arrow::AssertTablesEqual(*expected, *result, /*same_chunk_layout=*/false); +} + TEST(ArrowReadWrite, Decimal256AsInt) { using ::arrow::Decimal256; using ::arrow::field; @@ -4431,8 +5645,7 @@ TEST(TestArrowReaderAdHoc, OldDataPageV2) { GTEST_SKIP() << "ARROW_TEST_DATA not set."; } std::stringstream ss; - ss << c_root << "/" - << "parquet/ARROW-17100.parquet"; + ss << c_root << "/" << "parquet/ARROW-17100.parquet"; std::string path = ss.str(); TryReadDataFile(path); } diff --git a/cpp/src/parquet/arrow/arrow_schema_test.cc b/cpp/src/parquet/arrow/arrow_schema_test.cc index 7a7b5a336939..5a626f523783 100644 --- a/cpp/src/parquet/arrow/arrow_schema_test.cc +++ b/cpp/src/parquet/arrow/arrow_schema_test.cc @@ -289,6 +289,176 @@ TEST_F(TestConvertParquetSchema, ParquetAnnotatedFields) { ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(arrow_schema)); } +TEST_F(TestConvertParquetSchema, UnannotatedVectorGroupRejected) { + // The VECTOR logical type annotation is required: a plain group containing a + // VECTOR-repeated child is not silently interpreted as a vector. + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("vector", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + std::vector parquet_fields = {GroupNode::Make( + "embedding", Repetition::OPTIONAL, {vector}, /*logical_type=*/nullptr)}; + + ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); +} + +TEST_F(TestConvertParquetSchema, VectorFixedSizeListAnnotated) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + arrow_fields.push_back(::arrow::field( + "embedding", + ::arrow::fixed_size_list(::arrow::field("element", ::arrow::float32(), false), 3), + true)); + + ASSERT_OK(ConvertSchema(parquet_fields)); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(::arrow::schema(arrow_fields))); +} + +TEST_F(TestConvertParquetSchema, VectorAnnotatedGroupWithoutVectorChildRejected) { + // A group annotated with the VECTOR logical type must contain a single + // VECTOR-repeated child. + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + std::vector parquet_fields = {GroupNode::Make( + "embedding", Repetition::OPTIONAL, {element}, LogicalType::Vector())}; + + ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); +} + +TEST_F(TestConvertParquetSchema, VectorMiddleGroupNameIrrelevant) { + // Mirroring LIST, readers do not depend on the VECTOR-repeated group's name. + std::vector parquet_fields; + std::vector> arrow_fields; + + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("vector", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + arrow_fields.push_back(::arrow::field( + "embedding", + ::arrow::fixed_size_list(::arrow::field("element", ::arrow::float32(), false), 3), + true)); + + ASSERT_OK(ConvertSchema(parquet_fields)); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(::arrow::schema(arrow_fields))); +} + +TEST_F(TestConvertParquetSchema, VectorMiddleGroupLogicalTypeRejected) { + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = + GroupNode::Make("list", Repetition::VECTOR, {element}, LogicalType::List(), -1, 3); + std::vector parquet_fields = {GroupNode::Make( + "embedding", Repetition::OPTIONAL, {vector}, LogicalType::Vector())}; + + ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); +} + +TEST_F(TestConvertParquetSchema, VectorOnPrimitiveRejected) { + // VECTOR repetition directly on a primitive is not part of the canonical + // 3-level vector structure; vectors must use a VECTOR group whose single + // child is a VECTOR-repeated group containing the element. + ASSERT_THROW(PrimitiveNode::Make("element", Repetition::VECTOR, ParquetType::FLOAT, + ConvertedType::NONE, -1, -1, -1, -1, 3), + ParquetException); +} + +TEST_F(TestConvertParquetSchema, VectorUnderNullableStructAncestor) { + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 2); + auto embedding = + GroupNode::Make("embedding", Repetition::REQUIRED, {vector}, LogicalType::Vector()); + std::vector parquet_fields = { + GroupNode::Make("outer", Repetition::OPTIONAL, {embedding})}; + + auto expected = ::arrow::schema({::arrow::field( + "outer", + ::arrow::struct_({::arrow::field( + "embedding", + ::arrow::fixed_size_list(::arrow::field("element", INT32, false), 2), false)}), + true)}); + + ASSERT_OK(ConvertSchema(parquet_fields)); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(expected)); +} + +TEST_F(TestConvertParquetSchema, VectorFixedSizeListNullableStructElement) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto x = PrimitiveNode::Make("x", Repetition::REQUIRED, ParquetType::FLOAT); + auto y = PrimitiveNode::Make("y", Repetition::OPTIONAL, ParquetType::INT32); + auto item = GroupNode::Make("element", Repetition::OPTIONAL, {x, y}); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {item}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + arrow_fields.push_back(::arrow::field( + "embedding", + ::arrow::fixed_size_list( + ::arrow::field("element", + ::arrow::struct_({::arrow::field("x", FLOAT, false), + ::arrow::field("y", INT32, true)}), + true), + 3), + true)); + + ASSERT_OK(ConvertSchema(parquet_fields)); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(::arrow::schema(arrow_fields))); +} + +TEST_F(TestConvertParquetSchema, VectorFixedSizeListNullableNestedStructElement) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto x = PrimitiveNode::Make("x", Repetition::REQUIRED, ParquetType::FLOAT); + auto y = PrimitiveNode::Make("y", Repetition::OPTIONAL, ParquetType::INT32); + auto point = GroupNode::Make("point", Repetition::REQUIRED, {x, y}); + auto z = PrimitiveNode::Make("z", Repetition::OPTIONAL, LogicalType::Int(16, true), + ParquetType::INT32); + auto item = GroupNode::Make("element", Repetition::OPTIONAL, {point, z}); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {item}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + arrow_fields.push_back(::arrow::field( + "embedding", + ::arrow::fixed_size_list( + ::arrow::field( + "element", + ::arrow::struct_( + {::arrow::field("point", + ::arrow::struct_({::arrow::field("x", FLOAT, false), + ::arrow::field("y", INT32, true)}), + false), + ::arrow::field("z", ::arrow::int16(), true)}), + true), + 3), + true)); + + ASSERT_OK(ConvertSchema(parquet_fields)); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(::arrow::schema(arrow_fields))); +} + +TEST_F(TestConvertParquetSchema, VectorFixedSizeListStructWithListRejected) { + auto list_element = + PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32); + auto list = GroupNode::Make("list", Repetition::REPEATED, {list_element}); + auto items = + GroupNode::Make("items", Repetition::REQUIRED, {list}, LogicalType::List()); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {items}, + /*logical_type=*/nullptr, -1, 3); + std::vector parquet_fields = {GroupNode::Make( + "embedding", Repetition::OPTIONAL, {vector}, LogicalType::Vector())}; + + ASSERT_RAISES(NotImplemented, ConvertSchema(parquet_fields)); +} + TEST_F(TestConvertParquetSchema, DuplicateFieldNames) { std::vector parquet_fields; std::vector> arrow_fields; @@ -1755,7 +1925,8 @@ TEST_F(TestConvertArrowSchema, ParquetOtherLists) { AddListLikeField(::arrow::list_view(arrow_element)); // LargeListView (list-like non-null, elements nullable) AddListLikeField(::arrow::large_list_view(arrow_element)); - // FixedSizeList[10] (list-like non-null, elements nullable) + // FixedSizeList defaults to the legacy LIST encoding unless experimental VECTOR + // encoding is explicitly enabled. AddListLikeField(::arrow::fixed_size_list(arrow_element, 10)); ASSERT_OK(ConvertSchema(arrow_fields)); @@ -1763,6 +1934,272 @@ TEST_F(TestConvertArrowSchema, ParquetOtherLists) { ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); } +TEST_F(TestConvertArrowSchema, + ParquetFixedSizeListVectorUnsupportedElementFallsBackToList) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto element = PrimitiveNode::Make("element", Repetition::OPTIONAL, + ParquetType::BYTE_ARRAY, ConvertedType::UTF8); + auto list = GroupNode::Make("list", Repetition::REPEATED, {element}); + parquet_fields.push_back( + GroupNode::Make("embedding", Repetition::OPTIONAL, {list}, ConvertedType::LIST)); + + auto arrow_element = ::arrow::field("string", UTF8, true); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorZeroLengthFallsBackToList) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto list = GroupNode::Make("list", Repetition::REPEATED, {element}); + parquet_fields.push_back( + GroupNode::Make("embedding", Repetition::OPTIONAL, {list}, ConvertedType::LIST)); + + auto arrow_element = ::arrow::field("element", FLOAT, false); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 0); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, + ParquetFixedSizeListVectorOuterLengthOverflowFallsBackToList) { + auto inner_type = + ::arrow::fixed_size_list(::arrow::field("element", FLOAT, false), 46341); + auto outer_type = ::arrow::fixed_size_list( + ::arrow::field("inner", inner_type, /*nullable=*/false), 46341); + std::vector> arrow_fields = { + ::arrow::field("embedding", outer_type, /*nullable=*/true)}; + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + + ASSERT_EQ(1, result_schema_->num_columns()); + // The outer FixedSizeList falls back to LIST because the nested VECTOR product + // would overflow, while the inner FixedSizeList can still use VECTOR. + EXPECT_EQ(46341, result_schema_->Column(0)->effective_vector_length()); + const auto* field = result_schema_->group_node()->field(0).get(); + ASSERT_TRUE(field->is_group()); + EXPECT_TRUE(static_cast(field)->logical_type()->is_list()); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorUnderList) { + // FixedSizeList values below a repeated field can use VECTOR encoding. + std::vector parquet_fields; + std::vector> arrow_fields; + + auto inner_element = + PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {inner_element}, + /*logical_type=*/nullptr, -1, 2); + auto element = + GroupNode::Make("element", Repetition::REQUIRED, {vector}, LogicalType::Vector()); + auto list = GroupNode::Make("list", Repetition::REPEATED, {element}); + parquet_fields.push_back( + GroupNode::Make("embeddings", Repetition::OPTIONAL, {list}, ConvertedType::LIST)); + + auto vector_type = + ::arrow::fixed_size_list(::arrow::field("element", INT32, /*nullable=*/false), 2); + arrow_fields.push_back(::arrow::field( + "embeddings", ::arrow::list(::arrow::field("element", vector_type, false)), true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorMixedSupportedAndFallback) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + + auto string_element = PrimitiveNode::Make("element", Repetition::OPTIONAL, + ParquetType::BYTE_ARRAY, ConvertedType::UTF8); + auto string_list = GroupNode::Make("list", Repetition::REPEATED, {string_element}); + parquet_fields.push_back(GroupNode::Make("labels", Repetition::OPTIONAL, {string_list}, + ConvertedType::LIST)); + + arrow_fields.push_back(::arrow::field( + "embedding", ::arrow::fixed_size_list(::arrow::field("element", FLOAT, false), 3), + true)); + arrow_fields.push_back(::arrow::field( + "labels", ::arrow::fixed_size_list(::arrow::field("string", UTF8, true), 3), true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorNullableElement) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto item = PrimitiveNode::Make("element", Repetition::OPTIONAL, ParquetType::FLOAT, + ConvertedType::NONE, -1, -1, -1, -1, -1); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {item}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + + auto arrow_element = ::arrow::field("element", FLOAT, true); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorNullableStructElement) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto x = PrimitiveNode::Make("x", Repetition::REQUIRED, ParquetType::FLOAT, + ConvertedType::NONE, -1, -1, -1, -1, -1); + auto y = PrimitiveNode::Make("y", Repetition::OPTIONAL, ParquetType::INT32, + ConvertedType::NONE, -1, -1, -1, -1, -1); + auto item = GroupNode::Make("element", Repetition::OPTIONAL, {x, y}); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {item}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + + auto arrow_element = ::arrow::field( + "element", + ::arrow::struct_({::arrow::field("x", FLOAT, false), ::arrow::field("y", INT32)}), + true); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorNullableNestedStructElement) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto x = PrimitiveNode::Make("x", Repetition::REQUIRED, ParquetType::FLOAT, + ConvertedType::NONE, -1, -1, -1, -1, -1); + auto y = PrimitiveNode::Make("y", Repetition::OPTIONAL, ParquetType::INT32, + ConvertedType::NONE, -1, -1, -1, -1, -1); + auto point = GroupNode::Make("point", Repetition::REQUIRED, {x, y}); + auto z = PrimitiveNode::Make("z", Repetition::OPTIONAL, LogicalType::Int(16, true), + ParquetType::INT32, -1, -1, -1); + auto item = GroupNode::Make("element", Repetition::OPTIONAL, {point, z}); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {item}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + + auto arrow_element = ::arrow::field( + "element", + ::arrow::struct_( + {::arrow::field("point", + ::arrow::struct_({::arrow::field("x", FLOAT, false), + ::arrow::field("y", INT32, true)}), + false), + ::arrow::field("z", ::arrow::int16(), true)}), + true); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorStructWithListFallsBackToList) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto item = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::INT32); + auto items_list = GroupNode::Make("list", Repetition::REPEATED, {item}); + auto items = + GroupNode::Make("items", Repetition::REQUIRED, {items_list}, ConvertedType::LIST); + auto element = GroupNode::Make("element", Repetition::REQUIRED, {items}); + auto list = GroupNode::Make("list", Repetition::REPEATED, {element}); + parquet_fields.push_back( + GroupNode::Make("embedding", Repetition::OPTIONAL, {list}, ConvertedType::LIST)); + + auto arrow_element = ::arrow::field( + "element", + ::arrow::struct_({::arrow::field( + "items", ::arrow::list(::arrow::field("element", INT32, false)), false)}), + false); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVector) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + + auto arrow_element = ::arrow::field("element", FLOAT, false); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + +TEST_F(TestConvertArrowSchema, ParquetFixedSizeListVectorElementNameCanonical) { + std::vector parquet_fields; + std::vector> arrow_fields; + + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); + parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector())); + + auto arrow_element = ::arrow::field("value", FLOAT, false); + auto arrow_list = ::arrow::fixed_size_list(arrow_element, 3); + arrow_fields.push_back(::arrow::field("embedding", arrow_list, true)); + + ArrowWriterProperties::Builder builder; + builder.disable_compliant_nested_types()->enable_experimental_vector_encoding(); + ASSERT_OK(ConvertSchema(arrow_fields, builder.build())); + ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields)); +} + TEST_F(TestConvertArrowSchema, ParquetNestedComplianceEnabledNullable) { std::vector parquet_fields; std::vector> arrow_fields; diff --git a/cpp/src/parquet/arrow/path_internal.cc b/cpp/src/parquet/arrow/path_internal.cc index c626dcbf48b6..5088d26779c1 100644 --- a/cpp/src/parquet/arrow/path_internal.cc +++ b/cpp/src/parquet/arrow/path_internal.cc @@ -86,6 +86,7 @@ #include #include +#include #include #include #include @@ -106,6 +107,7 @@ #include "arrow/util/macros.h" #include "arrow/visit_array_inline.h" +#include "parquet/arrow/schema_internal.h" #include "parquet/properties.h" namespace parquet::arrow { @@ -209,6 +211,29 @@ struct PathWriteContext { visited_elements.push_back(range); } + void RecordVectorPostListVisit(const ElementRange& logical_range, + const ElementRange& physical_range) { + if (!visited_elements.empty() && + visited_elements.back().start == logical_range.start && + visited_elements.back().end == logical_range.end) { + visited_elements.back() = physical_range; + return; + } + RecordPostListVisit(physical_range); + } + + // The enclosing list machinery records the logical (vector-index) child + // range before delegating to a VectorNullableNode, which then records the + // physical (slot-index) ranges of its present runs itself. Remove the + // logical range so only physical ranges remain. + void TrimVectorLogicalVisit(const ElementRange& logical_range) { + if (!visited_elements.empty() && + visited_elements.back().start == logical_range.start && + visited_elements.back().end == logical_range.end) { + visited_elements.pop_back(); + } + } + Status last_status; TypedBufferBuilder rep_levels; TypedBufferBuilder def_levels; @@ -475,6 +500,120 @@ struct FixedSizedRangeSelector { int list_size; }; +struct NoLevelTerminalNode { + IterationResult Run(const ElementRange&, PathWriteContext*) { return kDone; } +}; + +IterationResult ExpandLastRepLevels(int64_t logical_count, int32_t multiplier, + int16_t filler_rep_level, PathWriteContext* context); + +class VectorNullableNode { + public: + VectorNullableNode(const uint8_t* null_bitmap, int64_t entry_offset, + int32_t vector_length, int32_t child_slot_multiplier, + int16_t def_level_if_present, int16_t filler_rep_level, + bool child_emits_present_def_levels, + bool child_records_visited_elements, bool expand_rep_levels) + : null_bitmap_(null_bitmap), + entry_offset_(entry_offset), + vector_length_(vector_length), + child_slot_multiplier_(child_slot_multiplier), + valid_bits_reader_(MakeReader(ElementRange{0, 0})), + def_level_if_present_(def_level_if_present), + def_level_if_null_(def_level_if_present - 1), + filler_rep_level_(filler_rep_level), + child_emits_present_def_levels_(child_emits_present_def_levels), + child_records_visited_elements_(child_records_visited_elements), + expand_rep_levels_(expand_rep_levels), + new_range_(true) {} + + ::arrow::internal::BitRunReader MakeReader(const ElementRange& range) { + return ::arrow::internal::BitRunReader(null_bitmap_, entry_offset_ + range.start, + range.Size()); + } + + IterationResult Run(ElementRange* range, ElementRange* child_range, + PathWriteContext* context) { + if (range->Empty()) { + new_range_ = true; + return kDone; + } + if (new_range_) { + context->TrimVectorLogicalVisit(*range); + } + if (expand_rep_levels_ && new_range_) { + // Expand the trailing repetition levels (one per logical vector in this + // delegation, appended by the enclosing list machinery) once, before + // any run processing. Every vector contributes + // vector_length * child_slot_multiplier leaf slots regardless of + // validity, so the multiplier is constant and per-run expansion (which + // would expand positionally wrong entries for interleaved null and + // present runs) is not needed. Only the outermost vector node of a + // nested chain expands. + RETURN_IF_ERROR(ExpandLastRepLevels(range->Size(), + vector_length_ * child_slot_multiplier_, + filler_rep_level_, context)); + } + if (null_bitmap_ == nullptr) { + ElementRange logical_range = *range; + child_range->start = range->start * vector_length_; + child_range->end = child_range->start + range->Size() * vector_length_; + if (!child_records_visited_elements_) { + context->RecordVectorPostListVisit(logical_range, *child_range); + } + if (!child_emits_present_def_levels_) { + RETURN_IF_ERROR( + context->AppendDefLevels(child_range->Size(), def_level_if_present_)); + } + range->start = range->end; + new_range_ = false; + return kNext; + } + if (new_range_) { + valid_bits_reader_ = MakeReader(*range); + } + ::arrow::internal::BitRun run = valid_bits_reader_.NextRun(); + new_range_ = false; + while (!range->Empty() && !run.set) { + range->start += run.length; + RETURN_IF_ERROR(context->AppendDefLevels( + run.length * vector_length_ * child_slot_multiplier_, def_level_if_null_)); + run = valid_bits_reader_.NextRun(); + } + if (range->Empty()) { + new_range_ = true; + return kDone; + } + + ElementRange logical_range{range->start, range->start + run.length}; + child_range->start = range->start * vector_length_; + child_range->end = child_range->start + run.length * vector_length_; + if (!child_records_visited_elements_) { + context->RecordVectorPostListVisit(logical_range, *child_range); + } + if (!child_emits_present_def_levels_) { + RETURN_IF_ERROR( + context->AppendDefLevels(run.length * vector_length_, def_level_if_present_)); + } + range->start += run.length; + return kNext; + } + + private: + const uint8_t* null_bitmap_; + int64_t entry_offset_; + int32_t vector_length_; + int32_t child_slot_multiplier_; + ::arrow::internal::BitRunReader valid_bits_reader_; + int16_t def_level_if_present_; + int16_t def_level_if_null_; + int16_t filler_rep_level_; + bool child_emits_present_def_levels_; + bool child_records_visited_elements_; + bool expand_rep_levels_; + bool new_range_ = true; +}; + // An intermediate node that handles null values. class NullableNode { public: @@ -546,7 +685,8 @@ struct PathInfo { // Note index order matters here. using Node = std::variant; + VectorNullableNode, AllPresentTerminalNode, + AllNullsTerminalNode, NoLevelTerminalNode>; std::vector path; std::shared_ptr primitive_array; @@ -554,6 +694,18 @@ struct PathInfo { int16_t max_rep_level = 0; bool has_dictionary = false; bool leaf_is_nullable = false; + bool leaf_is_vector = false; + int32_t leaf_vector_length = 1; + // Definition level after the last repeated ancestor (0 when there is none). + // For VECTOR leaves, definition levels below it mark ancestors that made + // the vector value absent (empty or null lists); they occupy one level + // entry with no slots, while levels at or above it occupy fixed strides of + // leaf_vector_length slots. + int16_t repeated_ancestor_def_level = 0; + // Definition level at or above which a leaf slot carries a value from the + // leaf array (all vector ancestors present); slots below it (but at or + // above the stride level) belong to null vector values. + int16_t vector_values_def_level = 0; }; struct WritePathVisitor { @@ -569,6 +721,12 @@ struct WritePathVisitor { IterationResult operator()(AllNullsTerminalNode& node) { return node.Run(*stack_position, context); } + IterationResult operator()(VectorNullableNode& node) { + return node.Run(stack_position, stack_position + 1, context); + } + IterationResult operator()(NoLevelTerminalNode& node) { + return node.Run(*stack_position, context); + } template IterationResult operator()(ListPathNode& node) { return node.Run(stack_position, stack_position + 1, context); @@ -585,6 +743,53 @@ struct WritePathVisitor { /// values have been calculated for root_range with the calculated /// values. It is intended to abstract the complexity of writing /// the levels and values to parquet. +// Counts the physical leaf slots of a VECTOR leaf: definition levels at or +// above the stride definition level. Entries below it are absent-ancestor +// markers carrying no slot. +int64_t CountVectorSlots(const int16_t* def_levels, int64_t def_count, + int16_t repeated_ancestor_def_level) { + if (repeated_ancestor_def_level <= 0) { + return def_count; + } + int64_t slots = 0; + for (int64_t i = 0; i < def_count; ++i) { + slots += def_levels[i] >= repeated_ancestor_def_level; + } + return slots; +} + +IterationResult ExpandLastRepLevels(int64_t logical_count, int32_t multiplier, + int16_t filler_rep_level, PathWriteContext* context) { + if (multiplier <= 1 || logical_count == 0 || context->rep_levels.length() == 0) { + return kDone; + } + const int64_t old_length = context->rep_levels.length(); + if (ARROW_PREDICT_FALSE(old_length < logical_count)) { + context->last_status = + Status::Invalid("VECTOR repetition level expansion needed ", logical_count, + " source levels but only ", old_length, " were available"); + return kError; + } + const int64_t prefix_length = old_length - logical_count; + // Grow the buffer by the filler entries, then rewrite the trailing + // logical_count entries in place. Entry i moves from prefix_length + i to + // prefix_length + i * multiplier, so a backward walk never overwrites an + // unread source entry and the cost stays linear in the slots written. + context->last_status = + context->rep_levels.Append((multiplier - 1) * logical_count, filler_rep_level); + if (ARROW_PREDICT_FALSE(!context->last_status.ok())) { + return kError; + } + int16_t* data = context->rep_levels.mutable_data(); + for (int64_t i = logical_count - 1; i >= 0; --i) { + const int16_t source = data[prefix_length + i]; + int16_t* destination = data + prefix_length + i * multiplier; + destination[0] = source; + std::fill_n(destination + 1, multiplier - 1, filler_rep_level); + } + return kDone; +} + Status WritePath(ElementRange root_range, PathInfo* path_info, ArrowWriteContext* arrow_context, MultipathLevelBuilder::CallbackFunction writer) { @@ -592,12 +797,18 @@ Status WritePath(ElementRange root_range, PathInfo* path_info, MultipathLevelBuilderResult builder_result; builder_result.leaf_array = path_info->primitive_array; builder_result.leaf_is_nullable = path_info->leaf_is_nullable; + builder_result.leaf_is_vector = path_info->leaf_is_vector; + builder_result.leaf_vector_length = path_info->leaf_vector_length; + builder_result.vector_repeated_ancestor_def_level = + path_info->repeated_ancestor_def_level; + builder_result.vector_values_def_level = path_info->vector_values_def_level; if (path_info->max_def_level == 0) { // This case only occurs when there are no nullable or repeated // columns in the path from the root to leaf. int64_t leaf_length = builder_result.leaf_array->length(); builder_result.def_rep_level_count = leaf_length; + builder_result.leaf_slot_count = leaf_length; builder_result.post_list_visited_elements.push_back({0, leaf_length}); return writer(builder_result); } @@ -640,6 +851,11 @@ Status WritePath(ElementRange root_range, PathInfo* path_info, // This case only occurs when there was a repeated element that needs to be // processed. builder_result.rep_levels = context.rep_levels.data(); + if (path_info->leaf_is_vector) { + builder_result.leaf_slot_count = + CountVectorSlots(context.def_levels.data(), context.def_levels.length(), + path_info->repeated_ancestor_def_level); + } std::swap(builder_result.post_list_visited_elements, context.visited_elements); // If it is possible when processing lists that all lists where empty. In this // case no elements would have been added to post_list_visited_elements. By @@ -648,9 +864,18 @@ Status WritePath(ElementRange root_range, PathInfo* path_info, builder_result.post_list_visited_elements.push_back({0, 0}); } } else { - builder_result.post_list_visited_elements.push_back( - {0, builder_result.leaf_array->length()}); + if (!context.visited_elements.empty()) { + std::swap(builder_result.post_list_visited_elements, context.visited_elements); + } else { + builder_result.post_list_visited_elements.push_back( + {0, builder_result.leaf_array->length()}); + } builder_result.rep_levels = nullptr; + if (path_info->leaf_is_vector) { + builder_result.leaf_slot_count = + CountVectorSlots(context.def_levels.data(), context.def_levels.length(), + path_info->repeated_ancestor_def_level); + } } builder_result.def_levels = context.def_levels.data(); @@ -682,6 +907,7 @@ struct FixupVisitor { } void operator()(NullableNode& arg) { HandleIntermediateNode(arg); } + void operator()(VectorNullableNode&) {} void operator()(AllNullsTerminalNode& arg) { // Even though no processing happens past this point we @@ -692,6 +918,7 @@ struct FixupVisitor { void operator()(NullableTerminalNode&) {} void operator()(AllPresentTerminalNode&) {} + void operator()(NoLevelTerminalNode&) {} }; PathInfo Fixup(PathInfo info) { @@ -713,7 +940,9 @@ PathInfo Fixup(PathInfo info) { class PathBuilder { public: - explicit PathBuilder(bool start_nullable) : nullable_in_parent_(start_nullable) {} + PathBuilder(bool start_nullable, bool write_fixed_size_list_as_vector) + : nullable_in_parent_(start_nullable), + write_fixed_size_list_as_vector_(write_fixed_size_list_as_vector) {} template void AddTerminalInfo(const T& array) { info_.leaf_is_nullable = nullable_in_parent_; @@ -752,13 +981,18 @@ class PathBuilder { // Increment necessary due to empty lists. info_.max_def_level++; info_.max_rep_level++; + info_.repeated_ancestor_def_level = info_.max_def_level; // raw_value_offsets() accounts for any slice offset. ListPathNode> node( VarRangeSelector{array.raw_value_offsets()}, info_.max_rep_level, info_.max_def_level - 1); info_.path.emplace_back(std::move(node)); nullable_in_parent_ = array.list_type()->value_field()->nullable(); - return VisitInline(*array.values()); + const bool saved_nullable_group = nullable_group_since_repeated_; + nullable_group_since_repeated_ = false; + Status status = VisitInline(*array.values()); + nullable_group_since_repeated_ = saved_nullable_group; + return status; } template @@ -828,31 +1062,105 @@ class PathBuilder { } Status Visit(const ::arrow::StructArray& array) { + const bool struct_nullable = nullable_in_parent_; MaybeAddNullable(array); PathInfo info_backup = info_; + const bool saved_nullable_group = nullable_group_since_repeated_; + nullable_group_since_repeated_ = nullable_group_since_repeated_ || struct_nullable; for (int x = 0; x < array.num_fields(); x++) { nullable_in_parent_ = array.type()->field(x)->nullable(); RETURN_NOT_OK(VisitInline(*array.field(x))); info_ = info_backup; } + nullable_group_since_repeated_ = saved_nullable_group; return Status::OK(); } Status Visit(const ::arrow::FixedSizeListArray& array) { - MaybeAddNullable(array); int32_t list_size = array.list_type()->list_size(); + auto vector_leaf_slot_multiplier = VectorLeafSlotMultiplier(*array.type()); + if (write_fixed_size_list_as_vector_ && list_size > 0 && + !nullable_group_since_repeated_ && vector_leaf_slot_multiplier.ok() && + IsSupportedVectorElementType(*array.value_type())) { + const bool element_nullable = array.list_type()->value_field()->nullable(); + const bool nested_vector_path = info_.leaf_is_vector; + info_.leaf_is_vector = true; + if (info_.leaf_vector_length > std::numeric_limits::max() / list_size) { + return Status::Invalid("Nested VECTOR leaf slot multiplier overflow"); + } + info_.leaf_vector_length *= list_size; + const bool parent_nullable = nullable_in_parent_; + if (parent_nullable) { + info_.max_def_level++; + } + // Overwritten by nested vectors so that the final value is the present + // level of the innermost vector. + info_.vector_values_def_level = info_.max_def_level; + const bool value_type_is_struct = array.value_type()->id() == ::arrow::Type::STRUCT; + const bool value_type_is_nested_vector = + array.value_type()->id() == ::arrow::Type::FIXED_SIZE_LIST; + const bool child_emits_present_def_levels = + element_nullable || value_type_is_struct || value_type_is_nested_vector || + nested_vector_path; + const bool child_records_visited_elements = value_type_is_nested_vector; + const bool has_vector_node = parent_nullable || element_nullable || + value_type_is_struct || value_type_is_nested_vector || + nested_vector_path || info_.max_rep_level > 0; + const bool vector_node_emits_def_levels = + has_vector_node && !child_emits_present_def_levels; + if (has_vector_node) { + ARROW_ASSIGN_OR_RAISE(int32_t child_slot_multiplier, + VectorLeafSlotMultiplier(*array.value_type())); + info_.path.emplace_back(VectorNullableNode( + parent_nullable ? array.null_bitmap_data() : nullptr, array.offset(), + list_size, child_slot_multiplier, info_.max_def_level, info_.max_rep_level, + child_emits_present_def_levels, child_records_visited_elements, + /*expand_rep_levels=*/!nested_vector_path)); + } + auto values = + array.values()->Slice(array.value_offset(0), array.length() * list_size); + if (!nested_vector_path && !element_nullable && + !::arrow::is_nested(*array.value_type())) { + info_.leaf_is_nullable = false; + info_.primitive_array = values; + if (!vector_node_emits_def_levels && info_.max_rep_level > 0 && + info_.max_def_level > 0) { + info_.path.emplace_back(AllPresentTerminalNode{info_.max_def_level}); + } else { + info_.path.emplace_back(NoLevelTerminalNode{}); + } + paths_.push_back(Fixup(info_)); + return Status::OK(); + } + nullable_in_parent_ = element_nullable; + const bool saved_nullable_group = nullable_group_since_repeated_; + nullable_group_since_repeated_ = false; + Status status = VisitInline(*values); + nullable_group_since_repeated_ = saved_nullable_group; + return status; + } + + MaybeAddNullable(array); // Technically we could encode fixed size lists with two level encodings // but since we always use 3 level encoding we increment def levels as // well. info_.max_def_level++; info_.max_rep_level++; + info_.repeated_ancestor_def_level = info_.max_def_level; + // def_level_if_empty is max_def_level - 1 ("list present but empty"), not + // max_def_level ("element present"); it is only reachable for zero-length + // fixed-size lists, where every present value is an empty list. info_.path.emplace_back(FixedSizeListNode(FixedSizedRangeSelector{list_size}, - info_.max_rep_level, info_.max_def_level)); + info_.max_rep_level, + info_.max_def_level - 1)); nullable_in_parent_ = array.list_type()->value_field()->nullable(); - if (array.offset() > 0) { - return VisitInline(*array.values()->Slice(array.value_offset(0))); - } - return VisitInline(*array.values()); + const bool saved_nullable_group = nullable_group_since_repeated_; + nullable_group_since_repeated_ = false; + Status status = array.offset() > 0 + ? VisitInline(*array.values()->Slice(array.value_offset(0))) + : VisitInline(*array.values()); + nullable_group_since_repeated_ = saved_nullable_group; + return status; } Status Visit(const ::arrow::ExtensionArray& array) { @@ -876,6 +1184,12 @@ class PathBuilder { PathInfo info_; std::vector paths_; bool nullable_in_parent_; + bool write_fixed_size_list_as_vector_; + // True when a nullable group (struct) sits between the nearest repeated + // ancestor (or the root) and the current position. The writer uses LIST for + // such FixedSizeList fields so it does not have to expand null struct markers + // to vector_length leaf slots. + bool nullable_group_since_repeated_ = false; }; Status PathBuilder::VisitInline(const Array& array) { @@ -919,8 +1233,10 @@ class MultipathLevelBuilderImpl : public MultipathLevelBuilder { // static ::arrow::Result> MultipathLevelBuilder::Make( - const ::arrow::Array& array, bool array_field_nullable) { - auto constructor = std::make_unique(array_field_nullable); + const ::arrow::Array& array, bool array_field_nullable, + bool write_fixed_size_list_as_vector) { + auto constructor = std::make_unique(array_field_nullable, + write_fixed_size_list_as_vector); RETURN_NOT_OK(VisitArrayInline(array, constructor.get())); return std::make_unique(array.data(), std::move(constructor)); @@ -930,8 +1246,12 @@ ::arrow::Result> MultipathLevelBuilder::M Status MultipathLevelBuilder::Write(const Array& array, bool array_field_nullable, ArrowWriteContext* context, MultipathLevelBuilder::CallbackFunction callback) { + const bool write_fixed_size_list_as_vector = + context->properties != nullptr && + context->properties->write_fixed_size_list_as_vector(); ARROW_ASSIGN_OR_RAISE(std::unique_ptr builder, - MultipathLevelBuilder::Make(array, array_field_nullable)); + MultipathLevelBuilder::Make(array, array_field_nullable, + write_fixed_size_list_as_vector)); for (int leaf_idx = 0; leaf_idx < builder->GetLeafCount(); leaf_idx++) { RETURN_NOT_OK(builder->Write(leaf_idx, context, callback)); } diff --git a/cpp/src/parquet/arrow/path_internal.h b/cpp/src/parquet/arrow/path_internal.h index 50d2bf24291a..e2158b4ad6af 100644 --- a/cpp/src/parquet/arrow/path_internal.h +++ b/cpp/src/parquet/arrow/path_internal.h @@ -94,6 +94,32 @@ struct MultipathLevelBuilderResult { /// Whether the leaf array is nullable. bool leaf_is_nullable; + + /// Whether this leaf is produced from an Arrow FixedSizeList being written as + /// Parquet VECTOR. For nullable VECTOR elements the physical Parquet leaf is + /// below the VECTOR node, so checking only the primitive schema node is not + /// sufficient. + bool leaf_is_vector = false; + + /// Fixed number of physical leaf slots per logical VECTOR value for this leaf. + int32_t leaf_vector_length = 1; + + /// Number of physical leaf slots for a VECTOR leaf: the definition levels at + /// or above the vector's stride definition level. Equals def_rep_level_count + /// unless ancestors made some vector values absent (empty or null lists, null + /// structs), which contribute one definition level and no slot. + int64_t leaf_slot_count = 0; + + /// Definition level after the last repeated ancestor of a VECTOR leaf (0 + /// when there is none). Definition levels below it mark absent ancestors + /// (empty or null lists) with no slot. + int16_t vector_repeated_ancestor_def_level = 0; + + /// Definition level at or above which a VECTOR leaf slot carries a value + /// from the leaf array; slots between the repeated-ancestor level and this + /// level belong to null vector values at some nesting depth and carry no + /// value. + int16_t vector_values_def_level = 0; }; /// \brief Logic for being able to write out nesting (rep/def level) data that is @@ -132,7 +158,8 @@ class PARQUET_EXPORT MultipathLevelBuilder { /// the array column as nullable (as determined by its type's parent /// field). static ::arrow::Result> Make( - const ::arrow::Array& array, bool array_field_nullable); + const ::arrow::Array& array, bool array_field_nullable, + bool write_fixed_size_list_as_vector = false); virtual ~MultipathLevelBuilder() = default; diff --git a/cpp/src/parquet/arrow/path_internal_test.cc b/cpp/src/parquet/arrow/path_internal_test.cc index 2daa94e6226c..69640f118a87 100644 --- a/cpp/src/parquet/arrow/path_internal_test.cc +++ b/cpp/src/parquet/arrow/path_internal_test.cc @@ -576,6 +576,88 @@ TEST_F(MultipathLevelBuilderTest, TestFixedSizeList) { EXPECT_THAT(results_[0].post_list_elements[0].end, Eq(6)); } +TEST_F(MultipathLevelBuilderTest, TestFixedSizeListExperimentalVector) { + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + arrow_properties_ = builder.build(); + context_ = ArrowWriteContext(default_memory_pool(), arrow_properties_.get()); + + auto entries = field("Entries", ::arrow::int64(), /*nullable=*/false); + auto list_type = fixed_size_list(entries, 2); + auto array = ArrayFromJSON(list_type, "[[1, 2], [3, 4], [5, 6]]"); + + ASSERT_OK( + MultipathLevelBuilder::Write(*array, /*nullable=*/false, &context_, callback_)); + ASSERT_THAT(results_, SizeIs(1)); + EXPECT_TRUE(results_[0].null_rep_levels); + EXPECT_TRUE(results_[0].null_def_levels); + ASSERT_THAT(results_[0].post_list_elements, SizeIs(1)); + EXPECT_EQ(results_[0].post_list_elements[0].start, 0); + EXPECT_EQ(results_[0].post_list_elements[0].end, 6); +} + +TEST_F(MultipathLevelBuilderTest, TestFixedSizeListExperimentalVectorNullableElements) { + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + arrow_properties_ = builder.build(); + context_ = ArrowWriteContext(default_memory_pool(), arrow_properties_.get()); + + auto entries = field("Entries", ::arrow::int64(), /*nullable=*/true); + auto list_type = fixed_size_list(entries, 2); + auto array = ArrayFromJSON(list_type, "[[1, null], [3, 4], [null, 6]]"); + + ASSERT_OK( + MultipathLevelBuilder::Write(*array, /*nullable=*/false, &context_, callback_)); + ASSERT_THAT(results_, SizeIs(1)); + results_[0].CheckLevelsWithNullRepLevels(std::vector{1, 0, 1, 1, 0, 1}); + ASSERT_THAT(results_[0].post_list_elements, SizeIs(1)); + EXPECT_EQ(results_[0].post_list_elements[0].start, 0); + EXPECT_EQ(results_[0].post_list_elements[0].end, 6); +} + +TEST_F(MultipathLevelBuilderTest, + TestFixedSizeListExperimentalVectorNullableRowsAndElements) { + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + arrow_properties_ = builder.build(); + context_ = ArrowWriteContext(default_memory_pool(), arrow_properties_.get()); + + auto entries = field("Entries", ::arrow::int64(), /*nullable=*/true); + auto list_type = fixed_size_list(entries, 2); + auto array = ArrayFromJSON(list_type, "[[1, null], null, [null, 6]]"); + + ASSERT_OK( + MultipathLevelBuilder::Write(*array, /*nullable=*/true, &context_, callback_)); + ASSERT_THAT(results_, SizeIs(1)); + results_[0].CheckLevelsWithNullRepLevels(std::vector{2, 1, 0, 0, 1, 2}); + ASSERT_THAT(results_[0].post_list_elements, SizeIs(2)); + EXPECT_EQ(results_[0].post_list_elements[0].start, 0); + EXPECT_EQ(results_[0].post_list_elements[0].end, 2); + EXPECT_EQ(results_[0].post_list_elements[1].start, 4); + EXPECT_EQ(results_[0].post_list_elements[1].end, 6); +} + +TEST_F(MultipathLevelBuilderTest, TestFixedSizeListExperimentalVectorNullable) { + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + arrow_properties_ = builder.build(); + context_ = ArrowWriteContext(default_memory_pool(), arrow_properties_.get()); + + auto entries = field("Entries", ::arrow::int64(), /*nullable=*/false); + auto list_type = fixed_size_list(entries, 2); + auto array = ArrayFromJSON(list_type, "[[1, 2], null, [5, 6]]"); + + ASSERT_OK( + MultipathLevelBuilder::Write(*array, /*nullable=*/true, &context_, callback_)); + ASSERT_THAT(results_, SizeIs(1)); + results_[0].CheckLevelsWithNullRepLevels(std::vector{1, 1, 0, 0, 1, 1}); + ASSERT_THAT(results_[0].post_list_elements, SizeIs(2)); + EXPECT_EQ(results_[0].post_list_elements[0].start, 0); + EXPECT_EQ(results_[0].post_list_elements[0].end, 2); + EXPECT_EQ(results_[0].post_list_elements[1].start, 4); + EXPECT_EQ(results_[0].post_list_elements[1].end, 6); +} + TEST_F(MultipathLevelBuilderTest, TestFixedSizeListMissingMiddleHasTwoVisitedRanges) { auto entries = field("Entries", ::arrow::int64(), /*nullable=*/false); auto list_type = fixed_size_list(entries, 2); diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc index cc107c1802e3..4a9973be0bf7 100644 --- a/cpp/src/parquet/arrow/reader.cc +++ b/cpp/src/parquet/arrow/reader.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -26,6 +27,8 @@ #include #include "arrow/array.h" +#include "arrow/array/concatenate.h" +#include "arrow/array/util.h" #include "arrow/buffer.h" #include "arrow/extension_type.h" #include "arrow/io/memory.h" @@ -138,6 +141,16 @@ std::shared_ptr> VectorToSharedSet( return result; } +Result MultiplyVectorLength(int64_t value, int32_t list_size) { + if (list_size <= 0) { + return Status::Invalid("VECTOR FixedSizeList must have positive list_size"); + } + if (value > std::numeric_limits::max() / list_size) { + return Status::Invalid("VECTOR FixedSizeList length overflow"); + } + return value * list_size; +} + // Forward declaration Status GetReader(const SchemaField& field, const std::shared_ptr& context, std::unique_ptr* out); @@ -274,10 +287,24 @@ class FileReaderImpl : public FileReader { // TODO(wesm): This calculation doesn't make much sense when we have repeated // schema nodes int64_t records_to_read = 0; + DCHECK(dynamic_cast(reader) != nullptr); + const bool has_repeated_child = + static_cast(reader)->IsOrHasRepeatedChild(); for (auto row_group : row_groups) { - // Can throw exception - records_to_read += - reader_->metadata()->RowGroup(row_group)->ColumnChunk(i)->num_values(); + // Can throw exception. The column chunk metadata is deserialized + // unconditionally to keep upstream behavior (for example its decryption + // side effects for encrypted columns). + auto row_group_metadata = reader_->metadata()->RowGroup(row_group); + auto column_chunk = row_group_metadata->ColumnChunk(i); + // ColumnReader::NextBatch takes logical parent records. For readers + // without repeated children (flat and VECTOR columns) the row group row + // count is an exact bound; for VECTOR columns in particular, ColumnChunk + // num_values would over-count records by the vector length. + if (has_repeated_child) { + records_to_read += column_chunk->num_values(); + } else { + records_to_read += row_group_metadata->num_rows(); + } } #ifdef ARROW_WITH_OPENTELEMETRY std::string column_name = reader_->metadata()->schema()->Column(i)->name(); @@ -739,6 +766,344 @@ class PARQUET_NO_EXPORT FixedSizeListReader : public ListReader { } }; +// Reads Parquet VECTOR columns into Arrow FixedSizeList arrays. +// +// VECTOR stores one definition level per element even though the public Arrow result is a +// single FixedSizeList slot per row. For nullable VECTOR rows, the child reader therefore +// materializes spaced child slots and this reader collapses each vector's per-element def +// levels back into a parent validity bitmap. +class PARQUET_NO_EXPORT VectorFixedSizeListReader : public ColumnReaderImpl { + public: + VectorFixedSizeListReader(std::shared_ptr ctx, + std::shared_ptr field, + ::parquet::internal::LevelInfo level_info, + int16_t vector_stride_def_level, + std::unique_ptr child_reader) + : ctx_(std::move(ctx)), + field_(std::move(field)), + level_info_(level_info), + vector_stride_def_level_(vector_stride_def_level), + item_reader_(std::move(child_reader)), + list_size_(checked_cast(*field_->type()) + .list_size()) {} + + Status GetDefLevels(const int16_t** data, int64_t* length) override { + if (collapsed_def_levels_.empty()) { + *data = nullptr; + *length = vector_rows_; + } else { + *data = collapsed_def_levels_.data(); + *length = static_cast(collapsed_def_levels_.size()); + } + return Status::OK(); + } + + Status GetRepLevels(const int16_t** data, int64_t* length) override { + if (collapsed_rep_levels_.empty()) { + *data = nullptr; + *length = vector_rows_; + } else { + *data = collapsed_rep_levels_.data(); + *length = static_cast(collapsed_rep_levels_.size()); + } + return Status::OK(); + } + + bool IsOrHasRepeatedChild() const final { return false; } + + Status LoadBatch(int64_t number_of_records) final { + vector_rows_ = 0; + collapsed_def_levels_.clear(); + collapsed_rep_levels_.clear(); + vector_def_levels_.clear(); + int64_t child_records_to_read = number_of_records; + if (level_info_.rep_level == 0) { + ARROW_ASSIGN_OR_RAISE(child_records_to_read, + MultiplyVectorLength(number_of_records, list_size_)); + } + RETURN_NOT_OK(item_reader_->LoadBatch(child_records_to_read)); + + const int16_t* def_levels = nullptr; + int64_t num_levels = 0; + RETURN_NOT_OK(item_reader_->GetDefLevels(&def_levels, &num_levels)); + const int16_t* rep_levels = nullptr; + int64_t num_rep_levels = 0; + RETURN_NOT_OK(item_reader_->GetRepLevels(&rep_levels, &num_rep_levels)); + const bool collapse_rep_levels = level_info_.rep_level > 0 && rep_levels != nullptr; + if (collapse_rep_levels && num_rep_levels != num_levels) { + return Status::Invalid("VECTOR child produced ", num_rep_levels, + " repetition levels for ", num_levels, " definition levels"); + } + if (def_levels == nullptr || num_levels == 0) { + // No level information; the row count comes from the child array length. + return Status::OK(); + } + + // Existing vector values (present, or null when the vector field is + // nullable) occupy exactly list_size_ definition levels. Levels below + // the vector stride level mark ancestors that made the vector value absent + // (empty/null lists or null structs); they occupy a single level entry with + // no physical VECTOR slots and pass through unchanged for the parent reader. + const int16_t stride_def_level = vector_stride_def_level_; + const size_t reserve_hint = static_cast(num_levels / list_size_) + 1; + collapsed_def_levels_.reserve(reserve_hint); + vector_def_levels_.reserve(reserve_hint); + if (collapse_rep_levels) { + collapsed_rep_levels_.reserve(reserve_hint); + } + int64_t i = 0; + while (i < num_levels) { + const int16_t def = def_levels[i]; + // Definition levels arrive in long runs (RLE on disk), so scan the + // maximal run of identical levels and process it in bulk. + int64_t run_end = i + 1; + while (run_end < num_levels && def_levels[run_end] == def) { + ++run_end; + } + if (def < stride_def_level) { + // A run of absent-ancestor markers passes through one-to-one. + const int64_t run_length = run_end - i; + collapsed_def_levels_.insert(collapsed_def_levels_.end(), run_length, def); + if (collapse_rep_levels) { + collapsed_rep_levels_.insert(collapsed_rep_levels_.end(), rep_levels + i, + rep_levels + run_end); + } + i = run_end; + continue; + } + if (i + list_size_ > num_levels) { + return Status::Invalid("VECTOR column ended mid-vector: ", num_levels - i, + " definition levels remaining for list_size=", list_size_); + } + const bool is_present = def >= level_info_.def_level; + const int16_t collapsed = is_present ? level_info_.def_level : def; + const int64_t run_vectors = (run_end - i) / list_size_; + if (run_vectors > 0) { + // Whole vectors covered by one run of equal levels: equal levels imply + // uniform parent validity, so the per-slot validation is subsumed. + collapsed_def_levels_.insert(collapsed_def_levels_.end(), run_vectors, collapsed); + vector_def_levels_.insert(vector_def_levels_.end(), run_vectors, collapsed); + if (collapse_rep_levels) { + for (int64_t v = 0; v < run_vectors; ++v) { + collapsed_rep_levels_.push_back(rep_levels[i + v * list_size_]); + } + } + vector_rows_ += run_vectors; + i += run_vectors * list_size_; + continue; + } + // The run ends inside this vector (for example nullable elements mix + // definition levels); validate parent-validity uniformity per slot. + for (int32_t k = 1; k < list_size_; ++k) { + const bool slot_is_present = def_levels[i + k] >= level_info_.def_level; + if (slot_is_present != is_present) { + return Status::Invalid( + "VECTOR parent validity changed within one fixed-size vector at row ", + vector_rows_, + "; null VECTOR rows must still emit exactly list_size null child " + "slots and all slots for one parent must agree on parent " + "validity"); + } + } + collapsed_def_levels_.push_back(collapsed); + vector_def_levels_.push_back(collapsed); + if (collapse_rep_levels) { + collapsed_rep_levels_.push_back(rep_levels[i]); + } + ++vector_rows_; + i += list_size_; + } + return Status::OK(); + } + + Status BuildArray(int64_t length_upper_bound, + std::shared_ptr* out) override { + std::shared_ptr child_out; + ARROW_ASSIGN_OR_RAISE(const int64_t child_length_upper_bound, + MultiplyVectorLength(length_upper_bound, list_size_)); + RETURN_NOT_OK(item_reader_->BuildArray(child_length_upper_bound, &child_out)); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr child_data, + ChunksToSingle(*child_out)); + std::shared_ptr child_array = ::arrow::MakeArray(child_data); + if (vector_rows_ == 0 && vector_def_levels_.empty()) { + // No level information was loaded; derive the row count from the child. + if (child_array->length() % list_size_ != 0) { + return Status::Invalid("VECTOR FixedSizeList child length ", + child_array->length(), + " was not divisible by list_size=", list_size_); + } + vector_rows_ = child_array->length() / list_size_; + } + + int64_t output_rows = vector_rows_; + std::vector expanded_def_levels; + // Nullable struct ancestors can contribute a single absent level marker for + // a VECTOR value whose Arrow parent still requires child slots. Expand such + // markers here, while leaving empty/null list markers in their compact form. + const std::vector* output_def_levels = &vector_def_levels_; + int64_t expandable_absent_count = 0; + for (const int16_t def_level : collapsed_def_levels_) { + if (def_level >= level_info_.repeated_ancestor_def_level && + def_level < vector_stride_def_level_) { + ++expandable_absent_count; + } + } + // The expansion only triggers below struct (or root) parents: a list + // parent requests exactly the existing vector values, so + // length_upper_bound never exceeds vector_rows_ there, while a struct + // parent requests one output row per logical entry including nullable + // struct markers. + const bool expand_absent_ancestors = + expandable_absent_count > 0 && length_upper_bound > vector_rows_; + if (expand_absent_ancestors) { + const int64_t extra_rows = length_upper_bound - vector_rows_; + if (extra_rows % expandable_absent_count != 0) { + return Status::Invalid("VECTOR FixedSizeList cannot evenly expand ", + expandable_absent_count, " absent ancestor markers to ", + extra_rows, " child rows"); + } + const int64_t rows_per_absent = extra_rows / expandable_absent_count; + // The child array may take one of three shapes, depending on the child + // reader that produced it: + // - exactly the slots of existing vectors (a nested VECTOR reader that + // saw no expandable markers); + // - one placeholder slot per expandable marker in addition (a leaf reader + // without repeated ancestors materializes a slot for every definition + // level entry); + // - rows_per_absent * list_size_ null slots per expandable marker in + // addition (a nested reader that has itself expanded the markers). + ARROW_ASSIGN_OR_RAISE(const int64_t expected_child_slots, + MultiplyVectorLength(vector_rows_, list_size_)); + if (expected_child_slots > + std::numeric_limits::max() - expandable_absent_count) { + return Status::Invalid("VECTOR FixedSizeList length overflow"); + } + const int64_t expected_child_slots_with_placeholders = + expected_child_slots + expandable_absent_count; + ARROW_ASSIGN_OR_RAISE(const int64_t materialized_absent_slots, + MultiplyVectorLength(extra_rows, list_size_)); + if (expected_child_slots > + std::numeric_limits::max() - materialized_absent_slots) { + return Status::Invalid("VECTOR FixedSizeList length overflow"); + } + const int64_t expected_child_slots_with_materialized_absent = + expected_child_slots + materialized_absent_slots; + const bool child_has_absent_placeholders = + child_array->length() == expected_child_slots_with_placeholders; + const bool child_has_materialized_absent_slots = + child_array->length() == expected_child_slots_with_materialized_absent; + if (child_array->length() != expected_child_slots && + !child_has_absent_placeholders && !child_has_materialized_absent_slots) { + return Status::Invalid("VECTOR FixedSizeList child length ", + child_array->length(), " did not match expected ", + expected_child_slots); + } + + ::arrow::ArrayVector parts; + parts.reserve(static_cast(length_upper_bound)); + expanded_def_levels.reserve(static_cast(length_upper_bound)); + ARROW_ASSIGN_OR_RAISE(const int64_t null_slot_count, + MultiplyVectorLength(rows_per_absent, list_size_)); + int64_t child_offset = 0; + for (const int16_t def_level : collapsed_def_levels_) { + if (def_level < vector_stride_def_level_) { + if (def_level < level_info_.repeated_ancestor_def_level) { + continue; + } + if (child_has_absent_placeholders) { + ++child_offset; + } else if (child_has_materialized_absent_slots) { + child_offset += null_slot_count; + } + ARROW_ASSIGN_OR_RAISE( + auto null_values, + ::arrow::MakeArrayOfNull(child_array->type(), null_slot_count, ctx_->pool)); + parts.push_back(std::move(null_values)); + expanded_def_levels.insert(expanded_def_levels.end(), rows_per_absent, + def_level); + } else { + parts.push_back(child_array->Slice(child_offset, list_size_)); + child_offset += list_size_; + expanded_def_levels.push_back(def_level); + } + } + if (child_offset != child_array->length()) { + return Status::Invalid( + "VECTOR FixedSizeList child values out of sync with " + "definition levels"); + } + ARROW_ASSIGN_OR_RAISE(child_array, ::arrow::Concatenate(parts, ctx_->pool)); + child_data = child_array->data(); + output_rows = length_upper_bound; + output_def_levels = &expanded_def_levels; + } else { + ARROW_ASSIGN_OR_RAISE(const int64_t expected_child_slots, + MultiplyVectorLength(vector_rows_, list_size_)); + if (child_array->length() != expected_child_slots) { + return Status::Invalid("VECTOR FixedSizeList child length ", + child_array->length(), " did not match expected ", + expected_child_slots); + } + } + + ARROW_ASSIGN_OR_RAISE(const int64_t expected_output_child_slots, + MultiplyVectorLength(output_rows, list_size_)); + if (child_array->length() != expected_output_child_slots) { + return Status::Invalid("VECTOR FixedSizeList child length ", child_array->length(), + " did not match expected ", expected_output_child_slots); + } + + std::shared_ptr validity_buffer; + int64_t null_count = 0; + if (field_->nullable()) { + ARROW_ASSIGN_OR_RAISE( + validity_buffer, + AllocateResizableBuffer(bit_util::BytesForBits(output_rows), ctx_->pool)); + memset(validity_buffer->mutable_data(), 0, + static_cast(bit_util::BytesForBits(output_rows))); + if (output_def_levels->empty()) { + bit_util::SetBitsTo(validity_buffer->mutable_data(), 0, output_rows, true); + } else { + for (int64_t row = 0; row < output_rows; ++row) { + if ((*output_def_levels)[row] == level_info_.def_level) { + bit_util::SetBit(validity_buffer->mutable_data(), row); + } else { + ++null_count; + } + } + } + validity_buffer->ZeroPadding(); + } + + auto data = std::make_shared( + field_->type(), output_rows, + std::vector>{null_count > 0 ? validity_buffer : nullptr}, + null_count); + data->child_data.push_back(child_data); + *out = std::make_shared(::arrow::MakeArray(std::move(data))); + return Status::OK(); + } + + const std::shared_ptr field() override { return field_; } + + private: + std::shared_ptr ctx_; + std::shared_ptr field_; + ::parquet::internal::LevelInfo level_info_; + int16_t vector_stride_def_level_; + std::unique_ptr item_reader_; + int32_t list_size_; + // Number of existing vector values (present or null); absent ancestors do + // not count. + int64_t vector_rows_ = 0; + // One entry per logical entry (existing vector or absent ancestor), for the + // parent reader. + std::vector collapsed_def_levels_; + std::vector collapsed_rep_levels_; + // One entry per existing vector value, for this reader's validity bitmap. + std::vector vector_def_levels_; +}; + class PARQUET_NO_EXPORT StructReader : public ColumnReaderImpl { public: explicit StructReader(std::shared_ptr ctx, @@ -917,8 +1282,12 @@ Status GetReader(const SchemaField& field, const std::shared_ptr& arrow_f } std::unique_ptr input( ctx->iterator_factory(field.column_index, ctx->reader)); - *out = std::make_unique(ctx, arrow_field, std::move(input), - field.level_info); + auto leaf_field = arrow_field; + if (field.is_vector && field.level_info.def_level > 0) { + leaf_field = leaf_field->WithNullable(true); + } + *out = + std::make_unique(ctx, leaf_field, std::move(input), field.level_info); } else if (type_id == ::arrow::Type::LIST || type_id == ::arrow::Type::MAP || type_id == ::arrow::Type::FIXED_SIZE_LIST || type_id == ::arrow::Type::LARGE_LIST || @@ -999,8 +1368,17 @@ Status GetReader(const SchemaField& field, const std::shared_ptr& arrow_f list_field->WithType(::arrow::fixed_size_list(reader_child_type, list_size)); } - *out = std::make_unique(ctx, list_field, field.level_info, - std::move(child_reader)); + if (field.is_vector) { + if (child->is_leaf() && child->column_index >= 0 && list_field->nullable()) { + DCHECK(child_reader->field()->nullable()); + } + *out = std::make_unique( + ctx, list_field, field.level_info, field.vector_stride_def_level, + std::move(child_reader)); + } else { + *out = std::make_unique(ctx, list_field, field.level_info, + std::move(child_reader)); + } } else { return Status::UnknownError("Unknown list type: ", field.field->ToString()); } diff --git a/cpp/src/parquet/arrow/reader_writer_benchmark.cc b/cpp/src/parquet/arrow/reader_writer_benchmark.cc index 2f288fd2eb0f..63d196892ece 100644 --- a/cpp/src/parquet/arrow/reader_writer_benchmark.cc +++ b/cpp/src/parquet/arrow/reader_writer_benchmark.cc @@ -812,5 +812,84 @@ static void BM_ReadMultipleRowGroupsGenerator(::benchmark::State& state) { BENCHMARK(BM_ReadMultipleRowGroupsGenerator); +// +// Benchmark writing and reading fixed-size list (embedding) columns with the +// experimental VECTOR encoding versus the standard LIST encoding. +// + +constexpr int32_t kVectorBenchmarkDim = 128; + +static std::shared_ptr
MakeFixedSizeListTable(bool nullable) { + const int64_t num_rows = BENCHMARK_SIZE / kVectorBenchmarkDim; + auto type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::float32(), /*nullable=*/false), + kVectorBenchmarkDim); + ::arrow::random::RandomArrayGenerator generator(/*seed=*/42); + auto array = generator.ArrayOf(type, num_rows, nullable ? 0.05 : 0.0); + auto schema = ::arrow::schema({::arrow::field("column", type, nullable)}); + return Table::Make(schema, {array}); +} + +static std::shared_ptr VectorBenchmarkArrowProperties( + bool use_vector) { + ArrowWriterProperties::Builder builder; + builder.store_schema(); + if (use_vector) { + builder.enable_experimental_vector_encoding(); + } + return builder.build(); +} + +template +static void BM_WriteFixedSizeListColumn(::benchmark::State& state) { + auto table = MakeFixedSizeListTable(nullable); + auto arrow_properties = VectorBenchmarkArrowProperties(use_vector); + + while (state.KeepRunning()) { + auto output = CreateOutputStream(); + EXIT_NOT_OK(WriteTable(*table, ::arrow::default_memory_pool(), output, + table->num_rows(), default_writer_properties(), + arrow_properties)); + } + state.SetBytesProcessed(BENCHMARK_SIZE * sizeof(float) * state.iterations()); + state.SetItemsProcessed(table->num_rows() * state.iterations()); +} + +BENCHMARK_TEMPLATE2(BM_WriteFixedSizeListColumn, false, false); +BENCHMARK_TEMPLATE2(BM_WriteFixedSizeListColumn, true, false); +// There is no nullable LIST-encoding baseline: writing null FixedSizeList +// rows through the LIST encoding is NotImplemented upstream ("Lists with +// non-zero length null components"); the VECTOR encoding supports them. +BENCHMARK_TEMPLATE2(BM_WriteFixedSizeListColumn, true, true); + +template +static void BM_ReadFixedSizeListColumn(::benchmark::State& state) { + auto table = MakeFixedSizeListTable(nullable); + auto arrow_properties = VectorBenchmarkArrowProperties(use_vector); + + auto output = CreateOutputStream(); + EXIT_NOT_OK(WriteTable(*table, ::arrow::default_memory_pool(), output, + table->num_rows(), default_writer_properties(), + arrow_properties)); + PARQUET_ASSIGN_OR_THROW(auto buffer, output->Finish()); + + for (auto _ : state) { + auto reader = + ParquetFileReader::Open(std::make_shared<::arrow::io::BufferReader>(buffer)); + auto arrow_reader_result = + FileReader::Make(::arrow::default_memory_pool(), std::move(reader)); + EXIT_NOT_OK(arrow_reader_result.status()); + auto arrow_reader = std::move(*arrow_reader_result); + auto table_result = arrow_reader->ReadTable(); + EXIT_NOT_OK(table_result.status()); + } + state.SetBytesProcessed(BENCHMARK_SIZE * sizeof(float) * state.iterations()); + state.SetItemsProcessed(table->num_rows() * state.iterations()); +} + +BENCHMARK_TEMPLATE2(BM_ReadFixedSizeListColumn, false, false); +BENCHMARK_TEMPLATE2(BM_ReadFixedSizeListColumn, true, false); +BENCHMARK_TEMPLATE2(BM_ReadFixedSizeListColumn, true, true); + } // namespace benchmarks } // namespace parquet diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc index bc4de6c39b5e..5aa18c6fc5a1 100644 --- a/cpp/src/parquet/arrow/schema.cc +++ b/cpp/src/parquet/arrow/schema.cc @@ -31,6 +31,7 @@ #include "arrow/result.h" #include "arrow/status.h" #include "arrow/type.h" +#include "arrow/type_traits.h" #include "arrow/util/base64.h" #include "arrow/util/checked_cast.h" #include "arrow/util/key_value_metadata.h" @@ -92,9 +93,14 @@ Result> MakeArrowList( } } +// nullable_group_since_repeated is true when a nullable group (struct) sits +// between the nearest repeated ancestor (or the root) and the field being +// converted. The writer uses LIST for such FixedSizeList fields so it does +// not have to expand null struct markers to vector_length leaf slots. Status FieldToNode(const std::string& name, const std::shared_ptr& field, const WriterProperties& properties, - const ArrowWriterProperties& arrow_properties, NodePtr* out); + const ArrowWriterProperties& arrow_properties, NodePtr* out, + bool nullable_group_since_repeated = false); Status ListToNode(const std::shared_ptr<::arrow::BaseListType>& type, const std::string& name, bool nullable, int field_id, @@ -112,6 +118,86 @@ Status ListToNode(const std::shared_ptr<::arrow::BaseListType>& type, return Status::OK(); } +bool IsSupportedVectorStructNode(const Node& node) { + if (node.is_primitive()) { + return true; + } + if (!node.is_group() || node.is_repeated()) { + return false; + } + const auto& group = checked_cast(node); + if (group.logical_type() != nullptr && !group.logical_type()->is_none()) { + return false; + } + for (int i = 0; i < group.field_count(); ++i) { + if (!IsSupportedVectorStructNode(*group.field(i))) { + return false; + } + } + return true; +} + +Status ValidateSupportedVectorStructNode(const Node& node) { + if (IsSupportedVectorStructNode(node)) { + return Status::OK(); + } + return Status::NotImplemented( + "VECTOR elements only support primitive, nested VECTOR, or nested struct fields; " + "repeated/list/map descendants are not supported"); +} + +// Option B mapping for Arrow FixedSizeList -> Parquet VECTOR. +// +// The schema always uses a 3-level structure mirroring LIST, regardless of +// vector and element nullability: +// +// group (VECTOR) { +// vector group list [N] { +// element; +// } +// } +// +// VECTOR is a structural repetition type: vector nullability is carried by the +// outer group, the VECTOR-repeated middle group carries vector_length and +// contributes no repetition/definition level, and element nullability is +// carried by the element node. The outer group is annotated with the VECTOR +// logical type to make it explicit that it represents a vector value rather +// than, e.g., a single-field struct containing a vector, mirroring how LIST +// annotates list-shaped groups. +Status FixedSizeListToNode(const std::shared_ptr<::arrow::FixedSizeListType>& type, + const std::string& name, bool nullable, int field_id, + const WriterProperties& properties, + const ArrowWriterProperties& arrow_properties, NodePtr* out) { + if (type->list_size() <= 0) { + return Status::NotImplemented( + "VECTOR repetition does not support zero-length FixedSizeList values"); + } + if (!IsSupportedVectorElementType(*type->value_type())) { + return Status::NotImplemented( + "VECTOR repetition only supports fixed-width primitive, nested FixedSizeList, " + "or struct FixedSizeList elements"); + } + auto leaf_slot_multiplier = VectorLeafSlotMultiplier(*type); + if (!leaf_slot_multiplier.ok()) { + return Status::NotImplemented(leaf_slot_multiplier.status().message()); + } + const auto& value_field = type->value_field(); + const std::string value_name = "element"; + + NodePtr element; + RETURN_NOT_OK( + FieldToNode(value_name, value_field, properties, arrow_properties, &element)); + if (type->value_type()->id() == ::arrow::Type::STRUCT) { + RETURN_NOT_OK(ValidateSupportedVectorStructNode(*element)); + } + NodePtr vector = + GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, /*field_id=*/-1, type->list_size()); + *out = GroupNode::Make(name, RepetitionFromNullable(nullable), {vector}, + LogicalType::Vector(), field_id); + return Status::OK(); +} + Status MapToNode(const std::shared_ptr<::arrow::MapType>& type, const std::string& name, bool nullable, int field_id, const WriterProperties& properties, const ArrowWriterProperties& arrow_properties, NodePtr* out) { @@ -154,12 +240,14 @@ Status VariantToNode( Status StructToNode(const std::shared_ptr<::arrow::StructType>& type, const std::string& name, bool nullable, int field_id, const WriterProperties& properties, - const ArrowWriterProperties& arrow_properties, NodePtr* out) { + const ArrowWriterProperties& arrow_properties, NodePtr* out, + bool nullable_group_since_repeated) { std::vector children(type->num_fields()); if (type->num_fields() != 0) { for (int i = 0; i < type->num_fields(); i++) { RETURN_NOT_OK(FieldToNode(type->field(i)->name(), type->field(i), properties, - arrow_properties, &children[i])); + arrow_properties, &children[i], + nullable_group_since_repeated || nullable)); } } else { // XXX (ARROW-10928) We could add a dummy primitive node but that would @@ -316,7 +404,8 @@ int FieldIdFromMetadata( Status FieldToNode(const std::string& name, const std::shared_ptr& field, const WriterProperties& properties, - const ArrowWriterProperties& arrow_properties, NodePtr* out) { + const ArrowWriterProperties& arrow_properties, NodePtr* out, + bool nullable_group_since_repeated) { std::shared_ptr logical_type = LogicalType::None(); ParquetType::type type; Repetition::type repetition = RepetitionFromNullable(field->nullable()); @@ -452,9 +541,34 @@ Status FieldToNode(const std::string& name, const std::shared_ptr& field, case ArrowTypeId::STRUCT: { auto struct_type = std::static_pointer_cast<::arrow::StructType>(field->type()); return StructToNode(struct_type, name, field->nullable(), field_id, properties, - arrow_properties, out); + arrow_properties, out, nullable_group_since_repeated); + } + case ArrowTypeId::FIXED_SIZE_LIST: { + auto list_type = + std::static_pointer_cast<::arrow::FixedSizeListType>(field->type()); + // Experimental VECTOR encoding is opportunistic: use it for supported + // FixedSizeList fields, and preserve writability by falling back to the + // standard LIST encoding for unsupported fields. This lets schemas mix + // VECTOR-friendly dense numeric/struct vectors with FixedSizeList values + // that still need LIST machinery (for example strings). + if (arrow_properties.write_fixed_size_list_as_vector() && + !nullable_group_since_repeated && list_type->list_size() > 0 && + IsSupportedVectorElementType(*list_type->value_type())) { + Status vector_status = + FixedSizeListToNode(list_type, name, field->nullable(), field_id, properties, + arrow_properties, out); + if (vector_status.ok()) { + return vector_status; + } + if (!vector_status.IsNotImplemented()) { + return vector_status; + } + } + auto base_list_type = + std::static_pointer_cast<::arrow::BaseListType>(field->type()); + return ListToNode(base_list_type, name, field->nullable(), field_id, properties, + arrow_properties, out); } - case ArrowTypeId::FIXED_SIZE_LIST: case ArrowTypeId::LARGE_LIST: case ArrowTypeId::LIST: case ArrowTypeId::LARGE_LIST_VIEW: @@ -470,7 +584,8 @@ Status FieldToNode(const std::string& name, const std::shared_ptr& field, static_cast(*field->type()); std::shared_ptr<::arrow::Field> unpacked_field = ::arrow::field( name, dict_type.value_type(), field->nullable(), field->metadata()); - return FieldToNode(name, unpacked_field, properties, arrow_properties, out); + return FieldToNode(name, unpacked_field, properties, arrow_properties, out, + nullable_group_since_repeated); } case ArrowTypeId::EXTENSION: { auto ext_type = std::static_pointer_cast<::arrow::ExtensionType>(field->type()); @@ -501,7 +616,8 @@ Status FieldToNode(const std::string& name, const std::shared_ptr& field, std::shared_ptr<::arrow::Field> storage_field = ::arrow::field( name, ext_type->storage_type(), field->nullable(), field->metadata()); - return FieldToNode(name, storage_field, properties, arrow_properties, out); + return FieldToNode(name, storage_field, properties, arrow_properties, out, + nullable_group_since_repeated); } case ArrowTypeId::MAP: { auto map_type = std::static_pointer_cast<::arrow::MapType>(field->type()); @@ -577,6 +693,14 @@ Status PopulateLeaf(int column_index, const std::shared_ptr& field, return Status::OK(); } +void MarkVectorSubtree(SchemaField* field, int16_t vector_stride_def_level) { + field->is_vector = true; + field->vector_stride_def_level = vector_stride_def_level; + for (auto& child : field->children) { + MarkVectorSubtree(&child, vector_stride_def_level); + } +} + // Special case mentioned in the format spec: // If the name is array or uses the parent's name with `_tuple` appended, // this should be: @@ -602,7 +726,7 @@ Status GroupToStruct(const GroupNode& node, LevelInfo current_levels, arrow_fields.push_back(out->children[i].field); } auto struct_type = ::arrow::struct_(arrow_fields); - if (ctx->properties.get_arrow_extensions_enabled() && + if (ctx->properties.get_arrow_extensions_enabled() && node.logical_type() != nullptr && node.logical_type()->is_variant()) { auto extension_type = ::arrow::GetExtensionType("arrow.parquet.variant"); if (extension_type) { @@ -621,6 +745,108 @@ Status ListToSchemaField(const GroupNode& group, LevelInfo current_levels, SchemaTreeContext* ctx, const SchemaField* parent, SchemaField* out); +bool IsVectorGroup(const GroupNode& group) { + // The VECTOR logical type annotation is required to identify a vector group; + // this is what distinguishes a vector field from a struct-like group that + // happens to contain a VECTOR-repeated child. An unannotated group with a + // VECTOR-repeated child is rejected during conversion (see + // GroupToSchemaField) rather than silently interpreted as a vector. + return group.logical_type() != nullptr && group.logical_type()->is_vector(); +} + +// Reconstructs an Arrow FixedSizeList from the canonical 3-level vector form: +// +// group (VECTOR) { +// vector group list [N] { +// element; +// } +// } +// +// The element may be a primitive, a struct group, or a nested vector group +// (itself in canonical form). +Status VectorToSchemaField(const GroupNode& group, LevelInfo current_levels, + SchemaTreeContext* ctx, const SchemaField* parent, + SchemaField* out) { + if (group.is_repeated() || group.is_vector()) { + return Status::Invalid("VECTOR logical groups must be required or optional"); + } + if (group.field_count() != 1) { + return Status::Invalid("VECTOR groups must have a single VECTOR-repeated child"); + } + const Node& child = *group.field(0); + if (!child.is_vector()) { + return Status::Invalid("VECTOR groups must contain a VECTOR-repeated child"); + } + if (!child.is_group()) { + return Status::Invalid( + "VECTOR-repeated nodes must be groups containing a single element field"); + } + const auto& vector_group = static_cast(child); + // Mirroring LIST, the writer names the VECTOR-repeated group "list" but + // readers do not depend on group names. + if (vector_group.logical_type() != nullptr && !vector_group.logical_type()->is_none()) { + return Status::Invalid("VECTOR-repeated groups must not have a logical type"); + } + if (vector_group.vector_length() <= 0) { + return Status::Invalid("VECTOR-repeated groups must have positive vector_length"); + } + if (vector_group.field_count() != 1) { + return Status::Invalid("VECTOR-repeated groups must have a single element child"); + } + const Node& element = *vector_group.field(0); + if (element.is_repeated() || element.is_vector()) { + return Status::Invalid("VECTOR elements must be required or optional"); + } + + const int16_t vector_stride_def_level = current_levels.def_level; + if (group.is_optional()) { + current_levels.IncrementOptional(); + } + const LevelInfo vector_level = current_levels; + + out->children.resize(1); + SchemaField* child_field = &out->children[0]; + ctx->LinkParent(out, parent); + ctx->LinkParent(child_field, out); + + if (element.is_primitive()) { + const bool element_nullable = element.is_optional(); + if (element_nullable) { + current_levels.IncrementOptional(); + } + const auto& primitive_node = static_cast(element); + int column_index = ctx->schema->GetColumnIndex(primitive_node); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr type, + GetTypeForNode(column_index, primitive_node, ctx)); + auto item_field = ::arrow::field(element.name(), type, element_nullable, + FieldIdMetadata(element.field_id())); + RETURN_NOT_OK( + PopulateLeaf(column_index, item_field, current_levels, ctx, out, child_field)); + } else { + const auto& element_group = static_cast(element); + if (IsVectorGroup(element_group)) { + // Nested vector element; the recursive call handles element nullability. + RETURN_NOT_OK( + VectorToSchemaField(element_group, current_levels, ctx, out, child_field)); + } else { + RETURN_NOT_OK(ValidateSupportedVectorStructNode(element_group)); + if (element_group.is_optional()) { + current_levels.IncrementOptional(); + } + RETURN_NOT_OK(GroupToStruct(element_group, current_levels, ctx, out, child_field)); + } + } + + MarkVectorSubtree(child_field, vector_stride_def_level); + out->field = ::arrow::field( + group.name(), ::arrow::fixed_size_list(child_field->field, child.vector_length()), + group.is_optional(), FieldIdMetadata(group.field_id())); + out->level_info = vector_level; + out->is_vector = true; + out->vector_stride_def_level = vector_stride_def_level; + return Status::OK(); +} + Status MapToSchemaField(const GroupNode& group, LevelInfo current_levels, SchemaTreeContext* ctx, const SchemaField* parent, SchemaField* out) { @@ -845,6 +1071,14 @@ Status ListToSchemaField(const GroupNode& group, LevelInfo current_levels, Status GroupToSchemaField(const GroupNode& node, LevelInfo current_levels, SchemaTreeContext* ctx, const SchemaField* parent, SchemaField* out) { + if (IsVectorGroup(node)) { + return VectorToSchemaField(node, current_levels, ctx, parent, out); + } + if (node.is_vector()) { + return Status::Invalid( + "VECTOR-repeated groups must be the single child of a group annotated with " + "the VECTOR logical type"); + } if (node.logical_type()->is_list()) { return ListToSchemaField(node, current_levels, ctx, parent, out); } else if (node.logical_type()->is_map()) { @@ -906,6 +1140,13 @@ Status NodeToSchemaField(const Node& node, LevelInfo current_levels, int column_index = ctx->schema->GetColumnIndex(primitive_node); ARROW_ASSIGN_OR_RAISE(std::shared_ptr type, GetTypeForNode(column_index, primitive_node, ctx)); + if (node.is_vector()) { + // VECTOR repetition directly on a primitive is not part of the canonical + // 3-level vector structure; require the VECTOR group form instead. + return Status::Invalid( + "VECTOR repetition on primitive nodes is not supported; vectors must use " + "the 3-level VECTOR group structure"); + } if (node.is_repeated()) { // One-level list encoding, e.g. // a: repeated int32; diff --git a/cpp/src/parquet/arrow/schema.h b/cpp/src/parquet/arrow/schema.h index dd60fde43422..f8056545f4e7 100644 --- a/cpp/src/parquet/arrow/schema.h +++ b/cpp/src/parquet/arrow/schema.h @@ -96,6 +96,14 @@ struct PARQUET_EXPORT SchemaField { parquet::internal::LevelInfo level_info; + // True when this Arrow field is backed by a Parquet VECTOR logical group. + bool is_vector = false; + + // For VECTOR fields, the definition level at or above which a vector value + // exists (present or null). Definition levels below this are absent + // ancestors, such as null structs or empty/null lists, and carry no slots. + int16_t vector_stride_def_level = 0; + bool is_leaf() const { return column_index != -1; } }; diff --git a/cpp/src/parquet/arrow/schema_internal.cc b/cpp/src/parquet/arrow/schema_internal.cc index 2e8cf764b27f..c930b70bf9e5 100644 --- a/cpp/src/parquet/arrow/schema_internal.cc +++ b/cpp/src/parquet/arrow/schema_internal.cc @@ -17,9 +17,12 @@ #include "parquet/arrow/schema_internal.h" +#include + #include "arrow/extension/json.h" #include "arrow/extension/uuid.h" #include "arrow/type.h" +#include "arrow/type_traits.h" #include "arrow/util/key_value_metadata.h" #include "arrow/util/logging.h" #include "arrow/util/string.h" @@ -39,6 +42,78 @@ using ::arrow::internal::checked_cast; namespace { +// Whether a type may appear inside a struct VECTOR element. This must mirror +// the writer-side schema validation (ValidateSupportedVectorStructNode): +// list/map descendants require repetition levels and force the whole column +// back to the standard LIST encoding. Both the schema conversion and the +// write-path level generation gate on this predicate so they cannot disagree +// about the VECTOR-vs-LIST fallback. +bool IsSupportedVectorStructFieldType(const ::arrow::DataType& type) { + switch (type.id()) { + case ::arrow::Type::LIST: + case ::arrow::Type::LARGE_LIST: + case ::arrow::Type::LIST_VIEW: + case ::arrow::Type::LARGE_LIST_VIEW: + case ::arrow::Type::MAP: + return false; + case ::arrow::Type::FIXED_SIZE_LIST: + // A fixed-size list nested inside a struct element would give the + // struct's leaves different per-row multiplicities, which the write-path + // level generation does not support; fall back to LIST encoding. + return false; + case ::arrow::Type::STRUCT: { + const auto& struct_type = checked_cast(type); + for (const auto& field : struct_type.fields()) { + if (!IsSupportedVectorStructFieldType(*field->type())) { + return false; + } + } + return true; + } + case ::arrow::Type::DICTIONARY: + return IsSupportedVectorStructFieldType( + *checked_cast(type).value_type()); + case ::arrow::Type::EXTENSION: + return IsSupportedVectorStructFieldType( + *checked_cast(type).storage_type()); + default: + return true; + } +} + +} // namespace + +Result VectorLeafSlotMultiplier(const ::arrow::DataType& type) { + if (type.id() != ::arrow::Type::FIXED_SIZE_LIST) { + return 1; + } + const auto& list_type = checked_cast(type); + if (list_type.list_size() <= 0) { + return Status::Invalid("VECTOR repetition requires positive vector_length"); + } + ARROW_ASSIGN_OR_RAISE(int32_t child_multiplier, + VectorLeafSlotMultiplier(*list_type.value_type())); + if (child_multiplier > std::numeric_limits::max() / list_type.list_size()) { + return Status::Invalid("Nested VECTOR leaf slot multiplier overflow"); + } + return child_multiplier * list_type.list_size(); +} + +bool IsSupportedVectorElementType(const ::arrow::DataType& type) { + if (type.id() == ::arrow::Type::FIXED_SIZE_LIST) { + const auto& list_type = checked_cast(type); + return list_type.list_size() > 0 && VectorLeafSlotMultiplier(type).ok() && + IsSupportedVectorElementType(*list_type.value_type()); + } + if (type.id() == ::arrow::Type::STRUCT) { + return IsSupportedVectorStructFieldType(type); + } + return !::arrow::is_nested(type) && ::arrow::is_fixed_width(type) && + type.id() != ::arrow::Type::DICTIONARY && type.id() != ::arrow::Type::EXTENSION; +} + +namespace { + Result> MakeArrowDecimal(const LogicalType& logical_type, bool smallest_decimal_enabled) { const auto& decimal = checked_cast(logical_type); diff --git a/cpp/src/parquet/arrow/schema_internal.h b/cpp/src/parquet/arrow/schema_internal.h index 09ad891aad3b..f248e8d6b8c4 100644 --- a/cpp/src/parquet/arrow/schema_internal.h +++ b/cpp/src/parquet/arrow/schema_internal.h @@ -40,4 +40,8 @@ Result> GetArrowType( const ArrowReaderProperties& reader_properties, const std::shared_ptr& metadata = nullptr); +::arrow::Result VectorLeafSlotMultiplier(const ::arrow::DataType& type); + +bool IsSupportedVectorElementType(const ::arrow::DataType& type); + } // namespace parquet::arrow diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc index e0fbe308219c..12c78971c492 100644 --- a/cpp/src/parquet/arrow/writer.cc +++ b/cpp/src/parquet/arrow/writer.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -108,6 +109,104 @@ bool HasNullableRoot(const SchemaManifest& schema_manifest, return nullable; } +int64_t CountVisitedValues(const std::vector& visited_elements) { + return std::accumulate( + visited_elements.begin(), visited_elements.end(), int64_t{0}, + [](int64_t total, const ElementRange& range) { return total + range.Size(); }); +} + +Result> MaterializeVectorLeafArray( + const MultipathLevelBuilderResult& result, ArrowWriteContext* ctx, + bool* leaf_is_nullable) { + const auto& visited_elements = result.post_list_visited_elements; + DCHECK_GT(visited_elements.size(), 0); + + // Nullable VECTOR rows still need one child slot per vector element so that the + // generic spaced leaf writer can align leaf slots with the VECTOR def levels. Arrow + // FixedSizeList already stores child slots for null parent rows, so preserve a + // zero-copy slice over the complete child range and let WriteArrow's def-level-derived + // validity bitmap suppress values belonging to null vector rows. + // The materialized array must hold one entry per physical leaf slot; absent + // ancestors (empty or null lists, null structs) contribute definition levels + // but no slots, so the target is leaf_slot_count, not def_rep_level_count. + if (CountVisitedValues(visited_elements) == result.leaf_slot_count && + visited_elements.size() == 1) { + const ElementRange& range = visited_elements[0]; + auto values = result.leaf_array->Slice(range.start, range.Size()); + if (values->null_count() != 0) { + *leaf_is_nullable = true; + } + return values; + } + + // General path, driven by the definition levels (the only stream that + // distinguishes the three kinds of entries): + // def < ancestor level: absent ancestor, no slot, no value; + // def < values level: slot of a null vector value, no value; + // def >= values level: slot carrying the next value from the visited + // ranges (in order; ranges skip null vector slots). + const int16_t ancestor_def_level = result.vector_repeated_ancestor_def_level; + const int16_t values_def_level = result.vector_values_def_level; + const int16_t* def_levels = result.def_levels; + ::arrow::ArrayVector parts; + size_t range_idx = 0; + int64_t range_used = 0; + int64_t pending_nulls = 0; + auto flush_nulls = [&]() -> Status { + if (pending_nulls > 0) { + ARROW_ASSIGN_OR_RAISE(auto null_values, + ::arrow::MakeArrayOfNull(result.leaf_array->type(), + pending_nulls, ctx->memory_pool)); + parts.push_back(std::move(null_values)); + pending_nulls = 0; + } + return Status::OK(); + }; + int64_t i = 0; + while (i < result.def_rep_level_count) { + const int16_t def_level = def_levels[i]; + if (def_level < ancestor_def_level) { + ++i; + continue; + } + if (def_level < values_def_level) { + ++pending_nulls; + ++i; + continue; + } + // Batch a run of value slots from the current visited range. + int64_t run = 1; + while (i + run < result.def_rep_level_count && + def_levels[i + run] >= values_def_level && + range_idx < visited_elements.size() && + range_used + run < visited_elements[range_idx].Size()) { + ++run; + } + if (range_idx >= visited_elements.size() || + range_used + run > visited_elements[range_idx].Size()) { + return Status::Invalid("VECTOR leaf values out of sync with definition levels"); + } + RETURN_NOT_OK(flush_nulls()); + const ElementRange& range = visited_elements[range_idx]; + parts.push_back(result.leaf_array->Slice(range.start + range_used, run)); + range_used += run; + if (range_used == static_cast(range.Size())) { + ++range_idx; + range_used = 0; + } + i += run; + } + RETURN_NOT_OK(flush_nulls()); + if (range_idx != visited_elements.size() || range_used != 0) { + return Status::Invalid("VECTOR leaf values out of sync with definition levels"); + } + *leaf_is_nullable = true; + if (parts.empty()) { + return result.leaf_array->Slice(0, 0); + } + return ::arrow::Concatenate(parts, ctx->memory_pool); +} + Status GetSchemaMetadata(const ::arrow::Schema& schema, ::arrow::MemoryPool* pool, const ArrowWriterProperties& properties, std::shared_ptr* out) { @@ -171,7 +270,11 @@ class ArrowColumnWriterV2 { size_t visited_component_size = result.post_list_visited_elements.size(); DCHECK_GT(visited_component_size, 0); std::shared_ptr values_array; - if (visited_component_size == 1) { + bool leaf_is_nullable = result.leaf_is_nullable; + if (result.leaf_is_vector) { + ARROW_ASSIGN_OR_RAISE(values_array, MaterializeVectorLeafArray( + result, ctx, &leaf_is_nullable)); + } else if (visited_component_size == 1) { const ElementRange& range = result.post_list_visited_elements[0]; values_array = result.leaf_array->Slice(range.start, range.Size()); } else { @@ -191,7 +294,7 @@ class ArrowColumnWriterV2 { return column_writer->WriteArrow(result.def_levels, result.rep_levels, result.def_rep_level_count, *values_array, - ctx, result.leaf_is_nullable); + ctx, leaf_is_nullable); })); } @@ -213,7 +316,7 @@ class ArrowColumnWriterV2 { static ::arrow::Result> Make( const ChunkedArray& data, int64_t offset, const int64_t size, const SchemaManifest& schema_manifest, RowGroupWriter* row_group_writer, - int start_leaf_column_index = -1) { + bool write_fixed_size_list_as_vector, int start_leaf_column_index = -1) { int64_t absolute_position = 0; int chunk_index = 0; int64_t chunk_offset = 0; @@ -283,8 +386,10 @@ class ArrowColumnWriterV2 { std::shared_ptr array_to_write = chunk.Slice(chunk_offset, chunk_write_size); if (array_to_write->length() > 0) { - ARROW_ASSIGN_OR_RAISE(std::unique_ptr builder, - MultipathLevelBuilder::Make(*array_to_write, is_nullable)); + ARROW_ASSIGN_OR_RAISE( + std::unique_ptr builder, + MultipathLevelBuilder::Make(*array_to_write, is_nullable, + write_fixed_size_list_as_vector)); if (leaf_count != builder->GetLeafCount()) { return Status::UnknownError("data type leaf_count != builder_leaf_count", leaf_count, " ", builder->GetLeafCount()); @@ -387,10 +492,10 @@ class FileWriterImpl : public FileWriter { if (row_group_writer_->buffered()) { return Status::Invalid("Cannot write column chunk into the buffered row group."); } - ARROW_ASSIGN_OR_RAISE( - std::unique_ptr writer, - ArrowColumnWriterV2::Make(*data, offset, size, schema_manifest_, - row_group_writer_)); + ARROW_ASSIGN_OR_RAISE(std::unique_ptr writer, + ArrowColumnWriterV2::Make( + *data, offset, size, schema_manifest_, row_group_writer_, + arrow_properties_->write_fixed_size_list_as_vector())); return writer->Write(&column_write_context_); } @@ -471,8 +576,10 @@ class FileWriterImpl : public FileWriter { ChunkedArray chunked_array{batch.column(i)}; ARROW_ASSIGN_OR_RAISE( std::unique_ptr writer, - ArrowColumnWriterV2::Make(chunked_array, offset, size, schema_manifest_, - row_group_writer_, column_index_start)); + ArrowColumnWriterV2::Make( + chunked_array, offset, size, schema_manifest_, row_group_writer_, + arrow_properties_->write_fixed_size_list_as_vector(), + column_index_start)); column_index_start += writer->leaf_count(); if (arrow_properties_->use_threads()) { writers.emplace_back(std::move(writer)); diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h index e2244a1176e3..4cafe5ee2887 100644 --- a/cpp/src/parquet/properties.h +++ b/cpp/src/parquet/properties.h @@ -1329,6 +1329,7 @@ class PARQUET_EXPORT ArrowWriterProperties { truncated_timestamps_allowed_(false), store_schema_(false), compliant_nested_types_(true), + write_fixed_size_list_as_vector_(false), engine_version_(V2), use_threads_(kArrowDefaultUseThreads), executor_(NULLPTR), @@ -1399,6 +1400,16 @@ class PARQUET_EXPORT ArrowWriterProperties { return this; } + /// \brief EXPERIMENTAL: encode supported Arrow FixedSizeList values as Parquet + /// VECTOR. Unsupported FixedSizeList values fall back to the standard LIST + /// encoding: zero-length lists, variable-width or dictionary or extension + /// elements, struct elements containing fixed-size lists, and fixed-size + /// lists below nullable groups. + Builder* enable_experimental_vector_encoding() { + write_fixed_size_list_as_vector_ = true; + return this; + } + /// Set the version of the Parquet writer engine. Builder* set_engine_version(EngineVersion version) { engine_version_ = version; @@ -1441,7 +1452,8 @@ class PARQUET_EXPORT ArrowWriterProperties { return std::shared_ptr(new ArrowWriterProperties( write_timestamps_as_int96_, coerce_timestamps_enabled_, coerce_timestamps_unit_, truncated_timestamps_allowed_, store_schema_, compliant_nested_types_, - engine_version_, use_threads_, executor_, write_time_adjusted_to_utc_)); + write_fixed_size_list_as_vector_, engine_version_, use_threads_, executor_, + write_time_adjusted_to_utc_)); } private: @@ -1453,6 +1465,7 @@ class PARQUET_EXPORT ArrowWriterProperties { bool store_schema_; bool compliant_nested_types_; + bool write_fixed_size_list_as_vector_; EngineVersion engine_version_; bool use_threads_; @@ -1479,6 +1492,10 @@ class PARQUET_EXPORT ArrowWriterProperties { /// "element". bool compliant_nested_types() const { return compliant_nested_types_; } + bool write_fixed_size_list_as_vector() const { + return write_fixed_size_list_as_vector_; + } + /// \brief The underlying engine version to use when writing Arrow data. /// /// V2 is currently the latest V1 is considered deprecated but left in @@ -1503,6 +1520,7 @@ class PARQUET_EXPORT ArrowWriterProperties { ::arrow::TimeUnit::type coerce_timestamps_unit, bool truncated_timestamps_allowed, bool store_schema, bool compliant_nested_types, + bool write_fixed_size_list_as_vector, EngineVersion engine_version, bool use_threads, ::arrow::internal::Executor* executor, bool write_time_adjusted_to_utc) @@ -1512,6 +1530,7 @@ class PARQUET_EXPORT ArrowWriterProperties { truncated_timestamps_allowed_(truncated_timestamps_allowed), store_schema_(store_schema), compliant_nested_types_(compliant_nested_types), + write_fixed_size_list_as_vector_(write_fixed_size_list_as_vector), engine_version_(engine_version), use_threads_(use_threads), executor_(executor), @@ -1523,6 +1542,7 @@ class PARQUET_EXPORT ArrowWriterProperties { const bool truncated_timestamps_allowed_; const bool store_schema_; const bool compliant_nested_types_; + const bool write_fixed_size_list_as_vector_; const EngineVersion engine_version_; const bool use_threads_; ::arrow::internal::Executor* executor_; From 07d08bb4af8d141e3a35aa5480b062b1b646c48c Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Thu, 11 Jun 2026 12:44:22 +0200 Subject: [PATCH 4/8] Expose experimental vector encoding in pyarrow Adds experimental_vector_encoding to pyarrow.parquet write APIs and dataset writing, with fixed-shape tensor roundtrip tests. --- python/pyarrow/_dataset_parquet.pyx | 7 +++++ python/pyarrow/_parquet.pxd | 1 + python/pyarrow/_parquet.pyx | 12 ++++++-- python/pyarrow/includes/libparquet.pxd | 1 + python/pyarrow/parquet/core.py | 11 ++++++++ .../pyarrow/tests/parquet/test_data_types.py | 28 +++++++++++++++++++ 6 files changed, 58 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/_dataset_parquet.pyx b/python/pyarrow/_dataset_parquet.pyx index 534f7790923a..6567b95a9ab1 100644 --- a/python/pyarrow/_dataset_parquet.pyx +++ b/python/pyarrow/_dataset_parquet.pyx @@ -620,6 +620,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): "coerce_timestamps", "allow_truncated_timestamps", "use_compliant_nested_type", + "experimental_vector_encoding", } setters = set() @@ -676,6 +677,11 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): writer_engine_version="V2", use_compliant_nested_type=( self._properties["use_compliant_nested_type"] + ), + store_schema=True, + write_time_adjusted_to_utc=False, + experimental_vector_encoding=( + self._properties["experimental_vector_encoding"] ) ) @@ -705,6 +711,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): coerce_timestamps=None, allow_truncated_timestamps=False, use_compliant_nested_type=True, + experimental_vector_encoding=False, encryption_properties=None, write_batch_size=None, dictionary_pagesize_limit=None, diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 36fc2ccf2f33..fa0a06e53c7e 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -69,6 +69,7 @@ cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties( use_compliant_nested_type=*, store_schema=*, write_time_adjusted_to_utc=*, + experimental_vector_encoding=*, ) except * diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index 2358a961ebd9..39cb5dac42f0 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -2281,7 +2281,8 @@ cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties( writer_engine_version=None, use_compliant_nested_type=True, store_schema=True, - write_time_adjusted_to_utc=False) except *: + write_time_adjusted_to_utc=False, + experimental_vector_encoding=False) except *: """Arrow writer properties""" cdef: shared_ptr[ArrowWriterProperties] arrow_properties @@ -2322,6 +2323,11 @@ cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties( else: arrow_props.disable_compliant_nested_types() + # experimental_vector_encoding + + if experimental_vector_encoding: + arrow_props.enable_experimental_vector_encoding() + # writer_engine_version if writer_engine_version == "V1": @@ -2396,7 +2402,8 @@ cdef class ParquetWriter(_Weakrefable): store_decimal_as_integer=False, use_content_defined_chunking=False, write_time_adjusted_to_utc=False, - bloom_filter_options=None): + bloom_filter_options=None, + experimental_vector_encoding=False): cdef: shared_ptr[WriterProperties] properties shared_ptr[ArrowWriterProperties] arrow_properties @@ -2443,6 +2450,7 @@ cdef class ParquetWriter(_Weakrefable): use_compliant_nested_type=use_compliant_nested_type, store_schema=store_schema, write_time_adjusted_to_utc=write_time_adjusted_to_utc, + experimental_vector_encoding=experimental_vector_encoding, ) pool = maybe_unbox_memory_pool(memory_pool) diff --git a/python/pyarrow/includes/libparquet.pxd b/python/pyarrow/includes/libparquet.pxd index df353cc7805f..6515e6f1b75d 100644 --- a/python/pyarrow/includes/libparquet.pxd +++ b/python/pyarrow/includes/libparquet.pxd @@ -529,6 +529,7 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil: Builder* store_schema() Builder* enable_compliant_nested_types() Builder* disable_compliant_nested_types() + Builder* enable_experimental_vector_encoding() Builder* set_engine_version(ArrowWriterEngineVersion version) Builder* set_time_adjusted_to_utc(c_bool adjusted) shared_ptr[ArrowWriterProperties] build() diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index ff880fdcf52c..8fa296e60e50 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -876,6 +876,13 @@ def _sanitize_table(table, new_schema, flavor): dictionary_pagesize_limit : int, default None Specify the dictionary page size limit per row group. If None, use the default 1MB. +experimental_vector_encoding : bool, default False + EXPERIMENTAL: Encode supported fixed-size list values, including the storage + of fixed-shape tensor extension arrays, using Parquet VECTOR repetition. + Unsupported fixed-size list values fall back to the standard LIST encoding: + zero-length lists, variable-width or dictionary or extension elements, + struct elements containing fixed-size lists, and fixed-size lists below + nullable groups. store_schema : bool, default True By default, the Arrow schema is serialized and stored in the Parquet file metadata (in the "ARROW:schema" key). When reading the file, @@ -1083,6 +1090,7 @@ def __init__(self, where, schema, filesystem=None, store_decimal_as_integer=False, write_time_adjusted_to_utc=False, max_rows_per_page=None, + experimental_vector_encoding=False, **options): if use_deprecated_int96_timestamps is None: # Use int96 timestamps for Spark @@ -1138,6 +1146,7 @@ def __init__(self, where, schema, filesystem=None, store_decimal_as_integer=store_decimal_as_integer, write_time_adjusted_to_utc=write_time_adjusted_to_utc, max_rows_per_page=max_rows_per_page, + experimental_vector_encoding=experimental_vector_encoding, **options) self.is_open = True @@ -2017,6 +2026,7 @@ def write_table(table, where, row_group_size=None, version='2.6', write_time_adjusted_to_utc=False, max_rows_per_page=None, bloom_filter_options=None, + experimental_vector_encoding=False, **kwargs): # Implementor's note: when adding keywords here / updating defaults, also # update it in write_to_dataset and _dataset_parquet.pyx ParquetFileWriteOptions @@ -2051,6 +2061,7 @@ def write_table(table, where, row_group_size=None, version='2.6', write_time_adjusted_to_utc=write_time_adjusted_to_utc, max_rows_per_page=max_rows_per_page, bloom_filter_options=bloom_filter_options, + experimental_vector_encoding=experimental_vector_encoding, **kwargs) as writer: writer.write_table(table, row_group_size=row_group_size) except Exception: diff --git a/python/pyarrow/tests/parquet/test_data_types.py b/python/pyarrow/tests/parquet/test_data_types.py index c546bc1532ac..2d674a20036c 100644 --- a/python/pyarrow/tests/parquet/test_data_types.py +++ b/python/pyarrow/tests/parquet/test_data_types.py @@ -581,6 +581,34 @@ def test_json_extension_type(storage_type): store_schema=False) +def test_fixed_shape_tensor_vector_encoding(tempdir): + tensor_type = pa.fixed_shape_tensor(pa.float32(), [2, 3]) + storage = pa.array([ + [1, 2, 3, 4, 5, 6], + None, + [7, 8, 9, 10, 11, 12], + ], pa.list_(pa.float32(), 6)) + arr = pa.ExtensionArray.from_storage(tensor_type, storage) + table = pa.table({"tensor": arr}) + + sink = pa.BufferOutputStream() + pq.write_table(table, sink, experimental_vector_encoding=True) + buf = sink.getvalue() + + metadata = pq.read_metadata(pa.BufferReader(buf)) + assert "vector" in str(metadata.schema).lower() + + result = pq.read_table(pa.BufferReader(buf)) + assert result.schema == table.schema + assert result.equals(table) + + dataset_dir = tempdir / "vector_dataset" + pq.write_to_dataset(table, dataset_dir, experimental_vector_encoding=True) + dataset_result = pq.read_table(dataset_dir) + assert dataset_result.schema == table.schema + assert dataset_result.equals(table) + + def test_uuid_extension_type(): data = [ b'\xe4`\xf9p\x83QGN\xac\x7f\xa4g>K\xa8\xcb', From fae8ca002cdbf1970a144f9ee4121bbbde80be86 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Thu, 11 Jun 2026 16:57:02 +0200 Subject: [PATCH 5/8] MaterializeVectorLeafArray() now skips zero-length visited ranges --- .../parquet/arrow/arrow_reader_writer_test.cc | 18 ++++++++++++++++++ cpp/src/parquet/arrow/arrow_schema_test.cc | 11 +++++++++++ cpp/src/parquet/arrow/schema.cc | 5 ++++- cpp/src/parquet/arrow/writer.cc | 8 ++++++++ cpp/src/parquet/parquet.thrift | 4 +++- 5 files changed, 44 insertions(+), 2 deletions(-) diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc index f4ba0571444c..61a88f5926a3 100644 --- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc +++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc @@ -3878,6 +3878,24 @@ TEST(ArrowReadWrite, FixedSizeListVectorListOfNullableVectorsRoundTrip) { /*row_group_size=*/3)); } +TEST(ArrowReadWrite, FixedSizeListVectorListOfOnlyNullVectorsRoundTrip) { + // A row group containing only null VECTOR items still has physical null + // slots but no visited value ranges. + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto type = ::arrow::list(::arrow::field("item", vector_type, /*nullable=*/true)); + auto array = ::arrow::ArrayFromJSON(type, R"([ + [null], + [null, null], + [], + null, + [[1, 2]]])"); + auto table = + ::arrow::Table::Make(::arrow::schema({::arrow::field("root", type)}), {array}); + ASSERT_NO_FATAL_FAILURE(CheckVectorFixedSizeListRoundtrip(table, + /*row_group_size=*/1)); +} + TEST(ArrowReadWrite, FixedSizeListVectorNestedVectorsUnderListRoundTrip) { auto inner = ::arrow::fixed_size_list( ::arrow::field("value", ::arrow::int32(), /*nullable=*/false), 2); diff --git a/cpp/src/parquet/arrow/arrow_schema_test.cc b/cpp/src/parquet/arrow/arrow_schema_test.cc index 5a626f523783..4b9f768d1512 100644 --- a/cpp/src/parquet/arrow/arrow_schema_test.cc +++ b/cpp/src/parquet/arrow/arrow_schema_test.cc @@ -459,6 +459,17 @@ TEST_F(TestConvertParquetSchema, VectorFixedSizeListStructWithListRejected) { ASSERT_RAISES(NotImplemented, ConvertSchema(parquet_fields)); } +TEST_F(TestConvertParquetSchema, VectorFixedSizeListStructWithRepeatedPrimitiveRejected) { + auto values = PrimitiveNode::Make("values", Repetition::REPEATED, ParquetType::INT32); + auto item = GroupNode::Make("element", Repetition::REQUIRED, {values}); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {item}, + /*logical_type=*/nullptr, -1, 3); + std::vector parquet_fields = {GroupNode::Make( + "embedding", Repetition::OPTIONAL, {vector}, LogicalType::Vector())}; + + ASSERT_RAISES(NotImplemented, ConvertSchema(parquet_fields)); +} + TEST_F(TestConvertParquetSchema, DuplicateFieldNames) { std::vector parquet_fields; std::vector> arrow_fields; diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc index 5aa18c6fc5a1..16c88279411a 100644 --- a/cpp/src/parquet/arrow/schema.cc +++ b/cpp/src/parquet/arrow/schema.cc @@ -119,10 +119,13 @@ Status ListToNode(const std::shared_ptr<::arrow::BaseListType>& type, } bool IsSupportedVectorStructNode(const Node& node) { + if (node.is_repeated() || node.is_vector()) { + return false; + } if (node.is_primitive()) { return true; } - if (!node.is_group() || node.is_repeated()) { + if (!node.is_group()) { return false; } const auto& group = checked_cast(node); diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc index 12c78971c492..7ea471a8cef8 100644 --- a/cpp/src/parquet/arrow/writer.cc +++ b/cpp/src/parquet/arrow/writer.cc @@ -162,6 +162,12 @@ Result> MaterializeVectorLeafArray( } return Status::OK(); }; + auto skip_empty_ranges = [&]() { + while (range_idx < visited_elements.size() && range_used == 0 && + visited_elements[range_idx].Empty()) { + ++range_idx; + } + }; int64_t i = 0; while (i < result.def_rep_level_count) { const int16_t def_level = def_levels[i]; @@ -175,6 +181,7 @@ Result> MaterializeVectorLeafArray( continue; } // Batch a run of value slots from the current visited range. + skip_empty_ranges(); int64_t run = 1; while (i + run < result.def_rep_level_count && def_levels[i + run] >= values_def_level && @@ -197,6 +204,7 @@ Result> MaterializeVectorLeafArray( i += run; } RETURN_NOT_OK(flush_nulls()); + skip_empty_ranges(); if (range_idx != visited_elements.size() || range_used != 0) { return Status::Invalid("VECTOR leaf values out of sync with definition levels"); } diff --git a/cpp/src/parquet/parquet.thrift b/cpp/src/parquet/parquet.thrift index d5f9b9608d73..83c6c2f9c2ae 100644 --- a/cpp/src/parquet/parquet.thrift +++ b/cpp/src/parquet/parquet.thrift @@ -200,7 +200,9 @@ enum FieldRepetitionType { // from the fixed multiplicity declared in the schema instead of decoding // repetition levels. // - // Vector fields MUST use the following 3-level structure, mirroring LIST: + // Vector fields MUST use the following 3-level structure, mirroring LIST. + // The VECTOR-repeated group is shown with the canonical writer name "list"; + // readers SHOULD NOT depend on the group name. // // group (VECTOR) { // vector group list [vector_length] { From 9c1f829526db14092ee1b26447fd049f3f5b5010 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Mon, 29 Jun 2026 20:03:48 +0200 Subject: [PATCH 6/8] Align VECTOR layout with format proposal --- cpp/src/generated/parquet_types.tcc | 4 +- .../parquet/arrow/arrow_reader_writer_test.cc | 4 +- cpp/src/parquet/arrow/arrow_schema_test.cc | 41 ++++----- cpp/src/parquet/arrow/schema.cc | 16 ++-- cpp/src/parquet/parquet.thrift | 87 +++++-------------- cpp/src/parquet/properties.h | 8 +- cpp/src/parquet/schema.cc | 6 ++ cpp/src/parquet/schema_test.cc | 51 +++++++---- python/pyarrow/parquet/core.py | 10 +-- 9 files changed, 108 insertions(+), 119 deletions(-) diff --git a/cpp/src/generated/parquet_types.tcc b/cpp/src/generated/parquet_types.tcc index 073741b125f7..46dab3d15a4b 100644 --- a/cpp/src/generated/parquet_types.tcc +++ b/cpp/src/generated/parquet_types.tcc @@ -2066,7 +2066,7 @@ uint32_t SchemaElement::read(Protocol_* iprot) { xfer += iprot->skip(ftype); } break; - case 12: + case 11: if (ftype == ::apache::thrift::protocol::T_I32) { xfer += iprot->readI32(this->vector_length); this->__isset.vector_length = true; @@ -2144,7 +2144,7 @@ uint32_t SchemaElement::write(Protocol_* oprot) const { xfer += oprot->writeFieldEnd(); } if (this->__isset.vector_length) { - xfer += oprot->writeFieldBegin("vector_length", ::apache::thrift::protocol::T_I32, 12); + xfer += oprot->writeFieldBegin("vector_length", ::apache::thrift::protocol::T_I32, 11); xfer += oprot->writeI32(this->vector_length); xfer += oprot->writeFieldEnd(); } diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc index 61a88f5926a3..04693c1b1cda 100644 --- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc +++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc @@ -5041,7 +5041,7 @@ TEST(TestArrowReaderAdHoc, ReadNestedVectorUnderNullableStructAncestor) { auto inner_vector = GroupNode::Make("list", Repetition::VECTOR, {element}, ConvertedType::NONE, /*field_id=*/-1, /*vector_length=*/3); - auto inner = GroupNode::Make("inner", Repetition::REQUIRED, {inner_vector}, + auto inner = GroupNode::Make("element", Repetition::REQUIRED, {inner_vector}, LogicalType::Vector()); auto outer_vector = GroupNode::Make("list", Repetition::VECTOR, {inner}, ConvertedType::NONE, /*field_id=*/-1, @@ -5076,7 +5076,7 @@ TEST(TestArrowReaderAdHoc, ReadNestedVectorUnderNullableStructAncestor) { auto inner_type = ::arrow::fixed_size_list( ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 3); auto outer_type = ::arrow::fixed_size_list( - ::arrow::field("inner", inner_type, /*nullable=*/false), 2); + ::arrow::field("element", inner_type, /*nullable=*/false), 2); auto parent_type = ::arrow::struct_({::arrow::field("embedding", outer_type, /*nullable=*/false)}); auto rows_type = ::arrow::list(::arrow::field("parent", parent_type, diff --git a/cpp/src/parquet/arrow/arrow_schema_test.cc b/cpp/src/parquet/arrow/arrow_schema_test.cc index 4b9f768d1512..9e1e84d66781 100644 --- a/cpp/src/parquet/arrow/arrow_schema_test.cc +++ b/cpp/src/parquet/arrow/arrow_schema_test.cc @@ -329,35 +329,35 @@ TEST_F(TestConvertParquetSchema, VectorAnnotatedGroupWithoutVectorChildRejected) ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); } -TEST_F(TestConvertParquetSchema, VectorMiddleGroupNameIrrelevant) { - // Mirroring LIST, readers do not depend on the VECTOR-repeated group's name. - std::vector parquet_fields; - std::vector> arrow_fields; - +TEST_F(TestConvertParquetSchema, VectorMiddleGroupNameRejected) { + // The canonical VECTOR layout requires the middle group to be named "list". auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); auto vector = GroupNode::Make("vector", Repetition::VECTOR, {element}, /*logical_type=*/nullptr, -1, 3); - parquet_fields.push_back(GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, - LogicalType::Vector())); - arrow_fields.push_back(::arrow::field( - "embedding", - ::arrow::fixed_size_list(::arrow::field("element", ::arrow::float32(), false), 3), - true)); + std::vector parquet_fields = {GroupNode::Make( + "embedding", Repetition::OPTIONAL, {vector}, LogicalType::Vector())}; - ASSERT_OK(ConvertSchema(parquet_fields)); - ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(::arrow::schema(arrow_fields))); + ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); } -TEST_F(TestConvertParquetSchema, VectorMiddleGroupLogicalTypeRejected) { - auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); - auto vector = - GroupNode::Make("list", Repetition::VECTOR, {element}, LogicalType::List(), -1, 3); +TEST_F(TestConvertParquetSchema, VectorElementNameRejected) { + // The canonical VECTOR layout requires the element field to be named "element". + auto element = PrimitiveNode::Make("item", Repetition::REQUIRED, ParquetType::FLOAT); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*logical_type=*/nullptr, -1, 3); std::vector parquet_fields = {GroupNode::Make( "embedding", Repetition::OPTIONAL, {vector}, LogicalType::Vector())}; ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); } +TEST_F(TestConvertParquetSchema, VectorMiddleGroupLogicalTypeRejected) { + auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); + ASSERT_THROW(GroupNode::Make("list", Repetition::VECTOR, {element}, + LogicalType::List(), -1, 3), + ParquetException); +} + TEST_F(TestConvertParquetSchema, VectorOnPrimitiveRejected) { // VECTOR repetition directly on a primitive is not part of the canonical // 3-level vector structure; vectors must use a VECTOR group whose single @@ -451,12 +451,13 @@ TEST_F(TestConvertParquetSchema, VectorFixedSizeListStructWithListRejected) { auto list = GroupNode::Make("list", Repetition::REPEATED, {list_element}); auto items = GroupNode::Make("items", Repetition::REQUIRED, {list}, LogicalType::List()); - auto vector = GroupNode::Make("list", Repetition::VECTOR, {items}, + auto element = GroupNode::Make("element", Repetition::REQUIRED, {items}); + auto vector = GroupNode::Make("list", Repetition::VECTOR, {element}, /*logical_type=*/nullptr, -1, 3); std::vector parquet_fields = {GroupNode::Make( "embedding", Repetition::OPTIONAL, {vector}, LogicalType::Vector())}; - ASSERT_RAISES(NotImplemented, ConvertSchema(parquet_fields)); + ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); } TEST_F(TestConvertParquetSchema, VectorFixedSizeListStructWithRepeatedPrimitiveRejected) { @@ -467,7 +468,7 @@ TEST_F(TestConvertParquetSchema, VectorFixedSizeListStructWithRepeatedPrimitiveR std::vector parquet_fields = {GroupNode::Make( "embedding", Repetition::OPTIONAL, {vector}, LogicalType::Vector())}; - ASSERT_RAISES(NotImplemented, ConvertSchema(parquet_fields)); + ASSERT_RAISES(Invalid, ConvertSchema(parquet_fields)); } TEST_F(TestConvertParquetSchema, DuplicateFieldNames) { diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc index 16c88279411a..2519665b39a8 100644 --- a/cpp/src/parquet/arrow/schema.cc +++ b/cpp/src/parquet/arrow/schema.cc @@ -144,9 +144,8 @@ Status ValidateSupportedVectorStructNode(const Node& node) { if (IsSupportedVectorStructNode(node)) { return Status::OK(); } - return Status::NotImplemented( - "VECTOR elements only support primitive, nested VECTOR, or nested struct fields; " - "repeated/list/map descendants are not supported"); + return Status::Invalid( + "VECTOR element subtrees must not contain repeated, list, or map fields"); } // Option B mapping for Arrow FixedSizeList -> Parquet VECTOR. @@ -766,7 +765,8 @@ bool IsVectorGroup(const GroupNode& group) { // } // // The element may be a primitive, a struct group, or a nested vector group -// (itself in canonical form). +// (itself in canonical form). The middle group and element field names are +// part of the canonical VECTOR layout and are validated. Status VectorToSchemaField(const GroupNode& group, LevelInfo current_levels, SchemaTreeContext* ctx, const SchemaField* parent, SchemaField* out) { @@ -785,8 +785,9 @@ Status VectorToSchemaField(const GroupNode& group, LevelInfo current_levels, "VECTOR-repeated nodes must be groups containing a single element field"); } const auto& vector_group = static_cast(child); - // Mirroring LIST, the writer names the VECTOR-repeated group "list" but - // readers do not depend on group names. + if (vector_group.name() != "list") { + return Status::Invalid("VECTOR-repeated middle groups must be named 'list'"); + } if (vector_group.logical_type() != nullptr && !vector_group.logical_type()->is_none()) { return Status::Invalid("VECTOR-repeated groups must not have a logical type"); } @@ -797,6 +798,9 @@ Status VectorToSchemaField(const GroupNode& group, LevelInfo current_levels, return Status::Invalid("VECTOR-repeated groups must have a single element child"); } const Node& element = *vector_group.field(0); + if (element.name() != "element") { + return Status::Invalid("VECTOR element fields must be named 'element'"); + } if (element.is_repeated() || element.is_vector()) { return Status::Invalid("VECTOR elements must be required or optional"); } diff --git a/cpp/src/parquet/parquet.thrift b/cpp/src/parquet/parquet.thrift index 83c6c2f9c2ae..83575cfc8f75 100644 --- a/cpp/src/parquet/parquet.thrift +++ b/cpp/src/parquet/parquet.thrift @@ -191,18 +191,10 @@ enum FieldRepetitionType { /** The field is repeated and can contain 0 or more values */ REPEATED = 2; - // EXPERIMENTAL VECTOR layout rules (kept out of the doc comment so they are - // not copied into generated code): - // - // A VECTOR field repeats exactly vector_length times for every parent value - // and, unlike REPEATED, does not increase the maximum definition or - // repetition level of its descendants: readers reconstruct vector values - // from the fixed multiplicity declared in the schema instead of decoding - // repetition levels. + // EXPERIMENTAL: fixed-size repetition. // - // Vector fields MUST use the following 3-level structure, mirroring LIST. - // The VECTOR-repeated group is shown with the canonical writer name "list"; - // readers SHOULD NOT depend on the group name. + // VECTOR is used for the middle group of a fixed-size vector encoded with + // the canonical 3-level structure: // // group (VECTOR) { // vector group list [vector_length] { @@ -210,44 +202,22 @@ enum FieldRepetitionType { // } // } // - // - The outer group MUST be REQUIRED or OPTIONAL and carries the - // nullability of the vector value itself. It MUST be annotated with the - // VECTOR logical type and MUST have exactly one child: the - // VECTOR-repeated group. - // - The VECTOR-repeated middle group carries vector_length and MUST have - // exactly one child: the element field. - // - The element field MUST be REQUIRED or OPTIONAL and MAY be a primitive - // type, a group of fields (a struct), or another vector group in the same - // form. For nested vectors a leaf stores the product of the - // vector_length values of its VECTOR ancestors physical values per parent - // record. + // The outer group MUST be REQUIRED or OPTIONAL, MUST be annotated with + // LogicalType.VECTOR, and MUST contain exactly one field named "list". The + // middle group MUST have repetition_type = VECTOR, MUST be named "list", + // MUST carry a positive SchemaElement.vector_length, MUST have exactly one + // field named "element", and MUST NOT have a logical or converted type. + // The element field encodes the vector element type and MUST be REQUIRED or + // OPTIONAL. Repeated fields are not allowed inside the vector element subtree. // - // Data layout rules: - // - Writers MUST emit exactly vector_length child slots for every parent - // record in which the vector value is present or null; null vector values - // emit vector_length slots carrying the definition level of the null - // vector. Ancestors that make the vector value absent altogether (for - // example an empty or null list) contribute a single level entry with - // their usual definition level and no slots. Column chunk num_values and - // null counts therefore include the padding slots of null vector values. - // - Every slot of an existing vector occupies one position in both level - // streams. The repetition level of the first slot encodes record - // structure as usual; the repetition levels of the remaining - // vector_length - 1 slots MUST be written as the column's maximum - // repetition level and MUST be ignored by readers. Stream consumption is - // definition-driven: after reading a position whose definition level - // indicates an existing vector, a reader consumes that position and the - // following vector_length - 1 positions as one vector value. - // - Writers MUST NOT split the slots of one vector value across data pages; - // pages begin and end on whole-vector boundaries. - // - vector_length MUST be positive: a zero-length vector value would - // contribute no slots at all, leaving row counts and row nullability - // unrepresentable in this layout. Writers MUST represent zero-length - // fixed-size lists with the LIST encoding instead. + // A VECTOR level contributes no definition or repetition level. Each vector + // occurrence contributes exactly vector_length element slots / level entries + // to each primitive leaf in the element subtree, including null vector values + // and null element slots. Column chunk and page num_values count those slots, + // not necessarily the number of physical values. Writers MUST NOT split the + // slots of one vector value across data pages. // // Readers that do not understand VECTOR are expected to reject the file. - - // EXPERIMENTAL: fixed-size repetition. VECTOR = 3; } @@ -528,15 +498,11 @@ struct GeographyType { 2: optional EdgeInterpolationAlgorithm algorithm; } -// The VECTOR annotation marks the outer group of a fixed-size vector field; -// see FieldRepetitionType.VECTOR for the full structure and data layout -// rules. The annotated group MUST be REQUIRED or OPTIONAL and MUST have -// exactly one child: a VECTOR-repeated group whose -// SchemaElement.vector_length carries the fixed multiplicity. The annotation -// is required so that readers can distinguish a vector field from a -// struct-like group that happens to contain a VECTOR-repeated child. - // EXPERIMENTAL: Embedded Vector logical type annotation. +// +// The VECTOR annotation marks the outer group of a fixed-size vector field; +// see FieldRepetitionType.VECTOR for the full canonical structure and data +// layout rules. struct VectorType {} // allowed for group nodes only /** @@ -636,15 +602,10 @@ struct SchemaElement { */ 10: optional LogicalType logicalType - // vector_length MUST be set, and positive, when repetition_type is VECTOR - // and MUST NOT be set otherwise. Zero-length vectors are not representable - // as VECTOR and use the LIST encoding (see FieldRepetitionType.VECTOR). - // For nested VECTOR fields the number of physical leaf values per parent - // record is the product of vector_length over the leaf's VECTOR ancestors. - - // EXPERIMENTAL: The fixed number of times the field repeats per parent - // value when repetition_type is VECTOR. - 12: optional i32 vector_length; + // EXPERIMENTAL: For VECTOR repetition, the fixed number of element slots per + // parent occurrence. This MUST be set, and positive, when repetition_type is + // VECTOR and MUST NOT be set otherwise. + 11: optional i32 vector_length; } /** diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h index 4cafe5ee2887..23a37b733484 100644 --- a/cpp/src/parquet/properties.h +++ b/cpp/src/parquet/properties.h @@ -1401,10 +1401,10 @@ class PARQUET_EXPORT ArrowWriterProperties { } /// \brief EXPERIMENTAL: encode supported Arrow FixedSizeList values as Parquet - /// VECTOR. Unsupported FixedSizeList values fall back to the standard LIST - /// encoding: zero-length lists, variable-width or dictionary or extension - /// elements, struct elements containing fixed-size lists, and fixed-size - /// lists below nullable groups. + /// VECTOR using the canonical 3-level VECTOR layout. Unsupported FixedSizeList + /// values fall back to the standard LIST encoding: zero-length lists, + /// variable-width or dictionary or extension elements, struct elements containing + /// fixed-size lists, and fixed-size lists below nullable groups. Builder* enable_experimental_vector_encoding() { write_fixed_size_list_as_vector_ = true; return this; diff --git a/cpp/src/parquet/schema.cc b/cpp/src/parquet/schema.cc index f1573941c0ba..45729c1584eb 100644 --- a/cpp/src/parquet/schema.cc +++ b/cpp/src/parquet/schema.cc @@ -341,6 +341,9 @@ GroupNode::GroupNode(const std::string& name, Repetition::type repetition, : Node(Node::GROUP, name, repetition, converted_type, id, vector_length), fields_(fields) { ValidateVectorProperties(repetition, vector_length); + if (repetition == Repetition::VECTOR && converted_type != ConvertedType::NONE) { + throw ParquetException("VECTOR-repeated groups must not have a converted type"); + } // For forward compatibility, create an equivalent logical type logical_type_ = LogicalType::FromConvertedType(converted_type_); if (!(logical_type_ && (logical_type_->is_nested() || logical_type_->is_none()) && @@ -363,6 +366,9 @@ GroupNode::GroupNode(const std::string& name, Repetition::type repetition, : Node(Node::GROUP, name, repetition, std::move(logical_type), id, vector_length), fields_(fields) { ValidateVectorProperties(repetition, vector_length); + if (repetition == Repetition::VECTOR && logical_type_ && !logical_type_->is_none()) { + throw ParquetException("VECTOR-repeated groups must not have a logical type"); + } if (logical_type_) { // Check for logical type <=> node type consistency if (logical_type_->is_nested()) { diff --git a/cpp/src/parquet/schema_test.cc b/cpp/src/parquet/schema_test.cc index 17455e7e4e09..dd3fbd58d996 100644 --- a/cpp/src/parquet/schema_test.cc +++ b/cpp/src/parquet/schema_test.cc @@ -411,6 +411,14 @@ TEST_F(TestGroupNode, VectorValidation) { ConvertedType::NONE, /*field_id=*/-1, /*vector_length=*/4), ParquetException); + ASSERT_THROW(GroupNode::Make("list", Repetition::VECTOR, Fields1(), + ConvertedType::LIST, /*field_id=*/-1, + /*vector_length=*/4), + ParquetException); + ASSERT_THROW(GroupNode::Make("list", Repetition::VECTOR, Fields1(), + LogicalType::List(), /*field_id=*/-1, + /*vector_length=*/4), + ParquetException); } TEST_F(TestGroupNode, VectorLogicalTypeRoundTrip) { @@ -909,7 +917,8 @@ TEST_F(TestSchemaDescriptor, BuildTreeVector) { NodePtr vector = GroupNode::Make("list", Repetition::VECTOR, {element}, /*converted_type=*/ConvertedType::NONE, /*field_id=*/-1, /*vector_length=*/3); - NodePtr embedding = GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}); + NodePtr embedding = GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, + LogicalType::Vector()); descr_.Init(GroupNode::Make("schema", Repetition::REPEATED, {embedding})); @@ -923,13 +932,17 @@ TEST_F(TestSchemaDescriptor, BuildTreeVector) { TEST_F(TestSchemaDescriptor, BuildTreeNestedVector) { NodePtr element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::FLOAT); - NodePtr inner = GroupNode::Make("inner", Repetition::VECTOR, {element}, - /*converted_type=*/ConvertedType::NONE, - /*field_id=*/-1, /*vector_length=*/4); - NodePtr outer = GroupNode::Make("outer", Repetition::VECTOR, {inner}, - /*converted_type=*/ConvertedType::NONE, - /*field_id=*/-1, /*vector_length=*/3); - NodePtr embedding = GroupNode::Make("embedding", Repetition::OPTIONAL, {outer}); + NodePtr inner_list = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/4); + NodePtr inner_vector = GroupNode::Make("element", Repetition::REQUIRED, {inner_list}, + LogicalType::Vector()); + NodePtr outer_list = GroupNode::Make("list", Repetition::VECTOR, {inner_vector}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/3); + NodePtr embedding = + GroupNode::Make("embedding", Repetition::OPTIONAL, {outer_list}, + LogicalType::Vector()); descr_.Init(GroupNode::Make("schema", Repetition::REPEATED, {embedding})); @@ -942,13 +955,17 @@ TEST_F(TestSchemaDescriptor, BuildTreeNestedVector) { TEST_F(TestSchemaDescriptor, BuildTreeNestedVectorLengthOverflow) { NodePtr element = PrimitiveNode::Make("element", Repetition::REQUIRED, Type::FLOAT); - NodePtr inner = GroupNode::Make("inner", Repetition::VECTOR, {element}, - /*converted_type=*/ConvertedType::NONE, - /*field_id=*/-1, /*vector_length=*/46341); - NodePtr outer = GroupNode::Make("outer", Repetition::VECTOR, {inner}, - /*converted_type=*/ConvertedType::NONE, - /*field_id=*/-1, /*vector_length=*/46341); - NodePtr embedding = GroupNode::Make("embedding", Repetition::OPTIONAL, {outer}); + NodePtr inner_list = GroupNode::Make("list", Repetition::VECTOR, {element}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/46341); + NodePtr inner_vector = GroupNode::Make("element", Repetition::REQUIRED, {inner_list}, + LogicalType::Vector()); + NodePtr outer_list = GroupNode::Make("list", Repetition::VECTOR, {inner_vector}, + /*converted_type=*/ConvertedType::NONE, + /*field_id=*/-1, /*vector_length=*/46341); + NodePtr embedding = + GroupNode::Make("embedding", Repetition::OPTIONAL, {outer_list}, + LogicalType::Vector()); ASSERT_THROW(descr_.Init(GroupNode::Make("schema", Repetition::REPEATED, {embedding})), ParquetException); @@ -1047,14 +1064,14 @@ TEST(TestSchemaPrinter, Vector) { {GroupNode::Make( "embedding", Repetition::OPTIONAL, {GroupNode::Make( - "vector", Repetition::VECTOR, + "list", Repetition::VECTOR, {PrimitiveNode::Make("element", Repetition::REQUIRED, Type::FLOAT)}, /*logical_type=*/nullptr, -1, 3)}, LogicalType::Vector())}); std::string expected = R"(repeated group field_id=-1 schema { optional group field_id=-1 embedding (Vector) { - vector group field_id=-1 vector [3] { + vector group field_id=-1 list [3] { required float field_id=-1 element; } } diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index 8fa296e60e50..4ce1a495e69e 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -878,11 +878,11 @@ def _sanitize_table(table, new_schema, flavor): default 1MB. experimental_vector_encoding : bool, default False EXPERIMENTAL: Encode supported fixed-size list values, including the storage - of fixed-shape tensor extension arrays, using Parquet VECTOR repetition. - Unsupported fixed-size list values fall back to the standard LIST encoding: - zero-length lists, variable-width or dictionary or extension elements, - struct elements containing fixed-size lists, and fixed-size lists below - nullable groups. + of fixed-shape tensor extension arrays, using the canonical 3-level Parquet + VECTOR layout. Unsupported fixed-size list values fall back to the standard + LIST encoding: zero-length lists, variable-width or dictionary or extension + elements, struct elements containing fixed-size lists, and fixed-size lists + below nullable groups. store_schema : bool, default True By default, the Arrow schema is serialized and stored in the Parquet file metadata (in the "ARROW:schema" key). When reading the file, From b55c38d5d886f6ea04f14129f30b58eb947827f7 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Mon, 29 Jun 2026 20:32:44 +0200 Subject: [PATCH 7/8] Fix VECTOR scan and schema metadata --- .../parquet/arrow/arrow_reader_writer_test.cc | 52 +++++++++++++++++++ cpp/src/parquet/arrow/schema.cc | 14 +++++ cpp/src/parquet/file_reader.cc | 10 +++- 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc index 04693c1b1cda..6f37e45b0c98 100644 --- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc +++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc @@ -2876,6 +2876,32 @@ TEST(TestArrowReadWrite, ScanContents) { ASSERT_EQ(num_rows, num_rows_returned); } +TEST(TestArrowReadWrite, ScanContentsVector) { + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 3); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("id", ::arrow::int32(), /*nullable=*/false), + ::arrow::field("embedding", vector_type, /*nullable=*/true)}), + {::arrow::ArrayFromJSON(::arrow::int32(), "[0, 1, 2, 3]"), + ::arrow::ArrayFromJSON( + vector_type, R"([[1, 2, 3], null, [7, 8, 9], [10, 11, 12]])")}); + + ArrowWriterProperties::Builder builder; + builder.enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, + VectorWriterProperties(), builder.build())); + ASSERT_OK_AND_ASSIGN(auto reader, OpenFile(std::make_shared(buffer), + ::arrow::default_memory_pool())); + + int64_t num_rows_returned = 0; + ASSERT_OK_NO_THROW(reader->ScanContents({}, 2, &num_rows_returned)); + ASSERT_EQ(table->num_rows(), num_rows_returned); + + ASSERT_OK_NO_THROW(reader->ScanContents({1}, 2, &num_rows_returned)); + ASSERT_EQ(table->num_rows(), num_rows_returned); +} + TEST(TestArrowReadWrite, ReadColumnSubset) { const int num_columns = 20; const int num_rows = 1000; @@ -3596,6 +3622,32 @@ TEST(ArrowReadWrite, FixedSizeListVectorNullableRoundTripNullPatterns) { } } +TEST(ArrowReadWrite, FixedSizeListVectorRestoresStoredChildMetadata) { + auto child_metadata = ::arrow::key_value_metadata({{"child_key", "child_value"}}); + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("item", ::arrow::int32(), /*nullable=*/false, child_metadata), 2); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("root", vector_type, /*nullable=*/true)}), + {::arrow::ArrayFromJSON(vector_type, R"([[1, 2], null, [5, 6]])")}); + + ArrowWriterProperties::Builder builder; + builder.store_schema()->enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/3, + VectorWriterProperties(), builder.build())); + + std::unique_ptr reader; + FileReaderBuilder reader_builder; + ASSERT_OK_NO_THROW(reader_builder.Open(std::make_shared(buffer))); + ASSERT_OK(reader_builder.Build(&reader)); + ASSERT_OK_AND_ASSIGN(auto result, reader->ReadTable()); + + const auto& result_type = + checked_cast(*result->schema()->field(0)->type()); + ASSERT_NE(nullptr, result_type.value_field()->metadata()); + ASSERT_TRUE(result_type.value_field()->metadata()->Equals(*child_metadata)); +} + TEST(ArrowReadWrite, FixedSizeListVectorStructOfVectorsRoundTrip) { auto ids_type = ::arrow::fixed_size_list( ::arrow::field("value", ::arrow::int32(), /*nullable=*/true), 2); diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc index 2519665b39a8..2d1b7b481342 100644 --- a/cpp/src/parquet/arrow/schema.cc +++ b/cpp/src/parquet/arrow/schema.cc @@ -1252,6 +1252,20 @@ std::function(FieldVector)> GetNestedFactory( return [](FieldVector fields) { return ::arrow::struct_(std::move(fields)); }; } break; + case ::arrow::Type::FIXED_SIZE_LIST: + if (origin_type.id() == ::arrow::Type::FIXED_SIZE_LIST) { + const auto origin_list_size = + checked_cast(origin_type).list_size(); + const auto inferred_list_size = + checked_cast(inferred_type).list_size(); + if (origin_list_size == inferred_list_size) { + return [origin_list_size](FieldVector fields) { + DCHECK_EQ(fields.size(), 1); + return ::arrow::fixed_size_list(std::move(fields[0]), origin_list_size); + }; + } + } + break; case ::arrow::Type::LIST: case ::arrow::Type::LARGE_LIST: { switch (origin_type.id()) { diff --git a/cpp/src/parquet/file_reader.cc b/cpp/src/parquet/file_reader.cc index 2f46a5e296f8..df878ffb8779 100644 --- a/cpp/src/parquet/file_reader.cc +++ b/cpp/src/parquet/file_reader.cc @@ -980,10 +980,18 @@ int64_t ScanFileContents(std::vector columns, const int32_t column_batch_si total_rows[col]++; } } - } else { + } else if (!col_reader->descr()->in_vector_column()) { total_rows[col] += levels_read; } } + if (col_reader->descr()->max_repetition_level() == 0 && + col_reader->descr()->in_vector_column()) { + // VECTOR columns without repeated ancestors have one level entry per + // physical vector slot, so levels_read would over-count rows by the + // vector length. Use the row-group row count after still scanning the + // column contents above. + total_rows[col] += group_reader->metadata()->num_rows(); + } col++; } } From b33da543888591c7fadaeedeb57e2c35d7d54283 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Mon, 29 Jun 2026 20:59:10 +0200 Subject: [PATCH 8/8] Fix VECTOR level building for ListView --- .../parquet/arrow/arrow_reader_writer_test.cc | 80 ++++++++++++++++++- cpp/src/parquet/arrow/arrow_schema_test.cc | 6 +- cpp/src/parquet/arrow/path_internal.cc | 15 ++-- cpp/src/parquet/schema_test.cc | 22 +++-- 4 files changed, 99 insertions(+), 24 deletions(-) diff --git a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc index 6f37e45b0c98..a77936be0764 100644 --- a/cpp/src/parquet/arrow/arrow_reader_writer_test.cc +++ b/cpp/src/parquet/arrow/arrow_reader_writer_test.cc @@ -2883,8 +2883,8 @@ TEST(TestArrowReadWrite, ScanContentsVector) { ::arrow::schema({::arrow::field("id", ::arrow::int32(), /*nullable=*/false), ::arrow::field("embedding", vector_type, /*nullable=*/true)}), {::arrow::ArrayFromJSON(::arrow::int32(), "[0, 1, 2, 3]"), - ::arrow::ArrayFromJSON( - vector_type, R"([[1, 2, 3], null, [7, 8, 9], [10, 11, 12]])")}); + ::arrow::ArrayFromJSON(vector_type, + R"([[1, 2, 3], null, [7, 8, 9], [10, 11, 12]])")}); ArrowWriterProperties::Builder builder; builder.enable_experimental_vector_encoding(); @@ -3487,6 +3487,78 @@ TEST(ArrowReadWrite, LargeListView) { ::arrow::large_list(::arrow::int32()), reader_props); } +TEST(ArrowReadWrite, ListViewOfVectorsRoundTrip) { + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto list_view_type = + ::arrow::list_view(::arrow::field("item", vector_type, /*nullable=*/false)); + auto values = ::arrow::ArrayFromJSON( + vector_type, R"([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12]])"); + auto offsets = ::arrow::ArrayFromJSON(::arrow::int32(), "[0, 2, 0, 4, 1]"); + auto sizes = ::arrow::ArrayFromJSON(::arrow::int32(), "[2, 0, null, 2, 1]"); + ASSERT_OK_AND_ASSIGN(auto array, ::arrow::ListViewArray::FromArrays( + list_view_type, *offsets, *sizes, *values, + ::arrow::default_memory_pool())); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("root", list_view_type, /*nullable=*/true)}), + {array}); + + ArrowWriterProperties::Builder builder; + builder.store_schema()->enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, + VectorWriterProperties(), builder.build())); + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + ASSERT_TRUE(parquet_reader->metadata()->schema()->Column(0)->in_vector_column()); + + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE(DoRoundtrip(table, /*row_group_size=*/2, &result, + VectorWriterProperties(), builder.build())); + ::arrow::AssertTablesEqual(*table, *result, /*same_chunk_layout=*/false); +} + +TEST(ArrowReadWrite, ListViewOfVectorsUnderNullableStructRoundTrip) { + auto vector_type = ::arrow::fixed_size_list( + ::arrow::field("element", ::arrow::int32(), /*nullable=*/false), 2); + auto list_view_type = + ::arrow::list_view(::arrow::field("item", vector_type, /*nullable=*/false)); + auto values = ::arrow::ArrayFromJSON(vector_type, R"([[1, 2], [3, 4], [5, 6]])"); + auto offsets = ::arrow::ArrayFromJSON(::arrow::int32(), "[0, 0, 0, 2]"); + auto sizes = ::arrow::ArrayFromJSON(::arrow::int32(), "[2, 0, 0, 1]"); + ASSERT_OK_AND_ASSIGN( + auto list_view_array, + ::arrow::ListViewArray::FromArrays(list_view_type, *offsets, *sizes, *values, + ::arrow::default_memory_pool())); + auto list_view_field = ::arrow::field("vectors", list_view_type, /*nullable=*/false); + + ::arrow::TypedBufferBuilder bitmap_builder; + ASSERT_OK(bitmap_builder.Append(true)); + ASSERT_OK(bitmap_builder.Append(false)); + ASSERT_OK(bitmap_builder.Append(true)); + ASSERT_OK(bitmap_builder.Append(true)); + ASSERT_OK_AND_ASSIGN(auto null_bitmap, bitmap_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto struct_array, + ::arrow::StructArray::Make({list_view_array}, {list_view_field}, null_bitmap)); + auto struct_type = ::arrow::struct_({list_view_field}); + auto table = ::arrow::Table::Make( + ::arrow::schema({::arrow::field("root", struct_type, /*nullable=*/true)}), + {struct_array}); + + ArrowWriterProperties::Builder builder; + builder.store_schema()->enable_experimental_vector_encoding(); + ASSERT_OK_AND_ASSIGN(auto buffer, + WriteTableToBuffer(table, /*row_group_size=*/2, + VectorWriterProperties(), builder.build())); + auto parquet_reader = ParquetFileReader::Open(std::make_shared(buffer)); + ASSERT_TRUE(parquet_reader->metadata()->schema()->Column(0)->in_vector_column()); + + std::shared_ptr
result; + ASSERT_NO_FATAL_FAILURE(DoRoundtrip(table, /*row_group_size=*/2, &result, + VectorWriterProperties(), builder.build())); + ::arrow::AssertTablesEqual(*table, *result, /*same_chunk_layout=*/false); +} + TEST(ArrowReadWrite, EmptyListView) { auto type = ::arrow::list_view(::arrow::int32()); auto array = ArrayFromJSON(type, "[]"); @@ -3642,8 +3714,8 @@ TEST(ArrowReadWrite, FixedSizeListVectorRestoresStoredChildMetadata) { ASSERT_OK(reader_builder.Build(&reader)); ASSERT_OK_AND_ASSIGN(auto result, reader->ReadTable()); - const auto& result_type = - checked_cast(*result->schema()->field(0)->type()); + const auto& result_type = checked_cast( + *result->schema()->field(0)->type()); ASSERT_NE(nullptr, result_type.value_field()->metadata()); ASSERT_TRUE(result_type.value_field()->metadata()->Equals(*child_metadata)); } diff --git a/cpp/src/parquet/arrow/arrow_schema_test.cc b/cpp/src/parquet/arrow/arrow_schema_test.cc index 9e1e84d66781..a3f861d9d600 100644 --- a/cpp/src/parquet/arrow/arrow_schema_test.cc +++ b/cpp/src/parquet/arrow/arrow_schema_test.cc @@ -353,9 +353,9 @@ TEST_F(TestConvertParquetSchema, VectorElementNameRejected) { TEST_F(TestConvertParquetSchema, VectorMiddleGroupLogicalTypeRejected) { auto element = PrimitiveNode::Make("element", Repetition::REQUIRED, ParquetType::FLOAT); - ASSERT_THROW(GroupNode::Make("list", Repetition::VECTOR, {element}, - LogicalType::List(), -1, 3), - ParquetException); + ASSERT_THROW( + GroupNode::Make("list", Repetition::VECTOR, {element}, LogicalType::List(), -1, 3), + ParquetException); } TEST_F(TestConvertParquetSchema, VectorOnPrimitiveRejected) { diff --git a/cpp/src/parquet/arrow/path_internal.cc b/cpp/src/parquet/arrow/path_internal.cc index 5088d26779c1..54694a55c681 100644 --- a/cpp/src/parquet/arrow/path_internal.cc +++ b/cpp/src/parquet/arrow/path_internal.cc @@ -683,10 +683,10 @@ struct PathInfo { // The vectors are expected to the same length info. // Note index order matters here. - using Node = std::variant; + using Node = + std::variant; std::vector path; std::shared_ptr primitive_array; @@ -1002,6 +1002,7 @@ class PathBuilder { // Increment necessary due to empty lists. info_.max_def_level++; info_.max_rep_level++; + info_.repeated_ancestor_def_level = info_.max_def_level; // raw_value_offsets() and raw_value_sizes() account for any slice offset. ListPathNode> node( ListViewRangeSelector{array.raw_value_offsets(), @@ -1009,7 +1010,11 @@ class PathBuilder { info_.max_rep_level, info_.max_def_level - 1); info_.path.emplace_back(std::move(node)); nullable_in_parent_ = array.list_view_type()->value_field()->nullable(); - return VisitInline(*array.values()); + const bool saved_nullable_group = nullable_group_since_repeated_; + nullable_group_since_repeated_ = false; + Status status = VisitInline(*array.values()); + nullable_group_since_repeated_ = saved_nullable_group; + return status; } Status Visit(const ::arrow::DictionaryArray& array) { diff --git a/cpp/src/parquet/schema_test.cc b/cpp/src/parquet/schema_test.cc index dd3fbd58d996..0f1b5f96aa4f 100644 --- a/cpp/src/parquet/schema_test.cc +++ b/cpp/src/parquet/schema_test.cc @@ -411,12 +411,12 @@ TEST_F(TestGroupNode, VectorValidation) { ConvertedType::NONE, /*field_id=*/-1, /*vector_length=*/4), ParquetException); - ASSERT_THROW(GroupNode::Make("list", Repetition::VECTOR, Fields1(), - ConvertedType::LIST, /*field_id=*/-1, + ASSERT_THROW(GroupNode::Make("list", Repetition::VECTOR, Fields1(), ConvertedType::LIST, + /*field_id=*/-1, /*vector_length=*/4), ParquetException); - ASSERT_THROW(GroupNode::Make("list", Repetition::VECTOR, Fields1(), - LogicalType::List(), /*field_id=*/-1, + ASSERT_THROW(GroupNode::Make("list", Repetition::VECTOR, Fields1(), LogicalType::List(), + /*field_id=*/-1, /*vector_length=*/4), ParquetException); } @@ -917,8 +917,8 @@ TEST_F(TestSchemaDescriptor, BuildTreeVector) { NodePtr vector = GroupNode::Make("list", Repetition::VECTOR, {element}, /*converted_type=*/ConvertedType::NONE, /*field_id=*/-1, /*vector_length=*/3); - NodePtr embedding = GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, - LogicalType::Vector()); + NodePtr embedding = + GroupNode::Make("embedding", Repetition::OPTIONAL, {vector}, LogicalType::Vector()); descr_.Init(GroupNode::Make("schema", Repetition::REPEATED, {embedding})); @@ -940,9 +940,8 @@ TEST_F(TestSchemaDescriptor, BuildTreeNestedVector) { NodePtr outer_list = GroupNode::Make("list", Repetition::VECTOR, {inner_vector}, /*converted_type=*/ConvertedType::NONE, /*field_id=*/-1, /*vector_length=*/3); - NodePtr embedding = - GroupNode::Make("embedding", Repetition::OPTIONAL, {outer_list}, - LogicalType::Vector()); + NodePtr embedding = GroupNode::Make("embedding", Repetition::OPTIONAL, {outer_list}, + LogicalType::Vector()); descr_.Init(GroupNode::Make("schema", Repetition::REPEATED, {embedding})); @@ -963,9 +962,8 @@ TEST_F(TestSchemaDescriptor, BuildTreeNestedVectorLengthOverflow) { NodePtr outer_list = GroupNode::Make("list", Repetition::VECTOR, {inner_vector}, /*converted_type=*/ConvertedType::NONE, /*field_id=*/-1, /*vector_length=*/46341); - NodePtr embedding = - GroupNode::Make("embedding", Repetition::OPTIONAL, {outer_list}, - LogicalType::Vector()); + NodePtr embedding = GroupNode::Make("embedding", Repetition::OPTIONAL, {outer_list}, + LogicalType::Vector()); ASSERT_THROW(descr_.Init(GroupNode::Make("schema", Repetition::REPEATED, {embedding})), ParquetException);