From a7ac1ae6a19025975a2df2f20684cc78447d3c02 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Mon, 20 Apr 2026 23:58:37 +0000 Subject: [PATCH 01/31] Add PFOR core algorithm and tests Implements the PFOR (Patched Frame of Reference) integer compression algorithm as a standalone utility library in arrow/util/pfor/. Includes: - Cost model for optimal bit width selection (histogram-based) - Vector-level encode/decode with FOR + bit-packing + exceptions - Page-level wrapper with header, offset array, and multi-vector layout - Comprehensive unit tests covering edge cases and round-trips --- cpp/src/arrow/util/CMakeLists.txt | 6 + cpp/src/arrow/util/pfor/pfor.cc | 291 +++++++++++++++ cpp/src/arrow/util/pfor/pfor.h | 146 ++++++++ cpp/src/arrow/util/pfor/pfor_constants.h | 96 +++++ cpp/src/arrow/util/pfor/pfor_test.cc | 431 +++++++++++++++++++++++ cpp/src/arrow/util/pfor/pfor_wrapper.cc | 184 ++++++++++ cpp/src/arrow/util/pfor/pfor_wrapper.h | 83 +++++ 7 files changed, 1237 insertions(+) create mode 100644 cpp/src/arrow/util/pfor/pfor.cc create mode 100644 cpp/src/arrow/util/pfor/pfor.h create mode 100644 cpp/src/arrow/util/pfor/pfor_constants.h create mode 100644 cpp/src/arrow/util/pfor/pfor_test.cc create mode 100644 cpp/src/arrow/util/pfor/pfor_wrapper.cc create mode 100644 cpp/src/arrow/util/pfor/pfor_wrapper.h diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index 628e9a4d1c7e..f396b2a1c5da 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -118,6 +118,12 @@ add_arrow_test(threading-utility-test test_common.cc thread_pool_test.cc) +add_arrow_test(pfor-test + SOURCES + pfor/pfor_test.cc + pfor/pfor.cc + pfor/pfor_wrapper.cc) + add_arrow_benchmark(bit_block_counter_benchmark) add_arrow_benchmark(bit_util_benchmark) add_arrow_benchmark(bitmap_reader_benchmark) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc new file mode 100644 index 000000000000..54bb131d1964 --- /dev/null +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -0,0 +1,291 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Core PFOR (Patched Frame of Reference) compression implementation +// +// Adapted from the Snowflake PFOR encoder (PforEncoder.{hpp,cpp}). +// Key differences from the Snowflake implementation: +// - Vector size: 1024 (not 2048) +// - Max exceptions: uint16 (not uint8) +// - Exception values: original integers (not FOR offsets) +// - Bit packing: Arrow's BitWriter/unpack (not Snowflake's BitPacker) + +#include "arrow/util/pfor/pfor.h" + +#include +#include +#include +#include + +#include "arrow/util/bit_stream_utils_internal.h" +#include "arrow/util/bit_util.h" +#include "arrow/util/bpacking_internal.h" +#include "arrow/util/logging.h" + +namespace arrow { +namespace util { +namespace pfor { + +// ---------------------------------------------------------------------- +// FindOptimalBitWidth: histogram-based cost model + +template +BitWidthResult PforCompression::FindOptimalBitWidth(const UnsignedT* deltas, + uint32_t num_elements) { + constexpr uint8_t max_bits = PforTypeTraits::kMaxBitWidth; + constexpr uint8_t position_bits = 16; // uint16_t for exception position + constexpr uint8_t value_bits = sizeof(T) * 8; + + // Build histogram: histogram[b] = count of deltas requiring exactly b bits + std::array histogram{}; // Support up to 64 bits + for (uint32_t i = 0; i < num_elements; ++i) { + uint8_t bits = PforTypeTraits::BitsRequired(deltas[i]); + histogram[bits]++; + } + + // Evaluate each candidate bit width + uint64_t best_cost = std::numeric_limits::max(); + uint8_t best_bit_width = max_bits; + uint16_t best_num_exceptions = 0; + + uint64_t exceptions_above = num_elements; // All start as potential exceptions + + for (uint8_t b = 0; b <= max_bits; ++b) { + exceptions_above -= histogram[b]; + + if (exceptions_above > PforConstants::kMaxExceptions) { + continue; + } + + uint64_t packing_cost = static_cast(num_elements) * b; + uint64_t exception_cost = exceptions_above * (position_bits + value_bits); + uint64_t total_cost = packing_cost + exception_cost; + + if (total_cost < best_cost) { + best_cost = total_cost; + best_bit_width = b; + best_num_exceptions = static_cast(exceptions_above); + } + } + + return {best_bit_width, best_num_exceptions}; +} + +// ---------------------------------------------------------------------- +// EncodeVector + +template +PforEncodedVector PforCompression::EncodeVector(const T* values, + uint32_t num_elements) { + ARROW_DCHECK(num_elements > 0); + + // Step 1: Find min (frame of reference) + T min_val = values[0]; + for (uint32_t i = 1; i < num_elements; ++i) { + if (values[i] < min_val) min_val = values[i]; + } + + // Step 2: Compute unsigned deltas + const auto unsigned_min = static_cast(min_val); + std::vector deltas(num_elements); + for (uint32_t i = 0; i < num_elements; ++i) { + deltas[i] = static_cast(values[i]) - unsigned_min; + } + + // Step 3: Find optimal bit width + auto [bit_width, num_exceptions] = + FindOptimalBitWidth(deltas.data(), num_elements); + + // Step 4: Collect exceptions and replace with placeholder (0) + PforEncodedVector result; + result.info.frame_of_reference = min_val; + result.info.bit_width = bit_width; + result.info.num_exceptions = num_exceptions; + + if (num_exceptions > 0) { + result.exception_positions.reserve(num_exceptions); + result.exception_values.reserve(num_exceptions); + + UnsignedT mask = (bit_width >= PforTypeTraits::kMaxBitWidth) + ? static_cast(-1) + : (static_cast(1) << bit_width) - 1; + + for (uint32_t i = 0; i < num_elements; ++i) { + if (deltas[i] > mask) { + result.exception_positions.push_back(static_cast(i)); + result.exception_values.push_back(values[i]); // Store ORIGINAL value + deltas[i] = 0; // Placeholder + } + } + } + + // Step 5: Bit-pack the deltas + if (bit_width > 0) { + size_t packed_size = static_cast( + bit_util::BytesForBits(static_cast(num_elements) * bit_width)); + result.packed_values.resize(packed_size, 0); + + bit_util::BitWriter writer(result.packed_values.data(), + static_cast(packed_size)); + for (uint32_t i = 0; i < num_elements; ++i) { + writer.PutValue(static_cast(deltas[i]), bit_width); + } + writer.Flush(); + } + + return result; +} + +// ---------------------------------------------------------------------- +// DecodeVector + +template +size_t PforCompression::DecodeVector(T* values, const uint8_t* data, + uint32_t num_elements) { + // Step 1: Read vector info + auto info = PforVectorInfo::Load(data); + const uint8_t* read_ptr = data + PforVectorInfo::kSerializedSize; + + // Step 2: Handle constant data (bit_width == 0, no exceptions) + if (info.bit_width == 0 && info.num_exceptions == 0) { + std::fill(values, values + num_elements, info.frame_of_reference); + return PforVectorInfo::kSerializedSize; + } + + // Step 3: Unpack bit-packed deltas and add FOR + if (info.bit_width > 0) { + // Use SIMD-optimized unpack for batches of 32 (uint32) or 32 (uint64) + constexpr int kBatchSize = 32; + uint32_t full_batches = num_elements / kBatchSize; + uint32_t remainder = num_elements % kBatchSize; + + UnsignedT* unsigned_values = reinterpret_cast(values); + const auto unsigned_for = static_cast(info.frame_of_reference); + + // Unpack full batches using SIMD + for (uint32_t batch = 0; batch < full_batches; ++batch) { + arrow::internal::unpack(read_ptr, unsigned_values + batch * kBatchSize, + kBatchSize, info.bit_width, + batch * kBatchSize * info.bit_width); + } + + // Unpack remainder using BitReader + if (remainder > 0) { + size_t packed_size = static_cast( + bit_util::BytesForBits(static_cast(num_elements) * info.bit_width)); + bit_util::BitReader reader(read_ptr, static_cast(packed_size)); + // Skip past the full batches + for (uint32_t i = 0; i < full_batches * kBatchSize; ++i) { + uint64_t val; + reader.GetValue(info.bit_width, &val); + } + for (uint32_t i = full_batches * kBatchSize; i < num_elements; ++i) { + uint64_t val; + reader.GetValue(info.bit_width, &val); + unsigned_values[i] = static_cast(val); + } + } + + // Add FOR to all values + for (uint32_t i = 0; i < num_elements; ++i) { + unsigned_values[i] += unsigned_for; + } + + size_t packed_size = static_cast( + bit_util::BytesForBits(static_cast(num_elements) * info.bit_width)); + read_ptr += packed_size; + } else { + // bit_width == 0 but has exceptions - fill with FOR + std::fill(values, values + num_elements, info.frame_of_reference); + } + + // Step 4: Patch exceptions (stored as original values) + if (info.num_exceptions > 0) { + const uint8_t* positions_ptr = read_ptr; + read_ptr += info.num_exceptions * sizeof(uint16_t); + + const uint8_t* values_ptr = read_ptr; + read_ptr += info.num_exceptions * sizeof(T); + + for (uint16_t i = 0; i < info.num_exceptions; ++i) { + uint16_t pos; + std::memcpy(&pos, positions_ptr + i * sizeof(uint16_t), sizeof(uint16_t)); + + T value; + std::memcpy(&value, values_ptr + i * sizeof(T), sizeof(T)); + + values[pos] = value; + } + } + + return static_cast(read_ptr - data); +} + +// ---------------------------------------------------------------------- +// Serialization helpers + +template +size_t PforCompression::SerializedVectorSize(const PforEncodedVector& vec, + uint32_t num_elements) { + size_t size = PforVectorInfo::kSerializedSize; + if (vec.info.bit_width > 0) { + size += static_cast( + bit_util::BytesForBits(static_cast(num_elements) * vec.info.bit_width)); + } + size += vec.info.num_exceptions * sizeof(uint16_t); // positions + size += vec.info.num_exceptions * sizeof(T); // values + return size; +} + +template +size_t PforCompression::SerializeVector(const PforEncodedVector& vec, + uint32_t num_elements, + uint8_t* dest) { + uint8_t* write_ptr = dest; + + // Write vector info + vec.info.Store(write_ptr); + write_ptr += PforVectorInfo::kSerializedSize; + + // Write packed values + if (vec.info.bit_width > 0 && !vec.packed_values.empty()) { + std::memcpy(write_ptr, vec.packed_values.data(), vec.packed_values.size()); + write_ptr += vec.packed_values.size(); + } + + // Write exception positions + if (vec.info.num_exceptions > 0) { + std::memcpy(write_ptr, vec.exception_positions.data(), + vec.info.num_exceptions * sizeof(uint16_t)); + write_ptr += vec.info.num_exceptions * sizeof(uint16_t); + + // Write exception values (original integers) + std::memcpy(write_ptr, vec.exception_values.data(), + vec.info.num_exceptions * sizeof(T)); + write_ptr += vec.info.num_exceptions * sizeof(T); + } + + return static_cast(write_ptr - dest); +} + +// Explicit template instantiations +template class PforCompression; +template class PforCompression; + +} // namespace pfor +} // namespace util +} // namespace arrow diff --git a/cpp/src/arrow/util/pfor/pfor.h b/cpp/src/arrow/util/pfor/pfor.h new file mode 100644 index 000000000000..af25cbbedfc1 --- /dev/null +++ b/cpp/src/arrow/util/pfor/pfor.h @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Core PFOR (Patched Frame of Reference) compression algorithm +// +// PFOR compresses integer columns by: +// 1. Subtracting the minimum value (Frame of Reference) +// 2. Choosing an optimal bit width via a cost model +// 3. Bit-packing the deltas at the chosen width +// 4. Storing outlier values (exceptions) separately + +#pragma once + +#include +#include +#include + +#include "arrow/util/pfor/pfor_constants.h" + +namespace arrow { +namespace util { +namespace pfor { + +// ---------------------------------------------------------------------- +// Per-vector metadata + +/// \brief PFOR vector metadata stored at the start of each compressed vector. +/// +/// For INT32 (7 bytes): [frame_of_reference(4B)] [bit_width(1B)] [num_exceptions(2B)] +/// For INT64 (11 bytes): [frame_of_reference(8B)] [bit_width(1B)] [num_exceptions(2B)] +template +struct PforVectorInfo { + T frame_of_reference = 0; + uint8_t bit_width = 0; + uint16_t num_exceptions = 0; + + /// \brief Store this info to a byte buffer (little-endian) + void Store(uint8_t* dest) const { + std::memcpy(dest, &frame_of_reference, sizeof(T)); + dest[sizeof(T)] = bit_width; + uint16_t le_exceptions = num_exceptions; // Assume LE platform + std::memcpy(dest + sizeof(T) + 1, &le_exceptions, sizeof(uint16_t)); + } + + /// \brief Load this info from a byte buffer (little-endian) + static PforVectorInfo Load(const uint8_t* src) { + PforVectorInfo info; + std::memcpy(&info.frame_of_reference, src, sizeof(T)); + info.bit_width = src[sizeof(T)]; + std::memcpy(&info.num_exceptions, src + sizeof(T) + 1, sizeof(uint16_t)); + return info; + } + + /// \brief Serialized size in bytes + static constexpr uint8_t kSerializedSize = PforTypeTraits::kVectorInfoSize; +}; + +// ---------------------------------------------------------------------- +// Encoded vector representation + +/// \brief A PFOR-encoded vector with all its data sections +template +struct PforEncodedVector { + PforVectorInfo info; + std::vector packed_values; + std::vector exception_positions; + std::vector exception_values; +}; + +// ---------------------------------------------------------------------- +// Cost model result + +/// \brief Result of the optimal bit width search +struct BitWidthResult { + uint8_t bit_width = 0; + uint16_t num_exceptions = 0; +}; + +// ---------------------------------------------------------------------- +// Core compression/decompression + +/// \brief PFOR compression and decompression algorithms +/// +/// \tparam T the integer type (int32_t or int64_t) +template +class PforCompression { + public: + using UnsignedT = typename PforTypeTraits::UnsignedType; + + /// \brief Find the optimal bit width using the cost model + /// + /// Evaluates every candidate bit width and selects the one that + /// minimizes total encoded size (packing cost + exception cost). + /// + /// \param[in] deltas unsigned deltas after FOR subtraction + /// \param[in] num_elements number of elements + /// \return the optimal bit width and exception count + static BitWidthResult FindOptimalBitWidth(const UnsignedT* deltas, + uint32_t num_elements); + + /// \brief Encode a single vector of integers + /// + /// \param[in] values input integer values + /// \param[in] num_elements number of elements (up to vector_size) + /// \return the encoded vector with all sections + static PforEncodedVector EncodeVector(const T* values, uint32_t num_elements); + + /// \brief Decode a single vector from compressed data + /// + /// \param[out] values output buffer for decoded integers + /// \param[in] data pointer to the start of the vector data + /// \param[in] num_elements number of elements in this vector + /// \return number of bytes consumed from data + static size_t DecodeVector(T* values, const uint8_t* data, uint32_t num_elements); + + /// \brief Calculate the serialized size of an encoded vector + static size_t SerializedVectorSize(const PforEncodedVector& vec, + uint32_t num_elements); + + /// \brief Serialize an encoded vector to a byte buffer + /// + /// \param[in] vec the encoded vector + /// \param[in] num_elements number of elements + /// \param[out] dest output buffer (must be large enough) + /// \return number of bytes written + static size_t SerializeVector(const PforEncodedVector& vec, + uint32_t num_elements, uint8_t* dest); +}; + +} // namespace pfor +} // namespace util +} // namespace arrow diff --git a/cpp/src/arrow/util/pfor/pfor_constants.h b/cpp/src/arrow/util/pfor/pfor_constants.h new file mode 100644 index 000000000000..1820ede324f0 --- /dev/null +++ b/cpp/src/arrow/util/pfor/pfor_constants.h @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Constants for PFOR (Patched Frame of Reference) compression + +#pragma once + +#include +#include + +namespace arrow { +namespace util { +namespace pfor { + +/// \brief Constants used throughout PFOR compression +class PforConstants { + public: + /// Number of elements compressed together as a unit. + static constexpr uint32_t kPforVectorSize = 1024; + + /// log2(kPforVectorSize) + static constexpr uint8_t kDefaultLogVectorSize = 10; + + /// Minimum allowed log vector size + static constexpr uint8_t kMinLogVectorSize = 3; + + /// Maximum allowed log vector size + static constexpr uint8_t kMaxLogVectorSize = 15; + + /// Type used to store vector data offsets (supports pages up to 4GB) + using OffsetType = uint32_t; + + /// Type used to store exception positions within a compressed vector. + using PositionType = uint16_t; + + /// Maximum number of exceptions per vector (uint16 limit). + static constexpr uint16_t kMaxExceptions = 65535; + + /// Page header size in bytes. + static constexpr uint8_t kHeaderSize = 7; + + /// Packing mode: FOR + bit-packing (currently the only mode). + static constexpr uint8_t kPackingModeForBitPack = 0; +}; + +/// \brief Type traits for PFOR integer types +template +struct PforTypeTraits {}; + +template <> +struct PforTypeTraits { + using UnsignedType = uint32_t; + static constexpr uint8_t kMaxBitWidth = 32; + static constexpr uint8_t kValueByteWidth = 4; + + /// PforVectorInfo size: 4B FOR + 1B bitWidth + 2B numExceptions = 7 bytes + static constexpr uint8_t kVectorInfoSize = 7; + + static uint8_t BitsRequired(uint32_t value) { + if (value == 0) return 0; + return static_cast(32 - __builtin_clz(value)); + } +}; + +template <> +struct PforTypeTraits { + using UnsignedType = uint64_t; + static constexpr uint8_t kMaxBitWidth = 64; + static constexpr uint8_t kValueByteWidth = 8; + + /// PforVectorInfo size: 8B FOR + 1B bitWidth + 2B numExceptions = 11 bytes + static constexpr uint8_t kVectorInfoSize = 11; + + static uint8_t BitsRequired(uint64_t value) { + if (value == 0) return 0; + return static_cast(64 - __builtin_clzll(value)); + } +}; + +} // namespace pfor +} // namespace util +} // namespace arrow diff --git a/cpp/src/arrow/util/pfor/pfor_test.cc b/cpp/src/arrow/util/pfor/pfor_test.cc new file mode 100644 index 000000000000..c9e6f77f05ab --- /dev/null +++ b/cpp/src/arrow/util/pfor/pfor_test.cc @@ -0,0 +1,431 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include +#include + +#include "arrow/util/pfor/pfor.h" +#include "arrow/util/pfor/pfor_wrapper.h" + +namespace arrow::util::pfor { + +// ====================================================================== +// Constants Tests + +TEST(PforConstantsTest, VectorSizeIsPowerOfTwo) { + EXPECT_EQ(PforConstants::kPforVectorSize, 1024u); + EXPECT_EQ(1u << PforConstants::kDefaultLogVectorSize, + PforConstants::kPforVectorSize); +} + +TEST(PforConstantsTest, VectorInfoSizes) { + EXPECT_EQ(PforTypeTraits::kVectorInfoSize, 7); + EXPECT_EQ(PforTypeTraits::kVectorInfoSize, 11); +} + +// ====================================================================== +// BitsRequired Tests + +TEST(PforBitsRequiredTest, Int32) { + EXPECT_EQ(PforTypeTraits::BitsRequired(0), 0); + EXPECT_EQ(PforTypeTraits::BitsRequired(1), 1); + EXPECT_EQ(PforTypeTraits::BitsRequired(2), 2); + EXPECT_EQ(PforTypeTraits::BitsRequired(3), 2); + EXPECT_EQ(PforTypeTraits::BitsRequired(255), 8); + EXPECT_EQ(PforTypeTraits::BitsRequired(256), 9); + EXPECT_EQ(PforTypeTraits::BitsRequired(0xFFFFFFFF), 32); +} + +TEST(PforBitsRequiredTest, Int64) { + EXPECT_EQ(PforTypeTraits::BitsRequired(0), 0); + EXPECT_EQ(PforTypeTraits::BitsRequired(1), 1); + EXPECT_EQ(PforTypeTraits::BitsRequired(0xFFFFFFFFFFFFFFFFULL), 64); +} + +// ====================================================================== +// VectorInfo Serialization Tests + +TEST(PforVectorInfoTest, Int32RoundTrip) { + PforVectorInfo info; + info.frame_of_reference = -42; + info.bit_width = 17; + info.num_exceptions = 300; + + uint8_t buf[7]; + info.Store(buf); + auto loaded = PforVectorInfo::Load(buf); + + EXPECT_EQ(loaded.frame_of_reference, -42); + EXPECT_EQ(loaded.bit_width, 17); + EXPECT_EQ(loaded.num_exceptions, 300); +} + +TEST(PforVectorInfoTest, Int64RoundTrip) { + PforVectorInfo info; + info.frame_of_reference = -123456789012345LL; + info.bit_width = 48; + info.num_exceptions = 65000; + + uint8_t buf[11]; + info.Store(buf); + auto loaded = PforVectorInfo::Load(buf); + + EXPECT_EQ(loaded.frame_of_reference, -123456789012345LL); + EXPECT_EQ(loaded.bit_width, 48); + EXPECT_EQ(loaded.num_exceptions, 65000); +} + +// ====================================================================== +// Cost Model Tests + +TEST(PforCostModelTest, AllIdentical) { + // All deltas are 0 => bit_width should be 0, no exceptions + std::vector deltas(100, 0); + auto result = PforCompression::FindOptimalBitWidth(deltas.data(), 100); + EXPECT_EQ(result.bit_width, 0); + EXPECT_EQ(result.num_exceptions, 0); +} + +TEST(PforCostModelTest, SingleOutlier) { + // 99 values fit in 3 bits, 1 outlier needs 16 bits + std::vector deltas(100, 5); // all fit in 3 bits + deltas[50] = 50000; // outlier: 16 bits + auto result = PforCompression::FindOptimalBitWidth(deltas.data(), 100); + // Cost at bit_width=3: 100*3 + 1*(16+32) = 300 + 48 = 348 + // Cost at bit_width=16: 100*16 + 0 = 1600 + // 348 < 1600, so should pick 3 with 1 exception + EXPECT_EQ(result.bit_width, 3); + EXPECT_EQ(result.num_exceptions, 1); +} + +TEST(PforCostModelTest, NoOutliers) { + // All values fit in 8 bits + std::vector deltas(100); + for (uint32_t i = 0; i < 100; ++i) deltas[i] = i * 2; + auto result = PforCompression::FindOptimalBitWidth(deltas.data(), 100); + EXPECT_EQ(result.num_exceptions, 0); + EXPECT_LE(result.bit_width, 8); +} + +// ====================================================================== +// Vector Encode/Decode Round-Trip Tests + +TEST(PforVectorTest, Int32SimpleSequence) { + std::vector values(64); + std::iota(values.begin(), values.end(), 100); + + auto encoded = PforCompression::EncodeVector(values.data(), 64); + EXPECT_EQ(encoded.info.frame_of_reference, 100); + EXPECT_EQ(encoded.info.num_exceptions, 0); + + // Serialize then decode + size_t serialized_size = + PforCompression::SerializedVectorSize(encoded, 64); + std::vector buffer(serialized_size); + PforCompression::SerializeVector(encoded, 64, buffer.data()); + + std::vector decoded(64); + PforCompression::DecodeVector(decoded.data(), buffer.data(), 64); + + EXPECT_EQ(values, decoded); +} + +TEST(PforVectorTest, Int32WithOutlier) { + std::vector values = {100, 102, 101, 103, 100, 99, 50000, 104}; + + auto encoded = PforCompression::EncodeVector(values.data(), 8); + EXPECT_EQ(encoded.info.frame_of_reference, 99); + EXPECT_GT(encoded.info.num_exceptions, 0u); + + size_t serialized_size = + PforCompression::SerializedVectorSize(encoded, 8); + std::vector buffer(serialized_size); + PforCompression::SerializeVector(encoded, 8, buffer.data()); + + std::vector decoded(8); + PforCompression::DecodeVector(decoded.data(), buffer.data(), 8); + + EXPECT_EQ(values, decoded); +} + +TEST(PforVectorTest, Int32AllIdentical) { + std::vector values(100, 42); + + auto encoded = PforCompression::EncodeVector(values.data(), 100); + EXPECT_EQ(encoded.info.bit_width, 0); + EXPECT_EQ(encoded.info.num_exceptions, 0); + + size_t serialized_size = + PforCompression::SerializedVectorSize(encoded, 100); + std::vector buffer(serialized_size); + PforCompression::SerializeVector(encoded, 100, buffer.data()); + + std::vector decoded(100); + PforCompression::DecodeVector(decoded.data(), buffer.data(), 100); + + EXPECT_EQ(values, decoded); +} + +TEST(PforVectorTest, Int32NegativeValues) { + std::vector values = {-100, -50, -200, -1, -150}; + + auto encoded = PforCompression::EncodeVector(values.data(), 5); + EXPECT_EQ(encoded.info.frame_of_reference, -200); + + size_t serialized_size = + PforCompression::SerializedVectorSize(encoded, 5); + std::vector buffer(serialized_size); + PforCompression::SerializeVector(encoded, 5, buffer.data()); + + std::vector decoded(5); + PforCompression::DecodeVector(decoded.data(), buffer.data(), 5); + + EXPECT_EQ(values, decoded); +} + +TEST(PforVectorTest, Int32MinMaxEdge) { + std::vector values = {std::numeric_limits::min(), + std::numeric_limits::max(), 0, -1, 1}; + + auto encoded = PforCompression::EncodeVector(values.data(), 5); + + size_t serialized_size = + PforCompression::SerializedVectorSize(encoded, 5); + std::vector buffer(serialized_size); + PforCompression::SerializeVector(encoded, 5, buffer.data()); + + std::vector decoded(5); + PforCompression::DecodeVector(decoded.data(), buffer.data(), 5); + + EXPECT_EQ(values, decoded); +} + +TEST(PforVectorTest, Int64SimpleSequence) { + std::vector values(64); + std::iota(values.begin(), values.end(), 1000000000LL); + + auto encoded = PforCompression::EncodeVector(values.data(), 64); + + size_t serialized_size = + PforCompression::SerializedVectorSize(encoded, 64); + std::vector buffer(serialized_size); + PforCompression::SerializeVector(encoded, 64, buffer.data()); + + std::vector decoded(64); + PforCompression::DecodeVector(decoded.data(), buffer.data(), 64); + + EXPECT_EQ(values, decoded); +} + +TEST(PforVectorTest, Int64WithOutlier) { + std::vector values(100, 5000000LL); + values[42] = 999999999999LL; // Outlier + + auto encoded = PforCompression::EncodeVector(values.data(), 100); + EXPECT_GT(encoded.info.num_exceptions, 0u); + + size_t serialized_size = + PforCompression::SerializedVectorSize(encoded, 100); + std::vector buffer(serialized_size); + PforCompression::SerializeVector(encoded, 100, buffer.data()); + + std::vector decoded(100); + PforCompression::DecodeVector(decoded.data(), buffer.data(), 100); + + EXPECT_EQ(values, decoded); +} + +// ====================================================================== +// Page-Level Wrapper Tests + +TEST(PforWrapperTest, Int32SmallPage) { + std::vector values = {10, 20, 30, 40, 50}; + + size_t max_size = PforWrapper::GetMaxCompressedSize(5); + std::vector compressed(max_size); + size_t comp_size = max_size; + + PforWrapper::Encode(values.data(), 5, compressed.data(), &comp_size); + EXPECT_GT(comp_size, 0u); + + std::vector decoded(5); + PforWrapper::Decode(decoded.data(), 5, compressed.data(), comp_size); + + EXPECT_EQ(values, decoded); +} + +TEST(PforWrapperTest, Int32ExactOneVector) { + std::vector values(1024); + std::iota(values.begin(), values.end(), 0); + + size_t max_size = PforWrapper::GetMaxCompressedSize(1024); + std::vector compressed(max_size); + size_t comp_size = max_size; + + PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); + + std::vector decoded(1024); + PforWrapper::Decode(decoded.data(), 1024, compressed.data(), comp_size); + + EXPECT_EQ(values, decoded); +} + +TEST(PforWrapperTest, Int32MultipleVectors) { + // 2.5 vectors worth of data + const uint32_t n = 2560; + std::vector values(n); + std::mt19937 rng(42); + std::uniform_int_distribution dist(0, 1000); + for (auto& v : values) v = dist(rng); + + size_t max_size = PforWrapper::GetMaxCompressedSize(n); + std::vector compressed(max_size); + size_t comp_size = max_size; + + PforWrapper::Encode(values.data(), n, compressed.data(), &comp_size); + + std::vector decoded(n); + PforWrapper::Decode(decoded.data(), n, compressed.data(), comp_size); + + EXPECT_EQ(values, decoded); +} + +TEST(PforWrapperTest, Int32WithOutliers) { + std::vector values(1024, 100); + // Sprinkle outliers + values[0] = -999999; + values[100] = 888888; + values[500] = 777777; + values[1023] = -123456; + + size_t max_size = PforWrapper::GetMaxCompressedSize(1024); + std::vector compressed(max_size); + size_t comp_size = max_size; + + PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); + + std::vector decoded(1024); + PforWrapper::Decode(decoded.data(), 1024, compressed.data(), comp_size); + + EXPECT_EQ(values, decoded); +} + +TEST(PforWrapperTest, Int64MultipleVectors) { + const uint32_t n = 3000; + std::vector values(n); + std::mt19937 rng(123); + std::uniform_int_distribution dist(0, 100000); + for (auto& v : values) v = dist(rng); + // Add outliers + values[0] = 9999999999999LL; + values[1500] = -9999999999999LL; + + size_t max_size = PforWrapper::GetMaxCompressedSize(n); + std::vector compressed(max_size); + size_t comp_size = max_size; + + PforWrapper::Encode(values.data(), n, compressed.data(), &comp_size); + + std::vector decoded(n); + PforWrapper::Decode(decoded.data(), n, compressed.data(), comp_size); + + EXPECT_EQ(values, decoded); +} + +TEST(PforWrapperTest, Int32SingleElement) { + std::vector values = {42}; + + size_t max_size = PforWrapper::GetMaxCompressedSize(1); + std::vector compressed(max_size); + size_t comp_size = max_size; + + PforWrapper::Encode(values.data(), 1, compressed.data(), &comp_size); + + std::vector decoded(1); + PforWrapper::Decode(decoded.data(), 1, compressed.data(), comp_size); + + EXPECT_EQ(values, decoded); +} + +TEST(PforWrapperTest, Int32AllZeros) { + std::vector values(1024, 0); + + size_t max_size = PforWrapper::GetMaxCompressedSize(1024); + std::vector compressed(max_size); + size_t comp_size = max_size; + + PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); + + // Should compress very well (bit_width = 0) + EXPECT_LT(comp_size, 100u); + + std::vector decoded(1024); + PforWrapper::Decode(decoded.data(), 1024, compressed.data(), comp_size); + + EXPECT_EQ(values, decoded); +} + +TEST(PforWrapperTest, Int32LargeRandom) { + const uint32_t n = 10000; + std::vector values(n); + std::mt19937 rng(99); + std::uniform_int_distribution dist( + std::numeric_limits::min(), + std::numeric_limits::max()); + for (auto& v : values) v = dist(rng); + + size_t max_size = PforWrapper::GetMaxCompressedSize(n); + std::vector compressed(max_size); + size_t comp_size = max_size; + + PforWrapper::Encode(values.data(), n, compressed.data(), &comp_size); + + std::vector decoded(n); + PforWrapper::Decode(decoded.data(), n, compressed.data(), comp_size); + + EXPECT_EQ(values, decoded); +} + +// ====================================================================== +// Compression Ratio Test + +TEST(PforCompressionRatioTest, ClusteredDataCompresses) { + // Data clustered around 1000 with one outlier + std::vector values(1024); + std::mt19937 rng(42); + std::uniform_int_distribution dist(1000, 1255); + for (auto& v : values) v = dist(rng); + values[500] = 999999; // One outlier + + size_t max_size = PforWrapper::GetMaxCompressedSize(1024); + std::vector compressed(max_size); + size_t comp_size = max_size; + + PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); + + // PLAIN would be 4096 bytes. PFOR should be much smaller. + size_t plain_size = 1024 * sizeof(int32_t); + EXPECT_LT(comp_size, plain_size / 2); +} + +} // namespace arrow::util::pfor diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.cc b/cpp/src/arrow/util/pfor/pfor_wrapper.cc new file mode 100644 index 000000000000..5556a5e0bf53 --- /dev/null +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.cc @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// PFOR page-level wrapper implementation +// +// Page layout: +// [Header 7B] [Offset Array: numVectors * 4B] [Vector 0] [Vector 1] ... +// +// Each vector: +// [PforVectorInfo] [PackedValues] [ExceptionPositions] [ExceptionValues] + +#include "arrow/util/pfor/pfor_wrapper.h" + +#include +#include +#include + +#include "arrow/util/bit_util.h" +#include "arrow/util/logging.h" + +namespace arrow { +namespace util { +namespace pfor { + +// ---------------------------------------------------------------------- +// Header serialization + +template +void PforWrapper::StoreHeader(uint8_t* dest, const PforHeader& header) { + dest[0] = header.packing_mode; + dest[1] = header.log_vector_size; + dest[2] = header.value_byte_width; + uint32_t le_num = header.num_elements; // Assume LE platform + std::memcpy(dest + 3, &le_num, sizeof(uint32_t)); +} + +template +typename PforWrapper::PforHeader PforWrapper::LoadHeader(const uint8_t* src) { + PforHeader header; + header.packing_mode = src[0]; + header.log_vector_size = src[1]; + header.value_byte_width = src[2]; + std::memcpy(&header.num_elements, src + 3, sizeof(uint32_t)); + return header; +} + +// ---------------------------------------------------------------------- +// Encode + +template +void PforWrapper::Encode(const T* values, uint32_t num_values, char* comp, + size_t* comp_size) { + ARROW_DCHECK(num_values > 0); + ARROW_DCHECK(comp != nullptr); + ARROW_DCHECK(comp_size != nullptr); + + const uint32_t vector_size = kVectorSize; + const uint32_t num_vectors = + (num_values + vector_size - 1) / vector_size; + + auto* dest = reinterpret_cast(comp); + + // Step 1: Write header + PforHeader header; + header.packing_mode = PforConstants::kPackingModeForBitPack; + header.log_vector_size = PforConstants::kDefaultLogVectorSize; + header.value_byte_width = sizeof(T); + header.num_elements = num_values; + StoreHeader(dest, header); + uint8_t* write_ptr = dest + PforConstants::kHeaderSize; + + // Step 2: Reserve space for offset array + uint8_t* offset_array_start = write_ptr; + write_ptr += num_vectors * sizeof(uint32_t); + + // Step 3: Encode each vector and build offset array + const uint8_t* data_start = offset_array_start; // Offsets relative to offset array start + + for (uint32_t v = 0; v < num_vectors; ++v) { + // Record offset (from start of offset array) + uint32_t offset = static_cast(write_ptr - data_start); + std::memcpy(offset_array_start + v * sizeof(uint32_t), &offset, sizeof(uint32_t)); + + // Determine elements in this vector + uint32_t start_idx = v * vector_size; + uint32_t elements_in_vector = + std::min(vector_size, num_values - start_idx); + + // Encode vector + auto encoded = PforCompression::EncodeVector( + values + start_idx, elements_in_vector); + + // Serialize to output + size_t bytes_written = PforCompression::SerializeVector( + encoded, elements_in_vector, write_ptr); + write_ptr += bytes_written; + } + + *comp_size = static_cast(write_ptr - dest); +} + +// ---------------------------------------------------------------------- +// Decode + +template +void PforWrapper::Decode(T* values, uint32_t num_values, const char* comp, + size_t comp_size) { + ARROW_DCHECK(num_values > 0); + ARROW_DCHECK(comp != nullptr); + + const auto* src = reinterpret_cast(comp); + + // Step 1: Read header + PforHeader header = LoadHeader(src); + ARROW_DCHECK(header.packing_mode == PforConstants::kPackingModeForBitPack); + ARROW_DCHECK(header.value_byte_width == sizeof(T)); + + const uint32_t vector_size = 1u << header.log_vector_size; + const uint32_t num_vectors = + (header.num_elements + vector_size - 1) / vector_size; + + // Step 2: Read offset array + const uint8_t* offset_array_start = src + PforConstants::kHeaderSize; + + // Step 3: Decode each vector + for (uint32_t v = 0; v < num_vectors; ++v) { + uint32_t offset; + std::memcpy(&offset, offset_array_start + v * sizeof(uint32_t), + sizeof(uint32_t)); + + const uint8_t* vector_data = offset_array_start + offset; + + uint32_t start_idx = v * vector_size; + uint32_t elements_in_vector = + std::min(vector_size, header.num_elements - start_idx); + + PforCompression::DecodeVector( + values + start_idx, vector_data, elements_in_vector); + } +} + +// ---------------------------------------------------------------------- +// GetMaxCompressedSize + +template +size_t PforWrapper::GetMaxCompressedSize(uint32_t num_values) { + const uint32_t vector_size = kVectorSize; + const uint32_t num_vectors = + (num_values + vector_size - 1) / vector_size; + + // Header + offset array + size_t size = PforConstants::kHeaderSize + num_vectors * sizeof(uint32_t); + + // Worst case per vector: full bit width + all exceptions + size_t max_vector_size = PforVectorInfo::kSerializedSize + + vector_size * sizeof(T) // packed at full width + + vector_size * sizeof(uint16_t) // exception positions + + vector_size * sizeof(T); // exception values + + size += num_vectors * max_vector_size; + return size; +} + +// Explicit template instantiations +template class PforWrapper; +template class PforWrapper; + +} // namespace pfor +} // namespace util +} // namespace arrow diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.h b/cpp/src/arrow/util/pfor/pfor_wrapper.h new file mode 100644 index 000000000000..4a9b6435a5c8 --- /dev/null +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.h @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// High-level wrapper interface for PFOR compression +// +// Handles page-level serialization: header, offset array, and vectors. + +#pragma once + +#include +#include + +#include "arrow/util/pfor/pfor.h" + +namespace arrow { +namespace util { +namespace pfor { + +/// \class PforWrapper +/// \brief High-level interface for PFOR page-level compression +/// +/// Manages the page layout: [Header 7B] [Offset Array] [Vector 0] [Vector 1] ... +/// +/// \tparam T the integer type (int32_t or int64_t) +template +class PforWrapper { + public: + /// \brief Encode integer values into a PFOR-compressed page + /// + /// \param[in] values pointer to input integers + /// \param[in] num_values total number of values + /// \param[out] comp pointer to output buffer (caller must ensure sufficient size) + /// \param[in,out] comp_size input: available buffer size; output: bytes written + static void Encode(const T* values, uint32_t num_values, char* comp, + size_t* comp_size); + + /// \brief Decode a PFOR-compressed page + /// + /// \param[out] values pointer to output buffer + /// \param[in] num_values number of values to decode (from page context) + /// \param[in] comp pointer to compressed data + /// \param[in] comp_size size of compressed data + static void Decode(T* values, uint32_t num_values, const char* comp, + size_t comp_size); + + /// \brief Get the maximum compressed size for a given number of values + /// + /// \param[in] num_values number of integer values + /// \return maximum possible compressed page size in bytes + static size_t GetMaxCompressedSize(uint32_t num_values); + + private: + /// \brief Page header structure (7 bytes) + struct PforHeader { + uint8_t packing_mode; // 0 = FOR + bit-packing + uint8_t log_vector_size; // log2(vector_size) + uint8_t value_byte_width; // sizeof(T): 4 or 8 + uint32_t num_elements; // total element count + }; + + static constexpr uint32_t kVectorSize = PforConstants::kPforVectorSize; + + static void StoreHeader(uint8_t* dest, const PforHeader& header); + static PforHeader LoadHeader(const uint8_t* src); +}; + +} // namespace pfor +} // namespace util +} // namespace arrow From c8501f5236f622c0e86e6ee75aa18bb74e8bcf5a Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 21 Apr 2026 00:00:07 +0000 Subject: [PATCH 02/31] Integrate PFOR encoding into parquet encoder/decoder Adds PFOR = 11 to the Encoding enum and wires it into the parquet read/write pipeline: - PforEncoder in encoder.cc (buffers values, calls PforWrapper::Encode) - PforDecoder in decoder.cc (decodes all values on first access) - PFOR case in column_reader.cc InitializeDataDecoder - Encoding string mapping in types.cc Supports INT32 and INT64 column types. --- cpp/src/parquet/column_reader.cc | 3 +- cpp/src/parquet/decoder.cc | 83 ++++++++++++++++++++++++++++++ cpp/src/parquet/encoder.cc | 86 ++++++++++++++++++++++++++++++++ cpp/src/parquet/types.cc | 2 + cpp/src/parquet/types.h | 3 +- 5 files changed, 175 insertions(+), 2 deletions(-) diff --git a/cpp/src/parquet/column_reader.cc b/cpp/src/parquet/column_reader.cc index 162a72bd1576..b2d5c18630c8 100644 --- a/cpp/src/parquet/column_reader.cc +++ b/cpp/src/parquet/column_reader.cc @@ -971,7 +971,8 @@ class ColumnReaderImplBase { case Encoding::RLE: case Encoding::DELTA_BINARY_PACKED: case Encoding::DELTA_BYTE_ARRAY: - case Encoding::DELTA_LENGTH_BYTE_ARRAY: { + case Encoding::DELTA_LENGTH_BYTE_ARRAY: + case Encoding::PFOR: { auto decoder = MakeTypedDecoder(encoding, descr_, pool_); current_decoder_.SetDecoder(decoder.get()); decoders_[static_cast(encoding)] = std::move(decoder); diff --git a/cpp/src/parquet/decoder.cc b/cpp/src/parquet/decoder.cc index c4d3fe5a8a5a..2d4daafea87f 100644 --- a/cpp/src/parquet/decoder.cc +++ b/cpp/src/parquet/decoder.cc @@ -46,6 +46,7 @@ #include "arrow/util/logging_internal.h" #include "arrow/util/rle_encoding_internal.h" #include "arrow/util/spaced_internal.h" +#include "arrow/util/pfor/pfor_wrapper.h" #include "arrow/util/ubsan.h" #include "arrow/visit_data_inline.h" @@ -2372,6 +2373,79 @@ class ByteStreamSplitDecoder : public ByteStreamSplitDecoderBase +class PforDecoder : public TypedDecoderImpl { + public: + using T = typename DType::c_type; + + explicit PforDecoder(const ColumnDescriptor* descr, + MemoryPool* pool = ::arrow::default_memory_pool()) + : TypedDecoderImpl(descr, Encoding::PFOR), pool_(pool) {} + + void SetData(int num_values, const uint8_t* data, int len) override { + this->num_values_ = num_values; + data_ = data; + data_len_ = len; + values_decoded_ = 0; + } + + int Decode(T* buffer, int max_values) override { + int values_to_decode = std::min(max_values, this->num_values_); + if (values_to_decode == 0) return 0; + + // Decode all values on first call, cache for subsequent calls + if (decoded_values_.empty() && this->num_values_ > 0) { + decoded_values_.resize(this->num_values_); + arrow::util::pfor::PforWrapper::Decode( + decoded_values_.data(), static_cast(this->num_values_), + reinterpret_cast(data_), static_cast(data_len_)); + } + + std::memcpy(buffer, decoded_values_.data() + values_decoded_, + values_to_decode * sizeof(T)); + values_decoded_ += values_to_decode; + this->num_values_ -= values_to_decode; + return values_to_decode; + } + + int DecodeArrow(int num_values, int null_count, const uint8_t* valid_bits, + int64_t valid_bits_offset, + typename EncodingTraits::Accumulator* out) override { + if (null_count != 0) { + ParquetException::NYI("PFOR DecodeArrow with null slots"); + } + std::vector values(num_values); + int decoded_count = Decode(values.data(), num_values); + PARQUET_THROW_NOT_OK(out->AppendValues(values.data(), decoded_count)); + return decoded_count; + } + + int DecodeArrow(int num_values, int null_count, const uint8_t* valid_bits, + int64_t valid_bits_offset, + typename EncodingTraits::DictAccumulator* out) override { + if (null_count != 0) { + ParquetException::NYI("PFOR DecodeArrow with null slots"); + } + std::vector values(num_values); + int decoded_count = Decode(values.data(), num_values); + PARQUET_THROW_NOT_OK(out->Reserve(decoded_count)); + for (int i = 0; i < decoded_count; ++i) { + PARQUET_THROW_NOT_OK(out->Append(values[i])); + } + return decoded_count; + } + + private: + MemoryPool* pool_; + const uint8_t* data_ = nullptr; + int data_len_ = 0; + int values_decoded_ = 0; + std::vector decoded_values_; +}; + } // namespace // ---------------------------------------------------------------------- @@ -2448,6 +2522,15 @@ std::unique_ptr MakeDecoder(Type::type type_num, Encoding::type encodin return std::make_unique(descr); } throw ParquetException("RLE encoding only supports BOOLEAN"); + } else if (encoding == Encoding::PFOR) { + switch (type_num) { + case Type::INT32: + return std::make_unique>(descr, pool); + case Type::INT64: + return std::make_unique>(descr, pool); + default: + throw ParquetException("PFOR decoder only supports INT32 and INT64"); + } } else { ParquetException::NYI("Selected encoding is not supported"); } diff --git a/cpp/src/parquet/encoder.cc b/cpp/src/parquet/encoder.cc index 3e469df277b8..b5a4e25e4016 100644 --- a/cpp/src/parquet/encoder.cc +++ b/cpp/src/parquet/encoder.cc @@ -42,6 +42,7 @@ #include "arrow/util/logging_internal.h" #include "arrow/util/rle_encoding_internal.h" #include "arrow/util/spaced_internal.h" +#include "arrow/util/pfor/pfor_wrapper.h" #include "arrow/util/ubsan.h" #include "arrow/visit_data_inline.h" @@ -1763,6 +1764,82 @@ std::shared_ptr RleBooleanEncoder::FlushValues() { } // namespace +// ---------------------------------------------------------------------- +// PFOR Encoder + +template +class PforEncoder : public EncoderImpl, virtual public TypedEncoder { + public: + using T = typename DType::c_type; + using TypedEncoder::Put; + + explicit PforEncoder(const ColumnDescriptor* descr, MemoryPool* pool) + : EncoderImpl(descr, Encoding::PFOR, pool), pool_(pool) {} + + std::shared_ptr FlushValues() override { + if (values_.empty()) { + PARQUET_ASSIGN_OR_THROW(auto empty_buf, ::arrow::AllocateBuffer(0, pool_)); + return std::move(empty_buf); + } + + const uint32_t num_values = static_cast(values_.size()); + size_t max_size = + arrow::util::pfor::PforWrapper::GetMaxCompressedSize(num_values); + PARQUET_ASSIGN_OR_THROW(auto buffer, ::arrow::AllocateResizableBuffer( + static_cast(max_size), pool_)); + + size_t comp_size = max_size; + arrow::util::pfor::PforWrapper::Encode( + values_.data(), num_values, + reinterpret_cast(buffer->mutable_data()), &comp_size); + + PARQUET_THROW_NOT_OK(buffer->Resize(static_cast(comp_size))); + values_.clear(); + return std::move(buffer); + } + + int64_t EstimatedDataEncodedSize() override { + return static_cast(values_.size() * sizeof(T)); + } + + void Put(const ::arrow::Array& values) override { + const auto& data = *values.data(); + if (data.length > std::numeric_limits::max()) { + throw ParquetException("Array cannot be longer than ", + std::numeric_limits::max()); + } + if (values.null_count() == 0) { + Put(data.template GetValues(1), static_cast(data.length)); + } else { + PutSpaced(data.template GetValues(1), static_cast(data.length), + data.GetValues(0, 0), data.offset); + } + } + + void Put(const T* buffer, int num_values) override { + values_.insert(values_.end(), buffer, buffer + num_values); + } + + void PutSpaced(const T* src, int num_values, const uint8_t* valid_bits, + int64_t valid_bits_offset) override { + if (valid_bits != NULLPTR) { + PARQUET_ASSIGN_OR_THROW(auto buffer, + ::arrow::AllocateBuffer(num_values * sizeof(T), pool_)); + T* dest = reinterpret_cast(buffer->mutable_data()); + int num_valid = + ::arrow::util::internal::SpacedCompress(src, num_values, valid_bits, + valid_bits_offset, dest); + Put(dest, num_valid); + } else { + Put(src, num_values); + } + } + + private: + MemoryPool* pool_; + std::vector values_; +}; + // ---------------------------------------------------------------------- // Factory function @@ -1862,6 +1939,15 @@ std::unique_ptr MakeEncoder(Type::type type_num, Encoding::type encodin throw ParquetException( "DELTA_BYTE_ARRAY only supports BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY"); } + } else if (encoding == Encoding::PFOR) { + switch (type_num) { + case Type::INT32: + return std::make_unique>(descr, pool); + case Type::INT64: + return std::make_unique>(descr, pool); + default: + throw ParquetException("PFOR encoder only supports INT32 and INT64"); + } } else { ParquetException::NYI("Selected encoding is not supported"); } diff --git a/cpp/src/parquet/types.cc b/cpp/src/parquet/types.cc index 9d7604faec30..1567d2b559e2 100644 --- a/cpp/src/parquet/types.cc +++ b/cpp/src/parquet/types.cc @@ -270,6 +270,8 @@ std::string EncodingToString(Encoding::type t) { return "RLE_DICTIONARY"; case Encoding::BYTE_STREAM_SPLIT: return "BYTE_STREAM_SPLIT"; + case Encoding::PFOR: + return "PFOR"; default: return "UNKNOWN"; } diff --git a/cpp/src/parquet/types.h b/cpp/src/parquet/types.h index 687353aa9bcb..acfedb45d795 100644 --- a/cpp/src/parquet/types.h +++ b/cpp/src/parquet/types.h @@ -539,8 +539,9 @@ struct Encoding { DELTA_BYTE_ARRAY = 7, RLE_DICTIONARY = 8, BYTE_STREAM_SPLIT = 9, + PFOR = 11, // Should always be last element (except UNKNOWN) - UNDEFINED = 10, + UNDEFINED = 12, UNKNOWN = 999 }; }; From 8ac3f28cc2c38f596b935ff6d29f36fecbbae68d Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 21 Apr 2026 00:40:10 +0000 Subject: [PATCH 03/31] Add PFOR encoding benchmark Benchmarks encode/decode throughput for int32/int64 across 10 data distributions inspired by Snowflake's NumericComprBenchmark: constant, sequential, small range, high-base-small-range (timestamps), with outliers (exception path), random, TPC-DS date/store/item/quantity keys. Each distribution runs at 1K/10K/100K/1M elements. Reports bytes/s, items/s, and compression ratio. --- cpp/src/arrow/util/CMakeLists.txt | 5 + cpp/src/arrow/util/pfor/pfor_benchmark.cc | 327 ++++++++++++++++++++++ 2 files changed, 332 insertions(+) create mode 100644 cpp/src/arrow/util/pfor/pfor_benchmark.cc diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index f396b2a1c5da..31b793845c4a 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -124,6 +124,11 @@ add_arrow_test(pfor-test pfor/pfor.cc pfor/pfor_wrapper.cc) +add_arrow_benchmark(pfor/pfor_benchmark + EXTRA_SOURCES + pfor/pfor.cc + pfor/pfor_wrapper.cc) + add_arrow_benchmark(bit_block_counter_benchmark) add_arrow_benchmark(bit_util_benchmark) add_arrow_benchmark(bitmap_reader_benchmark) diff --git a/cpp/src/arrow/util/pfor/pfor_benchmark.cc b/cpp/src/arrow/util/pfor/pfor_benchmark.cc new file mode 100644 index 000000000000..aa07025dd88f --- /dev/null +++ b/cpp/src/arrow/util/pfor/pfor_benchmark.cc @@ -0,0 +1,327 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// PFOR encoding/decoding benchmarks. +// +// Data distributions are inspired by Snowflake's NumericComprBenchmark.cpp, +// covering the key archetypes that exercise PFOR's cost model differently: +// - Constant: bit_width=0, best case +// - Sequential: small range, ideal FOR +// - SmallRange: clustered random, good FOR compression +// - HighBaseSmallRange: high absolute values, small delta range (timestamps) +// - WithOutliers: tests exception handling path +// - Random: worst case, full bit-width +// - TPC-DS DateSk/StoreSk/ItemSk/Quantity: realistic surrogate key distributions + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "benchmark/benchmark.h" + +#include "arrow/util/pfor/pfor.h" +#include "arrow/util/pfor/pfor_wrapper.h" + +namespace arrow::util::pfor { +namespace { + +// ====================================================================== +// Data Generators + +using Int32Gen = std::vector (*)(int64_t); +using Int64Gen = std::vector (*)(int64_t); + +template +std::vector GenConstant(int64_t n) { + return std::vector(n, static_cast(42)); +} + +template +std::vector GenSequential(int64_t n) { + std::vector v(n); + std::iota(v.begin(), v.end(), static_cast(0)); + return v; +} + +template +std::vector GenSmallRange(int64_t n) { + std::vector v(n); + std::mt19937_64 rng(12345); + std::uniform_int_distribution dist(100000, 200000); + for (auto& x : v) x = dist(rng); + return v; +} + +template +std::vector GenHighBaseSmallRange(int64_t n) { + std::vector v(n); + const T kBase = static_cast(1704067200); + std::mt19937_64 rng(12345); + std::uniform_int_distribution dist(0, 1000); + for (auto& x : v) x = kBase + dist(rng); + return v; +} + +template +std::vector GenWithOutliers(int64_t n) { + std::vector v(n); + std::mt19937_64 rng(42); + std::uniform_int_distribution small_dist(1000, 1255); + for (auto& x : v) x = small_dist(rng); + std::uniform_int_distribution pos_dist(0, n - 1); + int num_outliers = std::max(static_cast(1), n / 100); + for (int i = 0; i < num_outliers; ++i) { + v[pos_dist(rng)] = static_cast(std::numeric_limits::max() / 2 + i); + } + return v; +} + +template +std::vector GenRandom(int64_t n) { + std::vector v(n); + std::mt19937_64 rng(99); + std::uniform_int_distribution dist(std::numeric_limits::min(), + std::numeric_limits::max()); + for (auto& x : v) x = dist(rng); + return v; +} + +template +std::vector GenTpcdsSoldDateSk(int64_t n) { + std::vector v(n); + const T kBase = 2450815; + std::mt19937_64 rng(12345); + std::uniform_int_distribution dist(0, 1820); + for (auto& x : v) x = kBase + dist(rng); + return v; +} + +template +std::vector GenTpcdsStoreSk(int64_t n) { + std::vector v(n); + std::mt19937_64 rng(12345); + std::uniform_int_distribution dist(1, 1000); + for (auto& x : v) x = dist(rng); + return v; +} + +template +std::vector GenTpcdsItemSk(int64_t n) { + std::vector v(n); + const T kMax = 100000; + std::mt19937_64 rng(12345); + std::exponential_distribution exp_dist(0.00005); + for (auto& x : v) { + T val = static_cast(exp_dist(rng)); + x = std::min(static_cast(val + 1), kMax); + } + return v; +} + +template +std::vector GenTpcdsQuantity(int64_t n) { + std::vector v(n); + std::mt19937_64 rng(12345); + std::uniform_int_distribution small_dist(1, 10); + std::uniform_int_distribution large_dist(11, 100); + std::uniform_int_distribution chance(0, 99); + for (auto& x : v) { + x = (chance(rng) < 90) ? small_dist(rng) : large_dist(rng); + } + return v; +} + +// ====================================================================== +// Benchmark Core + +template +void BM_PforEncodeImpl(benchmark::State& state, + std::vector (*generator)(int64_t)) { + const int64_t num_values = state.range(0); + auto values = generator(num_values); + + size_t max_size = PforWrapper::GetMaxCompressedSize(num_values); + std::vector compressed(max_size); + + for (auto _ : state) { + size_t comp_size = max_size; + PforWrapper::Encode(values.data(), num_values, + compressed.data(), &comp_size); + benchmark::DoNotOptimize(comp_size); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * num_values * + static_cast(sizeof(T))); + state.SetItemsProcessed(state.iterations() * num_values); + + // Report compression ratio + size_t comp_size = max_size; + PforWrapper::Encode(values.data(), num_values, compressed.data(), &comp_size); + state.counters["CompRatio%"] = benchmark::Counter( + 100.0 * static_cast(comp_size) / + static_cast(num_values * sizeof(T))); +} + +template +void BM_PforDecodeImpl(benchmark::State& state, + std::vector (*generator)(int64_t)) { + const int64_t num_values = state.range(0); + auto values = generator(num_values); + + size_t max_size = PforWrapper::GetMaxCompressedSize(num_values); + std::vector compressed(max_size); + size_t comp_size = max_size; + PforWrapper::Encode(values.data(), num_values, compressed.data(), &comp_size); + + std::vector decoded(num_values); + + for (auto _ : state) { + PforWrapper::Decode(decoded.data(), num_values, + compressed.data(), comp_size); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * num_values * + static_cast(sizeof(T))); + state.SetItemsProcessed(state.iterations() * num_values); +} + +// ====================================================================== +// Non-template wrappers to avoid comma-in-macro issues with BENCHMARK_CAPTURE + +void BM_PforEncodeInt32(benchmark::State& state, Int32Gen gen) { + BM_PforEncodeImpl(state, gen); +} +void BM_PforDecodeInt32(benchmark::State& state, Int32Gen gen) { + BM_PforDecodeImpl(state, gen); +} +void BM_PforEncodeInt64(benchmark::State& state, Int64Gen gen) { + BM_PforEncodeImpl(state, gen); +} +void BM_PforDecodeInt64(benchmark::State& state, Int64Gen gen) { + BM_PforDecodeImpl(state, gen); +} + +// ====================================================================== +// Benchmark sizes: 1K, 10K, 100K, 1M + +static void CustomArgs(benchmark::internal::Benchmark* b) { + for (int64_t n : {1024, 10240, 102400, 1048576}) { + b->Arg(n); + } +} + +// ====================================================================== +// INT32 Encode + +BENCHMARK_CAPTURE(BM_PforEncodeInt32, Constant, &GenConstant) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt32, Sequential, &GenSequential) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt32, SmallRange, &GenSmallRange) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt32, HighBaseSmallRange, + &GenHighBaseSmallRange) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt32, WithOutliers, &GenWithOutliers) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt32, Random, &GenRandom) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt32, TpcdsSoldDateSk, + &GenTpcdsSoldDateSk) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt32, TpcdsStoreSk, &GenTpcdsStoreSk) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt32, TpcdsItemSk, &GenTpcdsItemSk) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt32, TpcdsQuantity, + &GenTpcdsQuantity) + ->Apply(CustomArgs); + +// INT32 Decode + +BENCHMARK_CAPTURE(BM_PforDecodeInt32, Constant, &GenConstant) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt32, Sequential, &GenSequential) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt32, SmallRange, &GenSmallRange) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt32, HighBaseSmallRange, + &GenHighBaseSmallRange) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt32, WithOutliers, &GenWithOutliers) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt32, Random, &GenRandom) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt32, TpcdsSoldDateSk, + &GenTpcdsSoldDateSk) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt32, TpcdsStoreSk, &GenTpcdsStoreSk) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt32, TpcdsItemSk, &GenTpcdsItemSk) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt32, TpcdsQuantity, + &GenTpcdsQuantity) + ->Apply(CustomArgs); + +// ====================================================================== +// INT64 Encode + +BENCHMARK_CAPTURE(BM_PforEncodeInt64, Constant, &GenConstant) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt64, Sequential, &GenSequential) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt64, SmallRange, &GenSmallRange) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt64, HighBaseSmallRange, + &GenHighBaseSmallRange) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt64, WithOutliers, &GenWithOutliers) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt64, Random, &GenRandom) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforEncodeInt64, TpcdsSoldDateSk, + &GenTpcdsSoldDateSk) + ->Apply(CustomArgs); + +// INT64 Decode + +BENCHMARK_CAPTURE(BM_PforDecodeInt64, Constant, &GenConstant) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt64, Sequential, &GenSequential) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt64, SmallRange, &GenSmallRange) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt64, HighBaseSmallRange, + &GenHighBaseSmallRange) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt64, WithOutliers, &GenWithOutliers) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt64, Random, &GenRandom) + ->Apply(CustomArgs); +BENCHMARK_CAPTURE(BM_PforDecodeInt64, TpcdsSoldDateSk, + &GenTpcdsSoldDateSk) + ->Apply(CustomArgs); + +} // namespace +} // namespace arrow::util::pfor From ff837d9b00b1e499a3d3a65ba5e62631bcfa7aa1 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Wed, 3 Jun 2026 17:47:14 +0000 Subject: [PATCH 04/31] Use signed integer types consistently per Arrow style guide --- cpp/src/arrow/util/pfor/pfor.cc | 117 +++++++++++------------ cpp/src/arrow/util/pfor/pfor.h | 27 +++--- cpp/src/arrow/util/pfor/pfor_constants.h | 13 +-- cpp/src/arrow/util/pfor/pfor_test.cc | 62 ++++++------ cpp/src/arrow/util/pfor/pfor_wrapper.cc | 56 +++++------ cpp/src/arrow/util/pfor/pfor_wrapper.h | 15 +-- 6 files changed, 142 insertions(+), 148 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc index 54bb131d1964..08d7490b66e8 100644 --- a/cpp/src/arrow/util/pfor/pfor.cc +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -20,7 +20,7 @@ // Adapted from the Snowflake PFOR encoder (PforEncoder.{hpp,cpp}). // Key differences from the Snowflake implementation: // - Vector size: 1024 (not 2048) -// - Max exceptions: uint16 (not uint8) +// - Max exceptions: int16 (not uint8) // - Exception values: original integers (not FOR offsets) // - Bit packing: Arrow's BitWriter/unpack (not Snowflake's BitPacker) @@ -45,40 +45,40 @@ namespace pfor { template BitWidthResult PforCompression::FindOptimalBitWidth(const UnsignedT* deltas, - uint32_t num_elements) { + int32_t num_elements) { constexpr uint8_t max_bits = PforTypeTraits::kMaxBitWidth; - constexpr uint8_t position_bits = 16; // uint16_t for exception position - constexpr uint8_t value_bits = sizeof(T) * 8; + constexpr int32_t position_bits = 16; + constexpr int32_t value_bits = sizeof(T) * 8; // Build histogram: histogram[b] = count of deltas requiring exactly b bits - std::array histogram{}; // Support up to 64 bits - for (uint32_t i = 0; i < num_elements; ++i) { + std::array histogram{}; + for (int32_t i = 0; i < num_elements; ++i) { uint8_t bits = PforTypeTraits::BitsRequired(deltas[i]); histogram[bits]++; } // Evaluate each candidate bit width - uint64_t best_cost = std::numeric_limits::max(); + int64_t best_cost = std::numeric_limits::max(); uint8_t best_bit_width = max_bits; - uint16_t best_num_exceptions = 0; + int16_t best_num_exceptions = 0; - uint64_t exceptions_above = num_elements; // All start as potential exceptions + int64_t exceptions_above = num_elements; for (uint8_t b = 0; b <= max_bits; ++b) { exceptions_above -= histogram[b]; - if (exceptions_above > PforConstants::kMaxExceptions) { + if (exceptions_above > std::numeric_limits::max()) { continue; } - uint64_t packing_cost = static_cast(num_elements) * b; - uint64_t exception_cost = exceptions_above * (position_bits + value_bits); - uint64_t total_cost = packing_cost + exception_cost; + int64_t packing_cost = static_cast(num_elements) * b; + int64_t exception_cost = exceptions_above * (position_bits + value_bits); + int64_t total_cost = packing_cost + exception_cost; if (total_cost < best_cost) { best_cost = total_cost; best_bit_width = b; - best_num_exceptions = static_cast(exceptions_above); + best_num_exceptions = static_cast(exceptions_above); } } @@ -90,19 +90,19 @@ BitWidthResult PforCompression::FindOptimalBitWidth(const UnsignedT* deltas, template PforEncodedVector PforCompression::EncodeVector(const T* values, - uint32_t num_elements) { + int32_t num_elements) { ARROW_DCHECK(num_elements > 0); // Step 1: Find min (frame of reference) T min_val = values[0]; - for (uint32_t i = 1; i < num_elements; ++i) { + for (int32_t i = 1; i < num_elements; ++i) { if (values[i] < min_val) min_val = values[i]; } // Step 2: Compute unsigned deltas const auto unsigned_min = static_cast(min_val); std::vector deltas(num_elements); - for (uint32_t i = 0; i < num_elements; ++i) { + for (int32_t i = 0; i < num_elements; ++i) { deltas[i] = static_cast(values[i]) - unsigned_min; } @@ -124,24 +124,24 @@ PforEncodedVector PforCompression::EncodeVector(const T* values, ? static_cast(-1) : (static_cast(1) << bit_width) - 1; - for (uint32_t i = 0; i < num_elements; ++i) { + for (int32_t i = 0; i < num_elements; ++i) { if (deltas[i] > mask) { - result.exception_positions.push_back(static_cast(i)); - result.exception_values.push_back(values[i]); // Store ORIGINAL value - deltas[i] = 0; // Placeholder + result.exception_positions.push_back(static_cast(i)); + result.exception_values.push_back(values[i]); + deltas[i] = 0; } } } // Step 5: Bit-pack the deltas if (bit_width > 0) { - size_t packed_size = static_cast( - bit_util::BytesForBits(static_cast(num_elements) * bit_width)); - result.packed_values.resize(packed_size, 0); + int64_t packed_size = + bit_util::BytesForBits(static_cast(num_elements) * bit_width); + result.packed_values.resize(static_cast(packed_size), 0); bit_util::BitWriter writer(result.packed_values.data(), static_cast(packed_size)); - for (uint32_t i = 0; i < num_elements; ++i) { + for (int32_t i = 0; i < num_elements; ++i) { writer.PutValue(static_cast(deltas[i]), bit_width); } writer.Flush(); @@ -154,30 +154,29 @@ PforEncodedVector PforCompression::EncodeVector(const T* values, // DecodeVector template -size_t PforCompression::DecodeVector(T* values, const uint8_t* data, - uint32_t num_elements) { +int64_t PforCompression::DecodeVector(T* values, const uint8_t* data, + int32_t num_elements) { // Step 1: Read vector info auto info = PforVectorInfo::Load(data); - const uint8_t* read_ptr = data + PforVectorInfo::kSerializedSize; + const uint8_t* read_ptr = data + PforVectorInfo::kStoredSize; // Step 2: Handle constant data (bit_width == 0, no exceptions) if (info.bit_width == 0 && info.num_exceptions == 0) { std::fill(values, values + num_elements, info.frame_of_reference); - return PforVectorInfo::kSerializedSize; + return PforVectorInfo::kStoredSize; } // Step 3: Unpack bit-packed deltas and add FOR if (info.bit_width > 0) { - // Use SIMD-optimized unpack for batches of 32 (uint32) or 32 (uint64) constexpr int kBatchSize = 32; - uint32_t full_batches = num_elements / kBatchSize; - uint32_t remainder = num_elements % kBatchSize; + int32_t full_batches = num_elements / kBatchSize; + int32_t remainder = num_elements % kBatchSize; UnsignedT* unsigned_values = reinterpret_cast(values); const auto unsigned_for = static_cast(info.frame_of_reference); // Unpack full batches using SIMD - for (uint32_t batch = 0; batch < full_batches; ++batch) { + for (int32_t batch = 0; batch < full_batches; ++batch) { arrow::internal::unpack(read_ptr, unsigned_values + batch * kBatchSize, kBatchSize, info.bit_width, batch * kBatchSize * info.bit_width); @@ -185,15 +184,14 @@ size_t PforCompression::DecodeVector(T* values, const uint8_t* data, // Unpack remainder using BitReader if (remainder > 0) { - size_t packed_size = static_cast( - bit_util::BytesForBits(static_cast(num_elements) * info.bit_width)); + int64_t packed_size = + bit_util::BytesForBits(static_cast(num_elements) * info.bit_width); bit_util::BitReader reader(read_ptr, static_cast(packed_size)); - // Skip past the full batches - for (uint32_t i = 0; i < full_batches * kBatchSize; ++i) { + for (int32_t i = 0; i < full_batches * kBatchSize; ++i) { uint64_t val; reader.GetValue(info.bit_width, &val); } - for (uint32_t i = full_batches * kBatchSize; i < num_elements; ++i) { + for (int32_t i = full_batches * kBatchSize; i < num_elements; ++i) { uint64_t val; reader.GetValue(info.bit_width, &val); unsigned_values[i] = static_cast(val); @@ -201,12 +199,12 @@ size_t PforCompression::DecodeVector(T* values, const uint8_t* data, } // Add FOR to all values - for (uint32_t i = 0; i < num_elements; ++i) { + for (int32_t i = 0; i < num_elements; ++i) { unsigned_values[i] += unsigned_for; } - size_t packed_size = static_cast( - bit_util::BytesForBits(static_cast(num_elements) * info.bit_width)); + int64_t packed_size = + bit_util::BytesForBits(static_cast(num_elements) * info.bit_width); read_ptr += packed_size; } else { // bit_width == 0 but has exceptions - fill with FOR @@ -216,14 +214,14 @@ size_t PforCompression::DecodeVector(T* values, const uint8_t* data, // Step 4: Patch exceptions (stored as original values) if (info.num_exceptions > 0) { const uint8_t* positions_ptr = read_ptr; - read_ptr += info.num_exceptions * sizeof(uint16_t); + read_ptr += info.num_exceptions * sizeof(int16_t); const uint8_t* values_ptr = read_ptr; read_ptr += info.num_exceptions * sizeof(T); - for (uint16_t i = 0; i < info.num_exceptions; ++i) { - uint16_t pos; - std::memcpy(&pos, positions_ptr + i * sizeof(uint16_t), sizeof(uint16_t)); + for (int16_t i = 0; i < info.num_exceptions; ++i) { + int16_t pos; + std::memcpy(&pos, positions_ptr + i * sizeof(int16_t), sizeof(int16_t)); T value; std::memcpy(&value, values_ptr + i * sizeof(T), sizeof(T)); @@ -232,34 +230,33 @@ size_t PforCompression::DecodeVector(T* values, const uint8_t* data, } } - return static_cast(read_ptr - data); + return static_cast(read_ptr - data); } // ---------------------------------------------------------------------- // Serialization helpers template -size_t PforCompression::SerializedVectorSize(const PforEncodedVector& vec, - uint32_t num_elements) { - size_t size = PforVectorInfo::kSerializedSize; +int64_t PforCompression::SerializedVectorSize(const PforEncodedVector& vec, + int32_t num_elements) { + int64_t size = PforVectorInfo::kStoredSize; if (vec.info.bit_width > 0) { - size += static_cast( - bit_util::BytesForBits(static_cast(num_elements) * vec.info.bit_width)); + size += bit_util::BytesForBits(static_cast(num_elements) * vec.info.bit_width); } - size += vec.info.num_exceptions * sizeof(uint16_t); // positions - size += vec.info.num_exceptions * sizeof(T); // values + size += vec.info.num_exceptions * static_cast(sizeof(int16_t)); + size += vec.info.num_exceptions * static_cast(sizeof(T)); return size; } template -size_t PforCompression::SerializeVector(const PforEncodedVector& vec, - uint32_t num_elements, - uint8_t* dest) { +int64_t PforCompression::SerializeVector(const PforEncodedVector& vec, + int32_t num_elements, + uint8_t* dest) { uint8_t* write_ptr = dest; // Write vector info vec.info.Store(write_ptr); - write_ptr += PforVectorInfo::kSerializedSize; + write_ptr += PforVectorInfo::kStoredSize; // Write packed values if (vec.info.bit_width > 0 && !vec.packed_values.empty()) { @@ -270,8 +267,8 @@ size_t PforCompression::SerializeVector(const PforEncodedVector& vec, // Write exception positions if (vec.info.num_exceptions > 0) { std::memcpy(write_ptr, vec.exception_positions.data(), - vec.info.num_exceptions * sizeof(uint16_t)); - write_ptr += vec.info.num_exceptions * sizeof(uint16_t); + vec.info.num_exceptions * sizeof(int16_t)); + write_ptr += vec.info.num_exceptions * sizeof(int16_t); // Write exception values (original integers) std::memcpy(write_ptr, vec.exception_values.data(), @@ -279,7 +276,7 @@ size_t PforCompression::SerializeVector(const PforEncodedVector& vec, write_ptr += vec.info.num_exceptions * sizeof(T); } - return static_cast(write_ptr - dest); + return static_cast(write_ptr - dest); } // Explicit template instantiations diff --git a/cpp/src/arrow/util/pfor/pfor.h b/cpp/src/arrow/util/pfor/pfor.h index af25cbbedfc1..47c85bbeb548 100644 --- a/cpp/src/arrow/util/pfor/pfor.h +++ b/cpp/src/arrow/util/pfor/pfor.h @@ -46,14 +46,13 @@ template struct PforVectorInfo { T frame_of_reference = 0; uint8_t bit_width = 0; - uint16_t num_exceptions = 0; + int16_t num_exceptions = 0; /// \brief Store this info to a byte buffer (little-endian) void Store(uint8_t* dest) const { std::memcpy(dest, &frame_of_reference, sizeof(T)); dest[sizeof(T)] = bit_width; - uint16_t le_exceptions = num_exceptions; // Assume LE platform - std::memcpy(dest + sizeof(T) + 1, &le_exceptions, sizeof(uint16_t)); + std::memcpy(dest + sizeof(T) + 1, &num_exceptions, sizeof(int16_t)); } /// \brief Load this info from a byte buffer (little-endian) @@ -61,12 +60,12 @@ struct PforVectorInfo { PforVectorInfo info; std::memcpy(&info.frame_of_reference, src, sizeof(T)); info.bit_width = src[sizeof(T)]; - std::memcpy(&info.num_exceptions, src + sizeof(T) + 1, sizeof(uint16_t)); + std::memcpy(&info.num_exceptions, src + sizeof(T) + 1, sizeof(int16_t)); return info; } /// \brief Serialized size in bytes - static constexpr uint8_t kSerializedSize = PforTypeTraits::kVectorInfoSize; + static constexpr int64_t kStoredSize = PforTypeTraits::kVectorInfoSize; }; // ---------------------------------------------------------------------- @@ -77,7 +76,7 @@ template struct PforEncodedVector { PforVectorInfo info; std::vector packed_values; - std::vector exception_positions; + std::vector exception_positions; std::vector exception_values; }; @@ -87,7 +86,7 @@ struct PforEncodedVector { /// \brief Result of the optimal bit width search struct BitWidthResult { uint8_t bit_width = 0; - uint16_t num_exceptions = 0; + int16_t num_exceptions = 0; }; // ---------------------------------------------------------------------- @@ -110,14 +109,14 @@ class PforCompression { /// \param[in] num_elements number of elements /// \return the optimal bit width and exception count static BitWidthResult FindOptimalBitWidth(const UnsignedT* deltas, - uint32_t num_elements); + int32_t num_elements); /// \brief Encode a single vector of integers /// /// \param[in] values input integer values /// \param[in] num_elements number of elements (up to vector_size) /// \return the encoded vector with all sections - static PforEncodedVector EncodeVector(const T* values, uint32_t num_elements); + static PforEncodedVector EncodeVector(const T* values, int32_t num_elements); /// \brief Decode a single vector from compressed data /// @@ -125,11 +124,11 @@ class PforCompression { /// \param[in] data pointer to the start of the vector data /// \param[in] num_elements number of elements in this vector /// \return number of bytes consumed from data - static size_t DecodeVector(T* values, const uint8_t* data, uint32_t num_elements); + static int64_t DecodeVector(T* values, const uint8_t* data, int32_t num_elements); /// \brief Calculate the serialized size of an encoded vector - static size_t SerializedVectorSize(const PforEncodedVector& vec, - uint32_t num_elements); + static int64_t SerializedVectorSize(const PforEncodedVector& vec, + int32_t num_elements); /// \brief Serialize an encoded vector to a byte buffer /// @@ -137,8 +136,8 @@ class PforCompression { /// \param[in] num_elements number of elements /// \param[out] dest output buffer (must be large enough) /// \return number of bytes written - static size_t SerializeVector(const PforEncodedVector& vec, - uint32_t num_elements, uint8_t* dest); + static int64_t SerializeVector(const PforEncodedVector& vec, + int32_t num_elements, uint8_t* dest); }; } // namespace pfor diff --git a/cpp/src/arrow/util/pfor/pfor_constants.h b/cpp/src/arrow/util/pfor/pfor_constants.h index 1820ede324f0..244330357c2f 100644 --- a/cpp/src/arrow/util/pfor/pfor_constants.h +++ b/cpp/src/arrow/util/pfor/pfor_constants.h @@ -30,7 +30,7 @@ namespace pfor { class PforConstants { public: /// Number of elements compressed together as a unit. - static constexpr uint32_t kPforVectorSize = 1024; + static constexpr int64_t kPforVectorSize = 1024; /// log2(kPforVectorSize) static constexpr uint8_t kDefaultLogVectorSize = 10; @@ -45,13 +45,10 @@ class PforConstants { using OffsetType = uint32_t; /// Type used to store exception positions within a compressed vector. - using PositionType = uint16_t; - - /// Maximum number of exceptions per vector (uint16 limit). - static constexpr uint16_t kMaxExceptions = 65535; + using PositionType = int16_t; /// Page header size in bytes. - static constexpr uint8_t kHeaderSize = 7; + static constexpr int64_t kHeaderSize = 7; /// Packing mode: FOR + bit-packing (currently the only mode). static constexpr uint8_t kPackingModeForBitPack = 0; @@ -68,7 +65,7 @@ struct PforTypeTraits { static constexpr uint8_t kValueByteWidth = 4; /// PforVectorInfo size: 4B FOR + 1B bitWidth + 2B numExceptions = 7 bytes - static constexpr uint8_t kVectorInfoSize = 7; + static constexpr int64_t kVectorInfoSize = 7; static uint8_t BitsRequired(uint32_t value) { if (value == 0) return 0; @@ -83,7 +80,7 @@ struct PforTypeTraits { static constexpr uint8_t kValueByteWidth = 8; /// PforVectorInfo size: 8B FOR + 1B bitWidth + 2B numExceptions = 11 bytes - static constexpr uint8_t kVectorInfoSize = 11; + static constexpr int64_t kVectorInfoSize = 11; static uint8_t BitsRequired(uint64_t value) { if (value == 0) return 0; diff --git a/cpp/src/arrow/util/pfor/pfor_test.cc b/cpp/src/arrow/util/pfor/pfor_test.cc index c9e6f77f05ab..698e83b53c84 100644 --- a/cpp/src/arrow/util/pfor/pfor_test.cc +++ b/cpp/src/arrow/util/pfor/pfor_test.cc @@ -33,8 +33,8 @@ namespace arrow::util::pfor { // Constants Tests TEST(PforConstantsTest, VectorSizeIsPowerOfTwo) { - EXPECT_EQ(PforConstants::kPforVectorSize, 1024u); - EXPECT_EQ(1u << PforConstants::kDefaultLogVectorSize, + EXPECT_EQ(PforConstants::kPforVectorSize, 1024); + EXPECT_EQ(1 << PforConstants::kDefaultLogVectorSize, PforConstants::kPforVectorSize); } @@ -84,7 +84,7 @@ TEST(PforVectorInfoTest, Int64RoundTrip) { PforVectorInfo info; info.frame_of_reference = -123456789012345LL; info.bit_width = 48; - info.num_exceptions = 65000; + info.num_exceptions = 30000; uint8_t buf[11]; info.Store(buf); @@ -92,7 +92,7 @@ TEST(PforVectorInfoTest, Int64RoundTrip) { EXPECT_EQ(loaded.frame_of_reference, -123456789012345LL); EXPECT_EQ(loaded.bit_width, 48); - EXPECT_EQ(loaded.num_exceptions, 65000); + EXPECT_EQ(loaded.num_exceptions, 30000); } // ====================================================================== @@ -101,7 +101,7 @@ TEST(PforVectorInfoTest, Int64RoundTrip) { TEST(PforCostModelTest, AllIdentical) { // All deltas are 0 => bit_width should be 0, no exceptions std::vector deltas(100, 0); - auto result = PforCompression::FindOptimalBitWidth(deltas.data(), 100); + auto result = PforCompression::FindOptimalBitWidth(deltas.data(), 100); // NOLINT EXPECT_EQ(result.bit_width, 0); EXPECT_EQ(result.num_exceptions, 0); } @@ -121,7 +121,7 @@ TEST(PforCostModelTest, SingleOutlier) { TEST(PforCostModelTest, NoOutliers) { // All values fit in 8 bits std::vector deltas(100); - for (uint32_t i = 0; i < 100; ++i) deltas[i] = i * 2; + for (int32_t i = 0; i < 100; ++i) deltas[i] = i * 2; auto result = PforCompression::FindOptimalBitWidth(deltas.data(), 100); EXPECT_EQ(result.num_exceptions, 0); EXPECT_LE(result.bit_width, 8); @@ -155,7 +155,7 @@ TEST(PforVectorTest, Int32WithOutlier) { auto encoded = PforCompression::EncodeVector(values.data(), 8); EXPECT_EQ(encoded.info.frame_of_reference, 99); - EXPECT_GT(encoded.info.num_exceptions, 0u); + EXPECT_GT(encoded.info.num_exceptions, 0); size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 8); @@ -242,7 +242,7 @@ TEST(PforVectorTest, Int64WithOutlier) { values[42] = 999999999999LL; // Outlier auto encoded = PforCompression::EncodeVector(values.data(), 100); - EXPECT_GT(encoded.info.num_exceptions, 0u); + EXPECT_GT(encoded.info.num_exceptions, 0); size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 100); @@ -261,12 +261,12 @@ TEST(PforVectorTest, Int64WithOutlier) { TEST(PforWrapperTest, Int32SmallPage) { std::vector values = {10, 20, 30, 40, 50}; - size_t max_size = PforWrapper::GetMaxCompressedSize(5); + int64_t max_size = PforWrapper::GetMaxCompressedSize(5); std::vector compressed(max_size); - size_t comp_size = max_size; + int64_t comp_size = max_size; PforWrapper::Encode(values.data(), 5, compressed.data(), &comp_size); - EXPECT_GT(comp_size, 0u); + EXPECT_GT(comp_size, 0); std::vector decoded(5); PforWrapper::Decode(decoded.data(), 5, compressed.data(), comp_size); @@ -278,9 +278,9 @@ TEST(PforWrapperTest, Int32ExactOneVector) { std::vector values(1024); std::iota(values.begin(), values.end(), 0); - size_t max_size = PforWrapper::GetMaxCompressedSize(1024); + int64_t max_size = PforWrapper::GetMaxCompressedSize(1024); std::vector compressed(max_size); - size_t comp_size = max_size; + int64_t comp_size = max_size; PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); @@ -292,15 +292,15 @@ TEST(PforWrapperTest, Int32ExactOneVector) { TEST(PforWrapperTest, Int32MultipleVectors) { // 2.5 vectors worth of data - const uint32_t n = 2560; + const int32_t n = 2560; std::vector values(n); std::mt19937 rng(42); std::uniform_int_distribution dist(0, 1000); for (auto& v : values) v = dist(rng); - size_t max_size = PforWrapper::GetMaxCompressedSize(n); + int64_t max_size = PforWrapper::GetMaxCompressedSize(n); std::vector compressed(max_size); - size_t comp_size = max_size; + int64_t comp_size = max_size; PforWrapper::Encode(values.data(), n, compressed.data(), &comp_size); @@ -318,9 +318,9 @@ TEST(PforWrapperTest, Int32WithOutliers) { values[500] = 777777; values[1023] = -123456; - size_t max_size = PforWrapper::GetMaxCompressedSize(1024); + int64_t max_size = PforWrapper::GetMaxCompressedSize(1024); std::vector compressed(max_size); - size_t comp_size = max_size; + int64_t comp_size = max_size; PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); @@ -331,7 +331,7 @@ TEST(PforWrapperTest, Int32WithOutliers) { } TEST(PforWrapperTest, Int64MultipleVectors) { - const uint32_t n = 3000; + const int32_t n = 3000; std::vector values(n); std::mt19937 rng(123); std::uniform_int_distribution dist(0, 100000); @@ -340,9 +340,9 @@ TEST(PforWrapperTest, Int64MultipleVectors) { values[0] = 9999999999999LL; values[1500] = -9999999999999LL; - size_t max_size = PforWrapper::GetMaxCompressedSize(n); + int64_t max_size = PforWrapper::GetMaxCompressedSize(n); std::vector compressed(max_size); - size_t comp_size = max_size; + int64_t comp_size = max_size; PforWrapper::Encode(values.data(), n, compressed.data(), &comp_size); @@ -355,9 +355,9 @@ TEST(PforWrapperTest, Int64MultipleVectors) { TEST(PforWrapperTest, Int32SingleElement) { std::vector values = {42}; - size_t max_size = PforWrapper::GetMaxCompressedSize(1); + int64_t max_size = PforWrapper::GetMaxCompressedSize(1); std::vector compressed(max_size); - size_t comp_size = max_size; + int64_t comp_size = max_size; PforWrapper::Encode(values.data(), 1, compressed.data(), &comp_size); @@ -370,14 +370,14 @@ TEST(PforWrapperTest, Int32SingleElement) { TEST(PforWrapperTest, Int32AllZeros) { std::vector values(1024, 0); - size_t max_size = PforWrapper::GetMaxCompressedSize(1024); + int64_t max_size = PforWrapper::GetMaxCompressedSize(1024); std::vector compressed(max_size); - size_t comp_size = max_size; + int64_t comp_size = max_size; PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); // Should compress very well (bit_width = 0) - EXPECT_LT(comp_size, 100u); + EXPECT_LT(comp_size, 100); std::vector decoded(1024); PforWrapper::Decode(decoded.data(), 1024, compressed.data(), comp_size); @@ -386,7 +386,7 @@ TEST(PforWrapperTest, Int32AllZeros) { } TEST(PforWrapperTest, Int32LargeRandom) { - const uint32_t n = 10000; + const int32_t n = 10000; std::vector values(n); std::mt19937 rng(99); std::uniform_int_distribution dist( @@ -394,9 +394,9 @@ TEST(PforWrapperTest, Int32LargeRandom) { std::numeric_limits::max()); for (auto& v : values) v = dist(rng); - size_t max_size = PforWrapper::GetMaxCompressedSize(n); + int64_t max_size = PforWrapper::GetMaxCompressedSize(n); std::vector compressed(max_size); - size_t comp_size = max_size; + int64_t comp_size = max_size; PforWrapper::Encode(values.data(), n, compressed.data(), &comp_size); @@ -417,9 +417,9 @@ TEST(PforCompressionRatioTest, ClusteredDataCompresses) { for (auto& v : values) v = dist(rng); values[500] = 999999; // One outlier - size_t max_size = PforWrapper::GetMaxCompressedSize(1024); + int64_t max_size = PforWrapper::GetMaxCompressedSize(1024); std::vector compressed(max_size); - size_t comp_size = max_size; + int64_t comp_size = max_size; PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.cc b/cpp/src/arrow/util/pfor/pfor_wrapper.cc index 5556a5e0bf53..f41ba4738c7c 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.cc +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.cc @@ -44,8 +44,7 @@ void PforWrapper::StoreHeader(uint8_t* dest, const PforHeader& header) { dest[0] = header.packing_mode; dest[1] = header.log_vector_size; dest[2] = header.value_byte_width; - uint32_t le_num = header.num_elements; // Assume LE platform - std::memcpy(dest + 3, &le_num, sizeof(uint32_t)); + std::memcpy(dest + 3, &header.num_elements, sizeof(int32_t)); } template @@ -54,7 +53,7 @@ typename PforWrapper::PforHeader PforWrapper::LoadHeader(const uint8_t* sr header.packing_mode = src[0]; header.log_vector_size = src[1]; header.value_byte_width = src[2]; - std::memcpy(&header.num_elements, src + 3, sizeof(uint32_t)); + std::memcpy(&header.num_elements, src + 3, sizeof(int32_t)); return header; } @@ -62,14 +61,14 @@ typename PforWrapper::PforHeader PforWrapper::LoadHeader(const uint8_t* sr // Encode template -void PforWrapper::Encode(const T* values, uint32_t num_values, char* comp, - size_t* comp_size) { +void PforWrapper::Encode(const T* values, int32_t num_values, char* comp, + int64_t* comp_size) { ARROW_DCHECK(num_values > 0); ARROW_DCHECK(comp != nullptr); ARROW_DCHECK(comp_size != nullptr); - const uint32_t vector_size = kVectorSize; - const uint32_t num_vectors = + const int32_t vector_size = kVectorSize; + const int32_t num_vectors = (num_values + vector_size - 1) / vector_size; auto* dest = reinterpret_cast(comp); @@ -88,16 +87,16 @@ void PforWrapper::Encode(const T* values, uint32_t num_values, char* comp, write_ptr += num_vectors * sizeof(uint32_t); // Step 3: Encode each vector and build offset array - const uint8_t* data_start = offset_array_start; // Offsets relative to offset array start + const uint8_t* data_start = offset_array_start; - for (uint32_t v = 0; v < num_vectors; ++v) { + for (int32_t v = 0; v < num_vectors; ++v) { // Record offset (from start of offset array) uint32_t offset = static_cast(write_ptr - data_start); std::memcpy(offset_array_start + v * sizeof(uint32_t), &offset, sizeof(uint32_t)); // Determine elements in this vector - uint32_t start_idx = v * vector_size; - uint32_t elements_in_vector = + int32_t start_idx = v * vector_size; + int32_t elements_in_vector = std::min(vector_size, num_values - start_idx); // Encode vector @@ -105,20 +104,20 @@ void PforWrapper::Encode(const T* values, uint32_t num_values, char* comp, values + start_idx, elements_in_vector); // Serialize to output - size_t bytes_written = PforCompression::SerializeVector( + int64_t bytes_written = PforCompression::SerializeVector( encoded, elements_in_vector, write_ptr); write_ptr += bytes_written; } - *comp_size = static_cast(write_ptr - dest); + *comp_size = static_cast(write_ptr - dest); } // ---------------------------------------------------------------------- // Decode template -void PforWrapper::Decode(T* values, uint32_t num_values, const char* comp, - size_t comp_size) { +void PforWrapper::Decode(T* values, int32_t num_values, const char* comp, + int64_t comp_size) { ARROW_DCHECK(num_values > 0); ARROW_DCHECK(comp != nullptr); @@ -129,23 +128,23 @@ void PforWrapper::Decode(T* values, uint32_t num_values, const char* comp, ARROW_DCHECK(header.packing_mode == PforConstants::kPackingModeForBitPack); ARROW_DCHECK(header.value_byte_width == sizeof(T)); - const uint32_t vector_size = 1u << header.log_vector_size; - const uint32_t num_vectors = + const int32_t vector_size = 1 << header.log_vector_size; + const int32_t num_vectors = (header.num_elements + vector_size - 1) / vector_size; // Step 2: Read offset array const uint8_t* offset_array_start = src + PforConstants::kHeaderSize; // Step 3: Decode each vector - for (uint32_t v = 0; v < num_vectors; ++v) { + for (int32_t v = 0; v < num_vectors; ++v) { uint32_t offset; std::memcpy(&offset, offset_array_start + v * sizeof(uint32_t), sizeof(uint32_t)); const uint8_t* vector_data = offset_array_start + offset; - uint32_t start_idx = v * vector_size; - uint32_t elements_in_vector = + int32_t start_idx = v * vector_size; + int32_t elements_in_vector = std::min(vector_size, header.num_elements - start_idx); PforCompression::DecodeVector( @@ -157,19 +156,20 @@ void PforWrapper::Decode(T* values, uint32_t num_values, const char* comp, // GetMaxCompressedSize template -size_t PforWrapper::GetMaxCompressedSize(uint32_t num_values) { - const uint32_t vector_size = kVectorSize; - const uint32_t num_vectors = +int64_t PforWrapper::GetMaxCompressedSize(int32_t num_values) { + const int32_t vector_size = kVectorSize; + const int32_t num_vectors = (num_values + vector_size - 1) / vector_size; // Header + offset array - size_t size = PforConstants::kHeaderSize + num_vectors * sizeof(uint32_t); + int64_t size = PforConstants::kHeaderSize + + num_vectors * static_cast(sizeof(uint32_t)); // Worst case per vector: full bit width + all exceptions - size_t max_vector_size = PforVectorInfo::kSerializedSize - + vector_size * sizeof(T) // packed at full width - + vector_size * sizeof(uint16_t) // exception positions - + vector_size * sizeof(T); // exception values + int64_t max_vector_size = PforVectorInfo::kStoredSize + + vector_size * static_cast(sizeof(T)) // packed at full width + + vector_size * static_cast(sizeof(int16_t)) // exception positions + + vector_size * static_cast(sizeof(T)); // exception values size += num_vectors * max_vector_size; return size; diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.h b/cpp/src/arrow/util/pfor/pfor_wrapper.h index 4a9b6435a5c8..2cb8d8573187 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.h +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.h @@ -45,8 +45,8 @@ class PforWrapper { /// \param[in] num_values total number of values /// \param[out] comp pointer to output buffer (caller must ensure sufficient size) /// \param[in,out] comp_size input: available buffer size; output: bytes written - static void Encode(const T* values, uint32_t num_values, char* comp, - size_t* comp_size); + static void Encode(const T* values, int32_t num_values, char* comp, + int64_t* comp_size); /// \brief Decode a PFOR-compressed page /// @@ -54,14 +54,14 @@ class PforWrapper { /// \param[in] num_values number of values to decode (from page context) /// \param[in] comp pointer to compressed data /// \param[in] comp_size size of compressed data - static void Decode(T* values, uint32_t num_values, const char* comp, - size_t comp_size); + static void Decode(T* values, int32_t num_values, const char* comp, + int64_t comp_size); /// \brief Get the maximum compressed size for a given number of values /// /// \param[in] num_values number of integer values /// \return maximum possible compressed page size in bytes - static size_t GetMaxCompressedSize(uint32_t num_values); + static int64_t GetMaxCompressedSize(int32_t num_values); private: /// \brief Page header structure (7 bytes) @@ -69,10 +69,11 @@ class PforWrapper { uint8_t packing_mode; // 0 = FOR + bit-packing uint8_t log_vector_size; // log2(vector_size) uint8_t value_byte_width; // sizeof(T): 4 or 8 - uint32_t num_elements; // total element count + int32_t num_elements; // total element count }; - static constexpr uint32_t kVectorSize = PforConstants::kPforVectorSize; + static constexpr int32_t kVectorSize = + static_cast(PforConstants::kPforVectorSize); static void StoreHeader(uint8_t* dest, const PforHeader& header); static PforHeader LoadHeader(const uint8_t* src); From f8c70e86b5138acca6180e497f1736fa185bb3bb Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Wed, 3 Jun 2026 17:54:17 +0000 Subject: [PATCH 05/31] Use arrow::util::span for buffer parameters in Store/Load/Decode/Serialize --- cpp/src/arrow/util/pfor/pfor.cc | 16 ++++++----- cpp/src/arrow/util/pfor/pfor.h | 27 ++++++++++-------- cpp/src/arrow/util/pfor/pfor_test.cc | 9 +++--- cpp/src/arrow/util/pfor/pfor_wrapper.cc | 37 +++++++++++++++---------- cpp/src/arrow/util/pfor/pfor_wrapper.h | 5 ++-- 5 files changed, 56 insertions(+), 38 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc index 08d7490b66e8..ef853659292f 100644 --- a/cpp/src/arrow/util/pfor/pfor.cc +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -35,6 +35,7 @@ #include "arrow/util/bit_util.h" #include "arrow/util/bpacking_internal.h" #include "arrow/util/logging.h" +#include "arrow/util/span.h" namespace arrow { namespace util { @@ -154,11 +155,12 @@ PforEncodedVector PforCompression::EncodeVector(const T* values, // DecodeVector template -int64_t PforCompression::DecodeVector(T* values, const uint8_t* data, +int64_t PforCompression::DecodeVector(T* values, + arrow::util::span data, int32_t num_elements) { // Step 1: Read vector info auto info = PforVectorInfo::Load(data); - const uint8_t* read_ptr = data + PforVectorInfo::kStoredSize; + const uint8_t* read_ptr = data.data() + PforVectorInfo::kStoredSize; // Step 2: Handle constant data (bit_width == 0, no exceptions) if (info.bit_width == 0 && info.num_exceptions == 0) { @@ -230,7 +232,7 @@ int64_t PforCompression::DecodeVector(T* values, const uint8_t* data, } } - return static_cast(read_ptr - data); + return static_cast(read_ptr - data.data()); } // ---------------------------------------------------------------------- @@ -251,11 +253,11 @@ int64_t PforCompression::SerializedVectorSize(const PforEncodedVector& vec template int64_t PforCompression::SerializeVector(const PforEncodedVector& vec, int32_t num_elements, - uint8_t* dest) { - uint8_t* write_ptr = dest; + arrow::util::span dest) { + uint8_t* write_ptr = dest.data(); // Write vector info - vec.info.Store(write_ptr); + vec.info.Store(arrow::util::span(write_ptr, PforVectorInfo::kStoredSize)); write_ptr += PforVectorInfo::kStoredSize; // Write packed values @@ -276,7 +278,7 @@ int64_t PforCompression::SerializeVector(const PforEncodedVector& vec, write_ptr += vec.info.num_exceptions * sizeof(T); } - return static_cast(write_ptr - dest); + return static_cast(write_ptr - dest.data()); } // Explicit template instantiations diff --git a/cpp/src/arrow/util/pfor/pfor.h b/cpp/src/arrow/util/pfor/pfor.h index 47c85bbeb548..93c7624d56e8 100644 --- a/cpp/src/arrow/util/pfor/pfor.h +++ b/cpp/src/arrow/util/pfor/pfor.h @@ -30,6 +30,7 @@ #include #include "arrow/util/pfor/pfor_constants.h" +#include "arrow/util/span.h" namespace arrow { namespace util { @@ -49,18 +50,20 @@ struct PforVectorInfo { int16_t num_exceptions = 0; /// \brief Store this info to a byte buffer (little-endian) - void Store(uint8_t* dest) const { - std::memcpy(dest, &frame_of_reference, sizeof(T)); - dest[sizeof(T)] = bit_width; - std::memcpy(dest + sizeof(T) + 1, &num_exceptions, sizeof(int16_t)); + void Store(arrow::util::span dest) const { + uint8_t* ptr = dest.data(); + std::memcpy(ptr, &frame_of_reference, sizeof(T)); + ptr[sizeof(T)] = bit_width; + std::memcpy(ptr + sizeof(T) + 1, &num_exceptions, sizeof(int16_t)); } /// \brief Load this info from a byte buffer (little-endian) - static PforVectorInfo Load(const uint8_t* src) { + static PforVectorInfo Load(arrow::util::span src) { PforVectorInfo info; - std::memcpy(&info.frame_of_reference, src, sizeof(T)); - info.bit_width = src[sizeof(T)]; - std::memcpy(&info.num_exceptions, src + sizeof(T) + 1, sizeof(int16_t)); + const uint8_t* ptr = src.data(); + std::memcpy(&info.frame_of_reference, ptr, sizeof(T)); + info.bit_width = ptr[sizeof(T)]; + std::memcpy(&info.num_exceptions, ptr + sizeof(T) + 1, sizeof(int16_t)); return info; } @@ -121,10 +124,11 @@ class PforCompression { /// \brief Decode a single vector from compressed data /// /// \param[out] values output buffer for decoded integers - /// \param[in] data pointer to the start of the vector data + /// \param[in] data span over the compressed vector data /// \param[in] num_elements number of elements in this vector /// \return number of bytes consumed from data - static int64_t DecodeVector(T* values, const uint8_t* data, int32_t num_elements); + static int64_t DecodeVector(T* values, arrow::util::span data, + int32_t num_elements); /// \brief Calculate the serialized size of an encoded vector static int64_t SerializedVectorSize(const PforEncodedVector& vec, @@ -137,7 +141,8 @@ class PforCompression { /// \param[out] dest output buffer (must be large enough) /// \return number of bytes written static int64_t SerializeVector(const PforEncodedVector& vec, - int32_t num_elements, uint8_t* dest); + int32_t num_elements, + arrow::util::span dest); }; } // namespace pfor diff --git a/cpp/src/arrow/util/pfor/pfor_test.cc b/cpp/src/arrow/util/pfor/pfor_test.cc index 698e83b53c84..156313b08ee2 100644 --- a/cpp/src/arrow/util/pfor/pfor_test.cc +++ b/cpp/src/arrow/util/pfor/pfor_test.cc @@ -26,6 +26,7 @@ #include "arrow/util/pfor/pfor.h" #include "arrow/util/pfor/pfor_wrapper.h" +#include "arrow/util/span.h" namespace arrow::util::pfor { @@ -72,8 +73,8 @@ TEST(PforVectorInfoTest, Int32RoundTrip) { info.num_exceptions = 300; uint8_t buf[7]; - info.Store(buf); - auto loaded = PforVectorInfo::Load(buf); + info.Store(arrow::util::span(buf, 7)); + auto loaded = PforVectorInfo::Load(arrow::util::span(buf, 7)); EXPECT_EQ(loaded.frame_of_reference, -42); EXPECT_EQ(loaded.bit_width, 17); @@ -87,8 +88,8 @@ TEST(PforVectorInfoTest, Int64RoundTrip) { info.num_exceptions = 30000; uint8_t buf[11]; - info.Store(buf); - auto loaded = PforVectorInfo::Load(buf); + info.Store(arrow::util::span(buf, 11)); + auto loaded = PforVectorInfo::Load(arrow::util::span(buf, 11)); EXPECT_EQ(loaded.frame_of_reference, -123456789012345LL); EXPECT_EQ(loaded.bit_width, 48); diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.cc b/cpp/src/arrow/util/pfor/pfor_wrapper.cc index f41ba4738c7c..638c1d9c1226 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.cc +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.cc @@ -31,6 +31,7 @@ #include "arrow/util/bit_util.h" #include "arrow/util/logging.h" +#include "arrow/util/span.h" namespace arrow { namespace util { @@ -40,20 +41,24 @@ namespace pfor { // Header serialization template -void PforWrapper::StoreHeader(uint8_t* dest, const PforHeader& header) { - dest[0] = header.packing_mode; - dest[1] = header.log_vector_size; - dest[2] = header.value_byte_width; - std::memcpy(dest + 3, &header.num_elements, sizeof(int32_t)); +void PforWrapper::StoreHeader(arrow::util::span dest, + const PforHeader& header) { + uint8_t* ptr = dest.data(); + ptr[0] = header.packing_mode; + ptr[1] = header.log_vector_size; + ptr[2] = header.value_byte_width; + std::memcpy(ptr + 3, &header.num_elements, sizeof(int32_t)); } template -typename PforWrapper::PforHeader PforWrapper::LoadHeader(const uint8_t* src) { +typename PforWrapper::PforHeader PforWrapper::LoadHeader( + arrow::util::span src) { PforHeader header; - header.packing_mode = src[0]; - header.log_vector_size = src[1]; - header.value_byte_width = src[2]; - std::memcpy(&header.num_elements, src + 3, sizeof(int32_t)); + const uint8_t* ptr = src.data(); + header.packing_mode = ptr[0]; + header.log_vector_size = ptr[1]; + header.value_byte_width = ptr[2]; + std::memcpy(&header.num_elements, ptr + 3, sizeof(int32_t)); return header; } @@ -79,7 +84,7 @@ void PforWrapper::Encode(const T* values, int32_t num_values, char* comp, header.log_vector_size = PforConstants::kDefaultLogVectorSize; header.value_byte_width = sizeof(T); header.num_elements = num_values; - StoreHeader(dest, header); + StoreHeader(arrow::util::span(dest, PforConstants::kHeaderSize), header); uint8_t* write_ptr = dest + PforConstants::kHeaderSize; // Step 2: Reserve space for offset array @@ -105,7 +110,8 @@ void PforWrapper::Encode(const T* values, int32_t num_values, char* comp, // Serialize to output int64_t bytes_written = PforCompression::SerializeVector( - encoded, elements_in_vector, write_ptr); + encoded, elements_in_vector, + arrow::util::span(write_ptr, dest + *comp_size - write_ptr)); write_ptr += bytes_written; } @@ -124,7 +130,8 @@ void PforWrapper::Decode(T* values, int32_t num_values, const char* comp, const auto* src = reinterpret_cast(comp); // Step 1: Read header - PforHeader header = LoadHeader(src); + PforHeader header = LoadHeader( + arrow::util::span(src, PforConstants::kHeaderSize)); ARROW_DCHECK(header.packing_mode == PforConstants::kPackingModeForBitPack); ARROW_DCHECK(header.value_byte_width == sizeof(T)); @@ -148,7 +155,9 @@ void PforWrapper::Decode(T* values, int32_t num_values, const char* comp, std::min(vector_size, header.num_elements - start_idx); PforCompression::DecodeVector( - values + start_idx, vector_data, elements_in_vector); + values + start_idx, + arrow::util::span(vector_data, src + comp_size - vector_data), + elements_in_vector); } } diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.h b/cpp/src/arrow/util/pfor/pfor_wrapper.h index 2cb8d8573187..542f371efa5c 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.h +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.h @@ -25,6 +25,7 @@ #include #include "arrow/util/pfor/pfor.h" +#include "arrow/util/span.h" namespace arrow { namespace util { @@ -75,8 +76,8 @@ class PforWrapper { static constexpr int32_t kVectorSize = static_cast(PforConstants::kPforVectorSize); - static void StoreHeader(uint8_t* dest, const PforHeader& header); - static PforHeader LoadHeader(const uint8_t* src); + static void StoreHeader(arrow::util::span dest, const PforHeader& header); + static PforHeader LoadHeader(arrow::util::span src); }; } // namespace pfor From 65b29f1e30e72f539babb7254103f7c038d5cfd8 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Wed, 3 Jun 2026 18:01:33 +0000 Subject: [PATCH 06/31] Return Result/Status on decode paths instead of ARROW_DCHECK --- cpp/src/arrow/util/pfor/pfor.cc | 16 ++++++--- cpp/src/arrow/util/pfor/pfor.h | 14 +++++--- cpp/src/arrow/util/pfor/pfor_test.cc | 45 +++++++++++++------------ cpp/src/arrow/util/pfor/pfor_wrapper.cc | 32 +++++++++++++----- cpp/src/arrow/util/pfor/pfor_wrapper.h | 6 ++-- 5 files changed, 73 insertions(+), 40 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc index ef853659292f..354ea7cf81fc 100644 --- a/cpp/src/arrow/util/pfor/pfor.cc +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -35,6 +35,7 @@ #include "arrow/util/bit_util.h" #include "arrow/util/bpacking_internal.h" #include "arrow/util/logging.h" +#include "arrow/util/macros.h" #include "arrow/util/span.h" namespace arrow { @@ -155,13 +156,20 @@ PforEncodedVector PforCompression::EncodeVector(const T* values, // DecodeVector template -int64_t PforCompression::DecodeVector(T* values, - arrow::util::span data, - int32_t num_elements) { +Result PforCompression::DecodeVector(T* values, + arrow::util::span data, + int32_t num_elements) { // Step 1: Read vector info - auto info = PforVectorInfo::Load(data); + ARROW_ASSIGN_OR_RAISE(auto info, PforVectorInfo::Load(data)); const uint8_t* read_ptr = data.data() + PforVectorInfo::kStoredSize; + if (info.bit_width > PforTypeTraits::kMaxBitWidth) { + return Status::Invalid("PFOR bit_width out of range: ", info.bit_width); + } + if (info.num_exceptions < 0) { + return Status::Invalid("PFOR num_exceptions negative: ", info.num_exceptions); + } + // Step 2: Handle constant data (bit_width == 0, no exceptions) if (info.bit_width == 0 && info.num_exceptions == 0) { std::fill(values, values + num_elements, info.frame_of_reference); diff --git a/cpp/src/arrow/util/pfor/pfor.h b/cpp/src/arrow/util/pfor/pfor.h index 93c7624d56e8..258bdbf74efd 100644 --- a/cpp/src/arrow/util/pfor/pfor.h +++ b/cpp/src/arrow/util/pfor/pfor.h @@ -29,6 +29,8 @@ #include #include +#include "arrow/result.h" +#include "arrow/status.h" #include "arrow/util/pfor/pfor_constants.h" #include "arrow/util/span.h" @@ -58,7 +60,11 @@ struct PforVectorInfo { } /// \brief Load this info from a byte buffer (little-endian) - static PforVectorInfo Load(arrow::util::span src) { + static Result Load(arrow::util::span src) { + if (src.size() < static_cast(kStoredSize)) { + return Status::Invalid("PFOR vector info buffer too small: ", src.size(), + " < ", kStoredSize); + } PforVectorInfo info; const uint8_t* ptr = src.data(); std::memcpy(&info.frame_of_reference, ptr, sizeof(T)); @@ -126,9 +132,9 @@ class PforCompression { /// \param[out] values output buffer for decoded integers /// \param[in] data span over the compressed vector data /// \param[in] num_elements number of elements in this vector - /// \return number of bytes consumed from data - static int64_t DecodeVector(T* values, arrow::util::span data, - int32_t num_elements); + /// \return number of bytes consumed from data, or error + static Result DecodeVector(T* values, arrow::util::span data, + int32_t num_elements); /// \brief Calculate the serialized size of an encoded vector static int64_t SerializedVectorSize(const PforEncodedVector& vec, diff --git a/cpp/src/arrow/util/pfor/pfor_test.cc b/cpp/src/arrow/util/pfor/pfor_test.cc index 156313b08ee2..5b9673445f0c 100644 --- a/cpp/src/arrow/util/pfor/pfor_test.cc +++ b/cpp/src/arrow/util/pfor/pfor_test.cc @@ -24,6 +24,7 @@ #include #include +#include "arrow/testing/gtest_util.h" #include "arrow/util/pfor/pfor.h" #include "arrow/util/pfor/pfor_wrapper.h" #include "arrow/util/span.h" @@ -143,10 +144,10 @@ TEST(PforVectorTest, Int32SimpleSequence) { size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 64); std::vector buffer(serialized_size); - PforCompression::SerializeVector(encoded, 64, buffer.data()); + PforCompression::SerializeVector(encoded, 64, buffer); std::vector decoded(64); - PforCompression::DecodeVector(decoded.data(), buffer.data(), 64); + ASSERT_OK(PforCompression::DecodeVector(decoded.data(), buffer, 64)); EXPECT_EQ(values, decoded); } @@ -161,10 +162,10 @@ TEST(PforVectorTest, Int32WithOutlier) { size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 8); std::vector buffer(serialized_size); - PforCompression::SerializeVector(encoded, 8, buffer.data()); + PforCompression::SerializeVector(encoded, 8, buffer); std::vector decoded(8); - PforCompression::DecodeVector(decoded.data(), buffer.data(), 8); + ASSERT_OK(PforCompression::DecodeVector(decoded.data(), buffer, 8)); EXPECT_EQ(values, decoded); } @@ -179,10 +180,10 @@ TEST(PforVectorTest, Int32AllIdentical) { size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 100); std::vector buffer(serialized_size); - PforCompression::SerializeVector(encoded, 100, buffer.data()); + PforCompression::SerializeVector(encoded, 100, buffer); std::vector decoded(100); - PforCompression::DecodeVector(decoded.data(), buffer.data(), 100); + ASSERT_OK(PforCompression::DecodeVector(decoded.data(), buffer, 100)); EXPECT_EQ(values, decoded); } @@ -196,10 +197,10 @@ TEST(PforVectorTest, Int32NegativeValues) { size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 5); std::vector buffer(serialized_size); - PforCompression::SerializeVector(encoded, 5, buffer.data()); + PforCompression::SerializeVector(encoded, 5, buffer); std::vector decoded(5); - PforCompression::DecodeVector(decoded.data(), buffer.data(), 5); + ASSERT_OK(PforCompression::DecodeVector(decoded.data(), buffer, 5)); EXPECT_EQ(values, decoded); } @@ -213,10 +214,10 @@ TEST(PforVectorTest, Int32MinMaxEdge) { size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 5); std::vector buffer(serialized_size); - PforCompression::SerializeVector(encoded, 5, buffer.data()); + PforCompression::SerializeVector(encoded, 5, buffer); std::vector decoded(5); - PforCompression::DecodeVector(decoded.data(), buffer.data(), 5); + ASSERT_OK(PforCompression::DecodeVector(decoded.data(), buffer, 5)); EXPECT_EQ(values, decoded); } @@ -230,10 +231,10 @@ TEST(PforVectorTest, Int64SimpleSequence) { size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 64); std::vector buffer(serialized_size); - PforCompression::SerializeVector(encoded, 64, buffer.data()); + PforCompression::SerializeVector(encoded, 64, buffer); std::vector decoded(64); - PforCompression::DecodeVector(decoded.data(), buffer.data(), 64); + ASSERT_OK(PforCompression::DecodeVector(decoded.data(), buffer, 64)); EXPECT_EQ(values, decoded); } @@ -248,10 +249,10 @@ TEST(PforVectorTest, Int64WithOutlier) { size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 100); std::vector buffer(serialized_size); - PforCompression::SerializeVector(encoded, 100, buffer.data()); + PforCompression::SerializeVector(encoded, 100, buffer); std::vector decoded(100); - PforCompression::DecodeVector(decoded.data(), buffer.data(), 100); + ASSERT_OK(PforCompression::DecodeVector(decoded.data(), buffer, 100)); EXPECT_EQ(values, decoded); } @@ -270,7 +271,7 @@ TEST(PforWrapperTest, Int32SmallPage) { EXPECT_GT(comp_size, 0); std::vector decoded(5); - PforWrapper::Decode(decoded.data(), 5, compressed.data(), comp_size); + ASSERT_OK(PforWrapper::Decode(decoded.data(), 5, compressed.data(), comp_size)); EXPECT_EQ(values, decoded); } @@ -286,7 +287,7 @@ TEST(PforWrapperTest, Int32ExactOneVector) { PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); std::vector decoded(1024); - PforWrapper::Decode(decoded.data(), 1024, compressed.data(), comp_size); + ASSERT_OK(PforWrapper::Decode(decoded.data(), 1024, compressed.data(), comp_size)); EXPECT_EQ(values, decoded); } @@ -306,7 +307,7 @@ TEST(PforWrapperTest, Int32MultipleVectors) { PforWrapper::Encode(values.data(), n, compressed.data(), &comp_size); std::vector decoded(n); - PforWrapper::Decode(decoded.data(), n, compressed.data(), comp_size); + ASSERT_OK(PforWrapper::Decode(decoded.data(), n, compressed.data(), comp_size)); EXPECT_EQ(values, decoded); } @@ -326,7 +327,7 @@ TEST(PforWrapperTest, Int32WithOutliers) { PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); std::vector decoded(1024); - PforWrapper::Decode(decoded.data(), 1024, compressed.data(), comp_size); + ASSERT_OK(PforWrapper::Decode(decoded.data(), 1024, compressed.data(), comp_size)); EXPECT_EQ(values, decoded); } @@ -348,7 +349,7 @@ TEST(PforWrapperTest, Int64MultipleVectors) { PforWrapper::Encode(values.data(), n, compressed.data(), &comp_size); std::vector decoded(n); - PforWrapper::Decode(decoded.data(), n, compressed.data(), comp_size); + ASSERT_OK(PforWrapper::Decode(decoded.data(), n, compressed.data(), comp_size)); EXPECT_EQ(values, decoded); } @@ -363,7 +364,7 @@ TEST(PforWrapperTest, Int32SingleElement) { PforWrapper::Encode(values.data(), 1, compressed.data(), &comp_size); std::vector decoded(1); - PforWrapper::Decode(decoded.data(), 1, compressed.data(), comp_size); + ASSERT_OK(PforWrapper::Decode(decoded.data(), 1, compressed.data(), comp_size)); EXPECT_EQ(values, decoded); } @@ -381,7 +382,7 @@ TEST(PforWrapperTest, Int32AllZeros) { EXPECT_LT(comp_size, 100); std::vector decoded(1024); - PforWrapper::Decode(decoded.data(), 1024, compressed.data(), comp_size); + ASSERT_OK(PforWrapper::Decode(decoded.data(), 1024, compressed.data(), comp_size)); EXPECT_EQ(values, decoded); } @@ -402,7 +403,7 @@ TEST(PforWrapperTest, Int32LargeRandom) { PforWrapper::Encode(values.data(), n, compressed.data(), &comp_size); std::vector decoded(n); - PforWrapper::Decode(decoded.data(), n, compressed.data(), comp_size); + ASSERT_OK(PforWrapper::Decode(decoded.data(), n, compressed.data(), comp_size)); EXPECT_EQ(values, decoded); } diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.cc b/cpp/src/arrow/util/pfor/pfor_wrapper.cc index 638c1d9c1226..8aefd32871e4 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.cc +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.cc @@ -122,18 +122,32 @@ void PforWrapper::Encode(const T* values, int32_t num_values, char* comp, // Decode template -void PforWrapper::Decode(T* values, int32_t num_values, const char* comp, - int64_t comp_size) { - ARROW_DCHECK(num_values > 0); - ARROW_DCHECK(comp != nullptr); +Status PforWrapper::Decode(T* values, int32_t num_values, const char* comp, + int64_t comp_size) { + if (num_values <= 0) { + return Status::Invalid("PFOR num_values must be positive: ", num_values); + } + if (comp == nullptr) { + return Status::Invalid("PFOR compressed data pointer is null"); + } + if (comp_size < PforConstants::kHeaderSize) { + return Status::Invalid("PFOR compressed buffer too small for header: ", comp_size, + " < ", PforConstants::kHeaderSize); + } const auto* src = reinterpret_cast(comp); // Step 1: Read header PforHeader header = LoadHeader( arrow::util::span(src, PforConstants::kHeaderSize)); - ARROW_DCHECK(header.packing_mode == PforConstants::kPackingModeForBitPack); - ARROW_DCHECK(header.value_byte_width == sizeof(T)); + + if (header.packing_mode != PforConstants::kPackingModeForBitPack) { + return Status::Invalid("PFOR unsupported packing mode: ", header.packing_mode); + } + if (header.value_byte_width != sizeof(T)) { + return Status::Invalid("PFOR value_byte_width mismatch: ", header.value_byte_width, + " vs expected ", sizeof(T)); + } const int32_t vector_size = 1 << header.log_vector_size; const int32_t num_vectors = @@ -154,11 +168,13 @@ void PforWrapper::Decode(T* values, int32_t num_values, const char* comp, int32_t elements_in_vector = std::min(vector_size, header.num_elements - start_idx); - PforCompression::DecodeVector( + ARROW_RETURN_NOT_OK(PforCompression::DecodeVector( values + start_idx, arrow::util::span(vector_data, src + comp_size - vector_data), - elements_in_vector); + elements_in_vector)); } + + return Status::OK(); } // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.h b/cpp/src/arrow/util/pfor/pfor_wrapper.h index 542f371efa5c..003a8b4693e6 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.h +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.h @@ -24,6 +24,7 @@ #include #include +#include "arrow/status.h" #include "arrow/util/pfor/pfor.h" #include "arrow/util/span.h" @@ -55,8 +56,9 @@ class PforWrapper { /// \param[in] num_values number of values to decode (from page context) /// \param[in] comp pointer to compressed data /// \param[in] comp_size size of compressed data - static void Decode(T* values, int32_t num_values, const char* comp, - int64_t comp_size); + /// \return Status::OK on success, or an error if the data is malformed + static Status Decode(T* values, int32_t num_values, const char* comp, + int64_t comp_size); /// \brief Get the maximum compressed size for a given number of values /// From 7c5e6e413f75bfd8cb72ceae1b176135ad41eb78 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Wed, 3 Jun 2026 18:03:23 +0000 Subject: [PATCH 07/31] Add static_assert(ARROW_LITTLE_ENDIAN) and replace reinterpret_cast with SafeCopy --- cpp/src/arrow/util/pfor/pfor.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc index 354ea7cf81fc..41d94c3aba9f 100644 --- a/cpp/src/arrow/util/pfor/pfor.cc +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -34,14 +34,19 @@ #include "arrow/util/bit_stream_utils_internal.h" #include "arrow/util/bit_util.h" #include "arrow/util/bpacking_internal.h" +#include "arrow/util/endian.h" #include "arrow/util/logging.h" #include "arrow/util/macros.h" #include "arrow/util/span.h" +#include "arrow/util/ubsan.h" namespace arrow { namespace util { namespace pfor { +static_assert(ARROW_LITTLE_ENDIAN, + "PFOR serialization assumes little-endian byte order"); + // ---------------------------------------------------------------------- // FindOptimalBitWidth: histogram-based cost model @@ -182,12 +187,12 @@ Result PforCompression::DecodeVector(T* values, int32_t full_batches = num_elements / kBatchSize; int32_t remainder = num_elements % kBatchSize; - UnsignedT* unsigned_values = reinterpret_cast(values); + std::vector unsigned_values(num_elements); const auto unsigned_for = static_cast(info.frame_of_reference); // Unpack full batches using SIMD for (int32_t batch = 0; batch < full_batches; ++batch) { - arrow::internal::unpack(read_ptr, unsigned_values + batch * kBatchSize, + arrow::internal::unpack(read_ptr, unsigned_values.data() + batch * kBatchSize, kBatchSize, info.bit_width, batch * kBatchSize * info.bit_width); } @@ -208,9 +213,10 @@ Result PforCompression::DecodeVector(T* values, } } - // Add FOR to all values + // Add FOR and convert to signed output via SafeCopy for (int32_t i = 0; i < num_elements; ++i) { unsigned_values[i] += unsigned_for; + values[i] = util::SafeCopy(unsigned_values[i]); } int64_t packed_size = From 0c5ef3e446cfb7cdcb413edf2c11a5d8d427aed3 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Wed, 3 Jun 2026 18:04:20 +0000 Subject: [PATCH 08/31] Use single-call unpack() instead of manual batch loop + BitReader remainder --- cpp/src/arrow/util/pfor/pfor.cc | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc index 41d94c3aba9f..1a50a788e7f6 100644 --- a/cpp/src/arrow/util/pfor/pfor.cc +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -183,35 +183,13 @@ Result PforCompression::DecodeVector(T* values, // Step 3: Unpack bit-packed deltas and add FOR if (info.bit_width > 0) { - constexpr int kBatchSize = 32; - int32_t full_batches = num_elements / kBatchSize; - int32_t remainder = num_elements % kBatchSize; - std::vector unsigned_values(num_elements); const auto unsigned_for = static_cast(info.frame_of_reference); - // Unpack full batches using SIMD - for (int32_t batch = 0; batch < full_batches; ++batch) { - arrow::internal::unpack(read_ptr, unsigned_values.data() + batch * kBatchSize, - kBatchSize, info.bit_width, - batch * kBatchSize * info.bit_width); - } - - // Unpack remainder using BitReader - if (remainder > 0) { - int64_t packed_size = - bit_util::BytesForBits(static_cast(num_elements) * info.bit_width); - bit_util::BitReader reader(read_ptr, static_cast(packed_size)); - for (int32_t i = 0; i < full_batches * kBatchSize; ++i) { - uint64_t val; - reader.GetValue(info.bit_width, &val); - } - for (int32_t i = full_batches * kBatchSize; i < num_elements; ++i) { - uint64_t val; - reader.GetValue(info.bit_width, &val); - unsigned_values[i] = static_cast(val); - } - } + // Arrow's unpack handles arbitrary sizes: SIMD for complete batches, + // then unpack_exact for the remainder. + arrow::internal::unpack(read_ptr, unsigned_values.data(), + static_cast(num_elements), info.bit_width); // Add FOR and convert to signed output via SafeCopy for (int32_t i = 0; i < num_elements; ++i) { From a99dee87e6ab1a425adb5d8193bd0de28111d24e Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Wed, 3 Jun 2026 18:05:32 +0000 Subject: [PATCH 09/31] Add pragma GCC unroll/ivdep to decode loops for better vectorization --- cpp/src/arrow/util/pfor/pfor.cc | 4 ++++ cpp/src/arrow/util/pfor/pfor_constants.h | 3 +++ 2 files changed, 7 insertions(+) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc index 1a50a788e7f6..7028be94ffdf 100644 --- a/cpp/src/arrow/util/pfor/pfor.cc +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -192,6 +192,8 @@ Result PforCompression::DecodeVector(T* values, static_cast(num_elements), info.bit_width); // Add FOR and convert to signed output via SafeCopy +#pragma GCC unroll PforConstants::kLoopUnrolls +#pragma GCC ivdep for (int32_t i = 0; i < num_elements; ++i) { unsigned_values[i] += unsigned_for; values[i] = util::SafeCopy(unsigned_values[i]); @@ -213,6 +215,8 @@ Result PforCompression::DecodeVector(T* values, const uint8_t* values_ptr = read_ptr; read_ptr += info.num_exceptions * sizeof(T); +#pragma GCC unroll PforConstants::kLoopUnrolls +#pragma GCC ivdep for (int16_t i = 0; i < info.num_exceptions; ++i) { int16_t pos; std::memcpy(&pos, positions_ptr + i * sizeof(int16_t), sizeof(int16_t)); diff --git a/cpp/src/arrow/util/pfor/pfor_constants.h b/cpp/src/arrow/util/pfor/pfor_constants.h index 244330357c2f..18e63b962e4c 100644 --- a/cpp/src/arrow/util/pfor/pfor_constants.h +++ b/cpp/src/arrow/util/pfor/pfor_constants.h @@ -52,6 +52,9 @@ class PforConstants { /// Packing mode: FOR + bit-packing (currently the only mode). static constexpr uint8_t kPackingModeForBitPack = 0; + + /// Loop unroll factor for compiler hints in decode loops. + static constexpr int64_t kLoopUnrolls = 4; }; /// \brief Type traits for PFOR integer types From c1a8eb57b3146a564379aa9eafc01a8efd17d92b Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Wed, 3 Jun 2026 18:06:43 +0000 Subject: [PATCH 10/31] Add PforEncodedVectorView for zero-copy decode path --- cpp/src/arrow/util/pfor/pfor.cc | 44 +++++++++++++++++++++++++++++++++ cpp/src/arrow/util/pfor/pfor.h | 24 ++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc index 7028be94ffdf..e9e7372e7154 100644 --- a/cpp/src/arrow/util/pfor/pfor.cc +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -234,6 +234,50 @@ Result PforCompression::DecodeVector(T* values, // ---------------------------------------------------------------------- // Serialization helpers +// ---------------------------------------------------------------------- +// PforEncodedVectorView::LoadView + +template +Result> PforEncodedVectorView::LoadView( + arrow::util::span data, int32_t num_elements) { + ARROW_ASSIGN_OR_RAISE(auto info, PforVectorInfo::Load(data)); + + PforEncodedVectorView view; + view.info = info; + view.num_elements = num_elements; + + const uint8_t* ptr = data.data() + PforVectorInfo::kStoredSize; + + // packed_values: zero-copy span into the buffer + int64_t packed_size = 0; + if (info.bit_width > 0) { + packed_size = + bit_util::BytesForBits(static_cast(num_elements) * info.bit_width); + view.packed_values = arrow::util::span(ptr, packed_size); + ptr += packed_size; + } + + // Exception positions and values: copy into aligned storage + if (info.num_exceptions > 0) { + view.exception_positions.resize(info.num_exceptions); + std::memcpy(view.exception_positions.data(), ptr, + info.num_exceptions * sizeof(int16_t)); + ptr += info.num_exceptions * sizeof(int16_t); + + view.exception_values.resize(info.num_exceptions); + std::memcpy(view.exception_values.data(), ptr, + info.num_exceptions * sizeof(T)); + } + + return view; +} + +template struct PforEncodedVectorView; +template struct PforEncodedVectorView; + +// ---------------------------------------------------------------------- +// Serialization helpers + template int64_t PforCompression::SerializedVectorSize(const PforEncodedVector& vec, int32_t num_elements) { diff --git a/cpp/src/arrow/util/pfor/pfor.h b/cpp/src/arrow/util/pfor/pfor.h index 258bdbf74efd..d2160bac8cd6 100644 --- a/cpp/src/arrow/util/pfor/pfor.h +++ b/cpp/src/arrow/util/pfor/pfor.h @@ -89,6 +89,30 @@ struct PforEncodedVector { std::vector exception_values; }; +// ---------------------------------------------------------------------- +// Zero-copy encoded vector view + +/// \brief A zero-copy view over a serialized PFOR vector +/// +/// The packed_values span points directly into the compressed buffer. +/// Exception positions and values are copied into aligned storage. +template +struct PforEncodedVectorView { + PforVectorInfo info; + int32_t num_elements = 0; + arrow::util::span packed_values; + std::vector exception_positions; + std::vector exception_values; + + /// \brief Create a zero-copy view from a serialized vector buffer + /// + /// \param[in] data span over the serialized vector data + /// \param[in] num_elements number of elements in this vector + /// \return the view, or an error if the buffer is too small + static Result LoadView( + arrow::util::span data, int32_t num_elements); +}; + // ---------------------------------------------------------------------- // Cost model result From 49ef1cea62af7daae9b5646fe9692ba017db4196 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Wed, 3 Jun 2026 18:08:35 +0000 Subject: [PATCH 11/31] Make vector_size configurable on encode path with default kPforVectorSize --- cpp/src/arrow/util/pfor/pfor_wrapper.cc | 21 +++++++++++++++------ cpp/src/arrow/util/pfor/pfor_wrapper.h | 11 ++++++++++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.cc b/cpp/src/arrow/util/pfor/pfor_wrapper.cc index 8aefd32871e4..15ae79cb5610 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.cc +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.cc @@ -66,22 +66,26 @@ typename PforWrapper::PforHeader PforWrapper::LoadHeader( // Encode template -void PforWrapper::Encode(const T* values, int32_t num_values, char* comp, - int64_t* comp_size) { +void PforWrapper::Encode(const T* values, int32_t num_values, int32_t vector_size, + char* comp, int64_t* comp_size) { ARROW_DCHECK(num_values > 0); ARROW_DCHECK(comp != nullptr); ARROW_DCHECK(comp_size != nullptr); + ARROW_DCHECK((vector_size & (vector_size - 1)) == 0); - const int32_t vector_size = kVectorSize; const int32_t num_vectors = (num_values + vector_size - 1) / vector_size; + // Compute log2(vector_size) + uint8_t log_vector_size = 0; + for (int32_t v = vector_size; v > 1; v >>= 1) ++log_vector_size; + auto* dest = reinterpret_cast(comp); // Step 1: Write header PforHeader header; header.packing_mode = PforConstants::kPackingModeForBitPack; - header.log_vector_size = PforConstants::kDefaultLogVectorSize; + header.log_vector_size = log_vector_size; header.value_byte_width = sizeof(T); header.num_elements = num_values; StoreHeader(arrow::util::span(dest, PforConstants::kHeaderSize), header); @@ -118,6 +122,12 @@ void PforWrapper::Encode(const T* values, int32_t num_values, char* comp, *comp_size = static_cast(write_ptr - dest); } +template +void PforWrapper::Encode(const T* values, int32_t num_values, char* comp, + int64_t* comp_size) { + Encode(values, num_values, kVectorSize, comp, comp_size); +} + // ---------------------------------------------------------------------- // Decode @@ -181,8 +191,7 @@ Status PforWrapper::Decode(T* values, int32_t num_values, const char* comp, // GetMaxCompressedSize template -int64_t PforWrapper::GetMaxCompressedSize(int32_t num_values) { - const int32_t vector_size = kVectorSize; +int64_t PforWrapper::GetMaxCompressedSize(int32_t num_values, int32_t vector_size) { const int32_t num_vectors = (num_values + vector_size - 1) / vector_size; diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.h b/cpp/src/arrow/util/pfor/pfor_wrapper.h index 003a8b4693e6..5512af4f37b4 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.h +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.h @@ -45,8 +45,14 @@ class PforWrapper { /// /// \param[in] values pointer to input integers /// \param[in] num_values total number of values + /// \param[in] vector_size number of elements per vector (must be a power of 2, + /// in [2^kMinLogVectorSize, 2^kMaxLogVectorSize]) /// \param[out] comp pointer to output buffer (caller must ensure sufficient size) /// \param[in,out] comp_size input: available buffer size; output: bytes written + static void Encode(const T* values, int32_t num_values, int32_t vector_size, + char* comp, int64_t* comp_size); + + /// Convenience overload with default vector_size = kPforVectorSize static void Encode(const T* values, int32_t num_values, char* comp, int64_t* comp_size); @@ -63,8 +69,11 @@ class PforWrapper { /// \brief Get the maximum compressed size for a given number of values /// /// \param[in] num_values number of integer values + /// \param[in] vector_size number of elements per vector /// \return maximum possible compressed page size in bytes - static int64_t GetMaxCompressedSize(int32_t num_values); + static int64_t GetMaxCompressedSize( + int32_t num_values, + int32_t vector_size = static_cast(PforConstants::kPforVectorSize)); private: /// \brief Page header structure (7 bytes) From 6f3a5bf76ca06386c815bd133e9675ad0fc7cfef Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Wed, 3 Jun 2026 19:23:55 +0000 Subject: [PATCH 12/31] Fix pfor_test.cc: unwrap Result<> from PforVectorInfo::Load() Load() now returns Result after the Status/Result refactoring. Use ASSERT_OK_AND_ASSIGN to properly unwrap the result in tests. --- cpp/src/arrow/util/pfor/pfor_test.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor_test.cc b/cpp/src/arrow/util/pfor/pfor_test.cc index 5b9673445f0c..665a85ad714d 100644 --- a/cpp/src/arrow/util/pfor/pfor_test.cc +++ b/cpp/src/arrow/util/pfor/pfor_test.cc @@ -75,7 +75,8 @@ TEST(PforVectorInfoTest, Int32RoundTrip) { uint8_t buf[7]; info.Store(arrow::util::span(buf, 7)); - auto loaded = PforVectorInfo::Load(arrow::util::span(buf, 7)); + ASSERT_OK_AND_ASSIGN(auto loaded, + PforVectorInfo::Load(arrow::util::span(buf, 7))); EXPECT_EQ(loaded.frame_of_reference, -42); EXPECT_EQ(loaded.bit_width, 17); @@ -90,7 +91,8 @@ TEST(PforVectorInfoTest, Int64RoundTrip) { uint8_t buf[11]; info.Store(arrow::util::span(buf, 11)); - auto loaded = PforVectorInfo::Load(arrow::util::span(buf, 11)); + ASSERT_OK_AND_ASSIGN(auto loaded, + PforVectorInfo::Load(arrow::util::span(buf, 11))); EXPECT_EQ(loaded.frame_of_reference, -123456789012345LL); EXPECT_EQ(loaded.bit_width, 48); From cd2ed56cd4ba72cc01553ecf0139253c21899c00 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Sun, 14 Jun 2026 23:46:04 +0000 Subject: [PATCH 13/31] Convert PforWrapper::LoadHeader to return Result Make LoadHeader fallible: move the header-size check from Decode into LoadHeader, return Result, and update Decode to use ARROW_ASSIGN_OR_RAISE. Mirrors the corresponding ALP review fix on gh540-alp-pseudoDecimal-encoding. --- cpp/src/arrow/util/pfor/pfor_wrapper.cc | 15 ++++++++------- cpp/src/arrow/util/pfor/pfor_wrapper.h | 3 ++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.cc b/cpp/src/arrow/util/pfor/pfor_wrapper.cc index 15ae79cb5610..999897212822 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.cc +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.cc @@ -51,8 +51,12 @@ void PforWrapper::StoreHeader(arrow::util::span dest, } template -typename PforWrapper::PforHeader PforWrapper::LoadHeader( +Result::PforHeader> PforWrapper::LoadHeader( arrow::util::span src) { + if (src.size() < static_cast(PforConstants::kHeaderSize)) { + return Status::Invalid("PFOR compressed buffer too small for header: ", + src.size(), " < ", PforConstants::kHeaderSize); + } PforHeader header; const uint8_t* ptr = src.data(); header.packing_mode = ptr[0]; @@ -140,16 +144,13 @@ Status PforWrapper::Decode(T* values, int32_t num_values, const char* comp, if (comp == nullptr) { return Status::Invalid("PFOR compressed data pointer is null"); } - if (comp_size < PforConstants::kHeaderSize) { - return Status::Invalid("PFOR compressed buffer too small for header: ", comp_size, - " < ", PforConstants::kHeaderSize); - } const auto* src = reinterpret_cast(comp); // Step 1: Read header - PforHeader header = LoadHeader( - arrow::util::span(src, PforConstants::kHeaderSize)); + ARROW_ASSIGN_OR_RAISE( + PforHeader header, + LoadHeader(arrow::util::span(src, comp_size))); if (header.packing_mode != PforConstants::kPackingModeForBitPack) { return Status::Invalid("PFOR unsupported packing mode: ", header.packing_mode); diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.h b/cpp/src/arrow/util/pfor/pfor_wrapper.h index 5512af4f37b4..f0687cf72200 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.h +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.h @@ -24,6 +24,7 @@ #include #include +#include "arrow/result.h" #include "arrow/status.h" #include "arrow/util/pfor/pfor.h" #include "arrow/util/span.h" @@ -88,7 +89,7 @@ class PforWrapper { static_cast(PforConstants::kPforVectorSize); static void StoreHeader(arrow::util::span dest, const PforHeader& header); - static PforHeader LoadHeader(arrow::util::span src); + static Result LoadHeader(arrow::util::span src); }; } // namespace pfor From b6aae6301963fa95c16dee9dae6686943ba4b707 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Sun, 14 Jun 2026 23:48:45 +0000 Subject: [PATCH 14/31] Use SafeLoadAs/SafeStore for header and offset array in PforWrapper Replace std::memcpy / raw byte writes in PforWrapper::StoreHeader, LoadHeader, and the offset-array read/write paths with util::SafeLoadAs and util::SafeStore. Mirrors the corresponding ALP review fix on gh540-alp-pseudoDecimal-encoding. --- cpp/src/arrow/util/pfor/pfor_wrapper.cc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.cc b/cpp/src/arrow/util/pfor/pfor_wrapper.cc index 999897212822..0fb5f3a6a689 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.cc +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.cc @@ -32,6 +32,7 @@ #include "arrow/util/bit_util.h" #include "arrow/util/logging.h" #include "arrow/util/span.h" +#include "arrow/util/ubsan.h" namespace arrow { namespace util { @@ -44,10 +45,10 @@ template void PforWrapper::StoreHeader(arrow::util::span dest, const PforHeader& header) { uint8_t* ptr = dest.data(); - ptr[0] = header.packing_mode; - ptr[1] = header.log_vector_size; - ptr[2] = header.value_byte_width; - std::memcpy(ptr + 3, &header.num_elements, sizeof(int32_t)); + util::SafeStore(ptr + 0, header.packing_mode); + util::SafeStore(ptr + 1, header.log_vector_size); + util::SafeStore(ptr + 2, header.value_byte_width); + util::SafeStore(ptr + 3, header.num_elements); } template @@ -59,10 +60,10 @@ Result::PforHeader> PforWrapper::LoadHeader( } PforHeader header; const uint8_t* ptr = src.data(); - header.packing_mode = ptr[0]; - header.log_vector_size = ptr[1]; - header.value_byte_width = ptr[2]; - std::memcpy(&header.num_elements, ptr + 3, sizeof(int32_t)); + header.packing_mode = util::SafeLoadAs(ptr + 0); + header.log_vector_size = util::SafeLoadAs(ptr + 1); + header.value_byte_width = util::SafeLoadAs(ptr + 2); + header.num_elements = util::SafeLoadAs(ptr + 3); return header; } @@ -105,7 +106,7 @@ void PforWrapper::Encode(const T* values, int32_t num_values, int32_t vector_ for (int32_t v = 0; v < num_vectors; ++v) { // Record offset (from start of offset array) uint32_t offset = static_cast(write_ptr - data_start); - std::memcpy(offset_array_start + v * sizeof(uint32_t), &offset, sizeof(uint32_t)); + util::SafeStore(offset_array_start + v * sizeof(uint32_t), offset); // Determine elements in this vector int32_t start_idx = v * vector_size; @@ -169,9 +170,8 @@ Status PforWrapper::Decode(T* values, int32_t num_values, const char* comp, // Step 3: Decode each vector for (int32_t v = 0; v < num_vectors; ++v) { - uint32_t offset; - std::memcpy(&offset, offset_array_start + v * sizeof(uint32_t), - sizeof(uint32_t)); + uint32_t offset = + util::SafeLoadAs(offset_array_start + v * sizeof(uint32_t)); const uint8_t* vector_data = offset_array_start + offset; From 374eae6ad285faca118fab837c37c4a52a110f9c Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Sun, 14 Jun 2026 23:53:27 +0000 Subject: [PATCH 15/31] Validate header fields in PforWrapper::LoadHeader Reject invalid packing_mode, value_byte_width mismatch, log_vector_size out of [kMin, kMax] range, and negative num_elements when loading the PFOR page header. Removes the redundant packing_mode and value_byte_width checks from Decode now that they live in LoadHeader. Mirrors the corresponding ALP review fix on gh540-alp-pseudoDecimal-encoding. --- cpp/src/arrow/util/pfor/pfor_wrapper.cc | 26 +++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.cc b/cpp/src/arrow/util/pfor/pfor_wrapper.cc index 0fb5f3a6a689..89f4463ab617 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.cc +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.cc @@ -64,6 +64,24 @@ Result::PforHeader> PforWrapper::LoadHeader( header.log_vector_size = util::SafeLoadAs(ptr + 1); header.value_byte_width = util::SafeLoadAs(ptr + 2); header.num_elements = util::SafeLoadAs(ptr + 3); + + if (header.packing_mode != PforConstants::kPackingModeForBitPack) { + return Status::Invalid("PFOR unsupported packing mode: ", + static_cast(header.packing_mode)); + } + if (header.value_byte_width != sizeof(T)) { + return Status::Invalid("PFOR value_byte_width mismatch: ", + static_cast(header.value_byte_width), + " vs expected ", sizeof(T)); + } + if (header.log_vector_size < PforConstants::kMinLogVectorSize || + header.log_vector_size > PforConstants::kMaxLogVectorSize) { + return Status::Invalid("PFOR invalid log_vector_size: ", + static_cast(header.log_vector_size)); + } + if (header.num_elements < 0) { + return Status::Invalid("PFOR invalid num_elements: ", header.num_elements); + } return header; } @@ -153,14 +171,6 @@ Status PforWrapper::Decode(T* values, int32_t num_values, const char* comp, PforHeader header, LoadHeader(arrow::util::span(src, comp_size))); - if (header.packing_mode != PforConstants::kPackingModeForBitPack) { - return Status::Invalid("PFOR unsupported packing mode: ", header.packing_mode); - } - if (header.value_byte_width != sizeof(T)) { - return Status::Invalid("PFOR value_byte_width mismatch: ", header.value_byte_width, - " vs expected ", sizeof(T)); - } - const int32_t vector_size = 1 << header.log_vector_size; const int32_t num_vectors = (header.num_elements + vector_size - 1) / vector_size; From 87564b834eeb0ba3045134f669a08d5bbef267b0 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Mon, 15 Jun 2026 00:15:24 +0000 Subject: [PATCH 16/31] Use int64_t and global ::arrow:: prefix at PFOR encoder/decoder call sites Replace size_t with int64_t for max_size/comp_size to match the PforWrapper API signature, and qualify pfor::PforWrapper as ::arrow::util::pfor::PforWrapper to avoid ADL ambiguity. --- cpp/src/parquet/decoder.cc | 2 +- cpp/src/parquet/encoder.cc | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/src/parquet/decoder.cc b/cpp/src/parquet/decoder.cc index 2d4daafea87f..0612994cba7c 100644 --- a/cpp/src/parquet/decoder.cc +++ b/cpp/src/parquet/decoder.cc @@ -2399,7 +2399,7 @@ class PforDecoder : public TypedDecoderImpl { // Decode all values on first call, cache for subsequent calls if (decoded_values_.empty() && this->num_values_ > 0) { decoded_values_.resize(this->num_values_); - arrow::util::pfor::PforWrapper::Decode( + ::arrow::util::pfor::PforWrapper::Decode( decoded_values_.data(), static_cast(this->num_values_), reinterpret_cast(data_), static_cast(data_len_)); } diff --git a/cpp/src/parquet/encoder.cc b/cpp/src/parquet/encoder.cc index b5a4e25e4016..3e82713a69f6 100644 --- a/cpp/src/parquet/encoder.cc +++ b/cpp/src/parquet/encoder.cc @@ -1783,17 +1783,17 @@ class PforEncoder : public EncoderImpl, virtual public TypedEncoder { } const uint32_t num_values = static_cast(values_.size()); - size_t max_size = - arrow::util::pfor::PforWrapper::GetMaxCompressedSize(num_values); + int64_t max_size = + ::arrow::util::pfor::PforWrapper::GetMaxCompressedSize(num_values); PARQUET_ASSIGN_OR_THROW(auto buffer, ::arrow::AllocateResizableBuffer( - static_cast(max_size), pool_)); + max_size, pool_)); - size_t comp_size = max_size; - arrow::util::pfor::PforWrapper::Encode( + int64_t comp_size = max_size; + ::arrow::util::pfor::PforWrapper::Encode( values_.data(), num_values, reinterpret_cast(buffer->mutable_data()), &comp_size); - PARQUET_THROW_NOT_OK(buffer->Resize(static_cast(comp_size))); + PARQUET_THROW_NOT_OK(buffer->Resize(comp_size)); values_.clear(); return std::move(buffer); } From 4f305b9422673a8a0dd5fb8c299d8e92680f20bd Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Mon, 15 Jun 2026 00:20:01 +0000 Subject: [PATCH 17/31] Use uint8_t* consistently in PforWrapper API instead of char* Aligns with Arrow buffer conventions (Buffer::data() returns uint8_t*). Removes the reinterpret_cast at the parquet encoder/decoder call sites and switches std::vector compressed buffers to std::vector in the unit test and benchmark. Also fixes a pre-existing size_t / int64_t* mismatch in pfor_benchmark.cc that surfaced once the buffer pointer type was tightened. Mirrors the corresponding ALP review fix on gh540-alp-pseudoDecimal-encoding. --- cpp/src/arrow/util/pfor/pfor_benchmark.cc | 14 +++++++------- cpp/src/arrow/util/pfor/pfor_test.cc | 18 +++++++++--------- cpp/src/arrow/util/pfor/pfor_wrapper.cc | 10 +++++----- cpp/src/arrow/util/pfor/pfor_wrapper.h | 6 +++--- cpp/src/parquet/decoder.cc | 2 +- cpp/src/parquet/encoder.cc | 2 +- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor_benchmark.cc b/cpp/src/arrow/util/pfor/pfor_benchmark.cc index aa07025dd88f..134dd720f2a4 100644 --- a/cpp/src/arrow/util/pfor/pfor_benchmark.cc +++ b/cpp/src/arrow/util/pfor/pfor_benchmark.cc @@ -159,11 +159,11 @@ void BM_PforEncodeImpl(benchmark::State& state, const int64_t num_values = state.range(0); auto values = generator(num_values); - size_t max_size = PforWrapper::GetMaxCompressedSize(num_values); - std::vector compressed(max_size); + int64_t max_size = PforWrapper::GetMaxCompressedSize(num_values); + std::vector compressed(max_size); for (auto _ : state) { - size_t comp_size = max_size; + int64_t comp_size = max_size; PforWrapper::Encode(values.data(), num_values, compressed.data(), &comp_size); benchmark::DoNotOptimize(comp_size); @@ -175,7 +175,7 @@ void BM_PforEncodeImpl(benchmark::State& state, state.SetItemsProcessed(state.iterations() * num_values); // Report compression ratio - size_t comp_size = max_size; + int64_t comp_size = max_size; PforWrapper::Encode(values.data(), num_values, compressed.data(), &comp_size); state.counters["CompRatio%"] = benchmark::Counter( 100.0 * static_cast(comp_size) / @@ -188,9 +188,9 @@ void BM_PforDecodeImpl(benchmark::State& state, const int64_t num_values = state.range(0); auto values = generator(num_values); - size_t max_size = PforWrapper::GetMaxCompressedSize(num_values); - std::vector compressed(max_size); - size_t comp_size = max_size; + int64_t max_size = PforWrapper::GetMaxCompressedSize(num_values); + std::vector compressed(max_size); + int64_t comp_size = max_size; PforWrapper::Encode(values.data(), num_values, compressed.data(), &comp_size); std::vector decoded(num_values); diff --git a/cpp/src/arrow/util/pfor/pfor_test.cc b/cpp/src/arrow/util/pfor/pfor_test.cc index 665a85ad714d..70b0596109de 100644 --- a/cpp/src/arrow/util/pfor/pfor_test.cc +++ b/cpp/src/arrow/util/pfor/pfor_test.cc @@ -266,7 +266,7 @@ TEST(PforWrapperTest, Int32SmallPage) { std::vector values = {10, 20, 30, 40, 50}; int64_t max_size = PforWrapper::GetMaxCompressedSize(5); - std::vector compressed(max_size); + std::vector compressed(max_size); int64_t comp_size = max_size; PforWrapper::Encode(values.data(), 5, compressed.data(), &comp_size); @@ -283,7 +283,7 @@ TEST(PforWrapperTest, Int32ExactOneVector) { std::iota(values.begin(), values.end(), 0); int64_t max_size = PforWrapper::GetMaxCompressedSize(1024); - std::vector compressed(max_size); + std::vector compressed(max_size); int64_t comp_size = max_size; PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); @@ -303,7 +303,7 @@ TEST(PforWrapperTest, Int32MultipleVectors) { for (auto& v : values) v = dist(rng); int64_t max_size = PforWrapper::GetMaxCompressedSize(n); - std::vector compressed(max_size); + std::vector compressed(max_size); int64_t comp_size = max_size; PforWrapper::Encode(values.data(), n, compressed.data(), &comp_size); @@ -323,7 +323,7 @@ TEST(PforWrapperTest, Int32WithOutliers) { values[1023] = -123456; int64_t max_size = PforWrapper::GetMaxCompressedSize(1024); - std::vector compressed(max_size); + std::vector compressed(max_size); int64_t comp_size = max_size; PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); @@ -345,7 +345,7 @@ TEST(PforWrapperTest, Int64MultipleVectors) { values[1500] = -9999999999999LL; int64_t max_size = PforWrapper::GetMaxCompressedSize(n); - std::vector compressed(max_size); + std::vector compressed(max_size); int64_t comp_size = max_size; PforWrapper::Encode(values.data(), n, compressed.data(), &comp_size); @@ -360,7 +360,7 @@ TEST(PforWrapperTest, Int32SingleElement) { std::vector values = {42}; int64_t max_size = PforWrapper::GetMaxCompressedSize(1); - std::vector compressed(max_size); + std::vector compressed(max_size); int64_t comp_size = max_size; PforWrapper::Encode(values.data(), 1, compressed.data(), &comp_size); @@ -375,7 +375,7 @@ TEST(PforWrapperTest, Int32AllZeros) { std::vector values(1024, 0); int64_t max_size = PforWrapper::GetMaxCompressedSize(1024); - std::vector compressed(max_size); + std::vector compressed(max_size); int64_t comp_size = max_size; PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); @@ -399,7 +399,7 @@ TEST(PforWrapperTest, Int32LargeRandom) { for (auto& v : values) v = dist(rng); int64_t max_size = PforWrapper::GetMaxCompressedSize(n); - std::vector compressed(max_size); + std::vector compressed(max_size); int64_t comp_size = max_size; PforWrapper::Encode(values.data(), n, compressed.data(), &comp_size); @@ -422,7 +422,7 @@ TEST(PforCompressionRatioTest, ClusteredDataCompresses) { values[500] = 999999; // One outlier int64_t max_size = PforWrapper::GetMaxCompressedSize(1024); - std::vector compressed(max_size); + std::vector compressed(max_size); int64_t comp_size = max_size; PforWrapper::Encode(values.data(), 1024, compressed.data(), &comp_size); diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.cc b/cpp/src/arrow/util/pfor/pfor_wrapper.cc index 89f4463ab617..1f6fa3f4a8eb 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.cc +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.cc @@ -90,7 +90,7 @@ Result::PforHeader> PforWrapper::LoadHeader( template void PforWrapper::Encode(const T* values, int32_t num_values, int32_t vector_size, - char* comp, int64_t* comp_size) { + uint8_t* comp, int64_t* comp_size) { ARROW_DCHECK(num_values > 0); ARROW_DCHECK(comp != nullptr); ARROW_DCHECK(comp_size != nullptr); @@ -103,7 +103,7 @@ void PforWrapper::Encode(const T* values, int32_t num_values, int32_t vector_ uint8_t log_vector_size = 0; for (int32_t v = vector_size; v > 1; v >>= 1) ++log_vector_size; - auto* dest = reinterpret_cast(comp); + uint8_t* dest = comp; // Step 1: Write header PforHeader header; @@ -146,7 +146,7 @@ void PforWrapper::Encode(const T* values, int32_t num_values, int32_t vector_ } template -void PforWrapper::Encode(const T* values, int32_t num_values, char* comp, +void PforWrapper::Encode(const T* values, int32_t num_values, uint8_t* comp, int64_t* comp_size) { Encode(values, num_values, kVectorSize, comp, comp_size); } @@ -155,7 +155,7 @@ void PforWrapper::Encode(const T* values, int32_t num_values, char* comp, // Decode template -Status PforWrapper::Decode(T* values, int32_t num_values, const char* comp, +Status PforWrapper::Decode(T* values, int32_t num_values, const uint8_t* comp, int64_t comp_size) { if (num_values <= 0) { return Status::Invalid("PFOR num_values must be positive: ", num_values); @@ -164,7 +164,7 @@ Status PforWrapper::Decode(T* values, int32_t num_values, const char* comp, return Status::Invalid("PFOR compressed data pointer is null"); } - const auto* src = reinterpret_cast(comp); + const uint8_t* src = comp; // Step 1: Read header ARROW_ASSIGN_OR_RAISE( diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.h b/cpp/src/arrow/util/pfor/pfor_wrapper.h index f0687cf72200..727b942109a3 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.h +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.h @@ -51,10 +51,10 @@ class PforWrapper { /// \param[out] comp pointer to output buffer (caller must ensure sufficient size) /// \param[in,out] comp_size input: available buffer size; output: bytes written static void Encode(const T* values, int32_t num_values, int32_t vector_size, - char* comp, int64_t* comp_size); + uint8_t* comp, int64_t* comp_size); /// Convenience overload with default vector_size = kPforVectorSize - static void Encode(const T* values, int32_t num_values, char* comp, + static void Encode(const T* values, int32_t num_values, uint8_t* comp, int64_t* comp_size); /// \brief Decode a PFOR-compressed page @@ -64,7 +64,7 @@ class PforWrapper { /// \param[in] comp pointer to compressed data /// \param[in] comp_size size of compressed data /// \return Status::OK on success, or an error if the data is malformed - static Status Decode(T* values, int32_t num_values, const char* comp, + static Status Decode(T* values, int32_t num_values, const uint8_t* comp, int64_t comp_size); /// \brief Get the maximum compressed size for a given number of values diff --git a/cpp/src/parquet/decoder.cc b/cpp/src/parquet/decoder.cc index 0612994cba7c..be13a53ac15e 100644 --- a/cpp/src/parquet/decoder.cc +++ b/cpp/src/parquet/decoder.cc @@ -2401,7 +2401,7 @@ class PforDecoder : public TypedDecoderImpl { decoded_values_.resize(this->num_values_); ::arrow::util::pfor::PforWrapper::Decode( decoded_values_.data(), static_cast(this->num_values_), - reinterpret_cast(data_), static_cast(data_len_)); + data_, static_cast(data_len_)); } std::memcpy(buffer, decoded_values_.data() + values_decoded_, diff --git a/cpp/src/parquet/encoder.cc b/cpp/src/parquet/encoder.cc index 3e82713a69f6..cbfd4a3b43dc 100644 --- a/cpp/src/parquet/encoder.cc +++ b/cpp/src/parquet/encoder.cc @@ -1791,7 +1791,7 @@ class PforEncoder : public EncoderImpl, virtual public TypedEncoder { int64_t comp_size = max_size; ::arrow::util::pfor::PforWrapper::Encode( values_.data(), num_values, - reinterpret_cast(buffer->mutable_data()), &comp_size); + buffer->mutable_data(), &comp_size); PARQUET_THROW_NOT_OK(buffer->Resize(comp_size)); values_.clear(); From 9d5e82c13b843b90349fa2bb17ce82fcb322c750 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Mon, 15 Jun 2026 00:27:56 +0000 Subject: [PATCH 18/31] Convert PforVectorInfo to class with SafeLoadAs/SafeStore and bit_width validation Per Google C++ style, replace the PforVectorInfo struct with a class that has private trailing-underscore members and getter/setter accessors. Replace std::memcpy calls in Store/Load and the exception patch loop in DecodeVector with util::SafeLoadAs / util::SafeStore. Add bit_width range validation inside Load() so callers don't have to repeat the check. Updates all access sites in pfor.cc and pfor_test.cc to go through the new accessors. Caches num_exceptions() in a local in DecodeVector so the #pragma GCC unroll can still see a constant loop bound. Mirrors the corresponding ALP review fix on gh540-alp-pseudoDecimal-encoding. --- cpp/src/arrow/util/pfor/pfor.cc | 81 +++++++++++++--------------- cpp/src/arrow/util/pfor/pfor.h | 43 +++++++++++---- cpp/src/arrow/util/pfor/pfor_test.cc | 40 +++++++------- 3 files changed, 91 insertions(+), 73 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc index e9e7372e7154..9facbcc704a5 100644 --- a/cpp/src/arrow/util/pfor/pfor.cc +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -119,9 +119,9 @@ PforEncodedVector PforCompression::EncodeVector(const T* values, // Step 4: Collect exceptions and replace with placeholder (0) PforEncodedVector result; - result.info.frame_of_reference = min_val; - result.info.bit_width = bit_width; - result.info.num_exceptions = num_exceptions; + result.info.set_frame_of_reference(min_val); + result.info.set_bit_width(bit_width); + result.info.set_num_exceptions(num_exceptions); if (num_exceptions > 0) { result.exception_positions.reserve(num_exceptions); @@ -168,28 +168,25 @@ Result PforCompression::DecodeVector(T* values, ARROW_ASSIGN_OR_RAISE(auto info, PforVectorInfo::Load(data)); const uint8_t* read_ptr = data.data() + PforVectorInfo::kStoredSize; - if (info.bit_width > PforTypeTraits::kMaxBitWidth) { - return Status::Invalid("PFOR bit_width out of range: ", info.bit_width); - } - if (info.num_exceptions < 0) { - return Status::Invalid("PFOR num_exceptions negative: ", info.num_exceptions); + if (info.num_exceptions() < 0) { + return Status::Invalid("PFOR num_exceptions negative: ", info.num_exceptions()); } // Step 2: Handle constant data (bit_width == 0, no exceptions) - if (info.bit_width == 0 && info.num_exceptions == 0) { - std::fill(values, values + num_elements, info.frame_of_reference); + if (info.bit_width() == 0 && info.num_exceptions() == 0) { + std::fill(values, values + num_elements, info.frame_of_reference()); return PforVectorInfo::kStoredSize; } // Step 3: Unpack bit-packed deltas and add FOR - if (info.bit_width > 0) { + if (info.bit_width() > 0) { std::vector unsigned_values(num_elements); - const auto unsigned_for = static_cast(info.frame_of_reference); + const auto unsigned_for = static_cast(info.frame_of_reference()); // Arrow's unpack handles arbitrary sizes: SIMD for complete batches, // then unpack_exact for the remainder. arrow::internal::unpack(read_ptr, unsigned_values.data(), - static_cast(num_elements), info.bit_width); + static_cast(num_elements), info.bit_width()); // Add FOR and convert to signed output via SafeCopy #pragma GCC unroll PforConstants::kLoopUnrolls @@ -200,30 +197,27 @@ Result PforCompression::DecodeVector(T* values, } int64_t packed_size = - bit_util::BytesForBits(static_cast(num_elements) * info.bit_width); + bit_util::BytesForBits(static_cast(num_elements) * info.bit_width()); read_ptr += packed_size; } else { // bit_width == 0 but has exceptions - fill with FOR - std::fill(values, values + num_elements, info.frame_of_reference); + std::fill(values, values + num_elements, info.frame_of_reference()); } // Step 4: Patch exceptions (stored as original values) - if (info.num_exceptions > 0) { + const int16_t num_exceptions = info.num_exceptions(); + if (num_exceptions > 0) { const uint8_t* positions_ptr = read_ptr; - read_ptr += info.num_exceptions * sizeof(int16_t); + read_ptr += num_exceptions * sizeof(int16_t); const uint8_t* values_ptr = read_ptr; - read_ptr += info.num_exceptions * sizeof(T); + read_ptr += num_exceptions * sizeof(T); #pragma GCC unroll PforConstants::kLoopUnrolls #pragma GCC ivdep - for (int16_t i = 0; i < info.num_exceptions; ++i) { - int16_t pos; - std::memcpy(&pos, positions_ptr + i * sizeof(int16_t), sizeof(int16_t)); - - T value; - std::memcpy(&value, values_ptr + i * sizeof(T), sizeof(T)); - + for (int16_t i = 0; i < num_exceptions; ++i) { + int16_t pos = util::SafeLoadAs(positions_ptr + i * sizeof(int16_t)); + T value = util::SafeLoadAs(values_ptr + i * sizeof(T)); values[pos] = value; } } @@ -250,23 +244,23 @@ Result> PforEncodedVectorView::LoadView( // packed_values: zero-copy span into the buffer int64_t packed_size = 0; - if (info.bit_width > 0) { + if (info.bit_width() > 0) { packed_size = - bit_util::BytesForBits(static_cast(num_elements) * info.bit_width); + bit_util::BytesForBits(static_cast(num_elements) * info.bit_width()); view.packed_values = arrow::util::span(ptr, packed_size); ptr += packed_size; } // Exception positions and values: copy into aligned storage - if (info.num_exceptions > 0) { - view.exception_positions.resize(info.num_exceptions); + if (info.num_exceptions() > 0) { + view.exception_positions.resize(info.num_exceptions()); std::memcpy(view.exception_positions.data(), ptr, - info.num_exceptions * sizeof(int16_t)); - ptr += info.num_exceptions * sizeof(int16_t); + info.num_exceptions() * sizeof(int16_t)); + ptr += info.num_exceptions() * sizeof(int16_t); - view.exception_values.resize(info.num_exceptions); + view.exception_values.resize(info.num_exceptions()); std::memcpy(view.exception_values.data(), ptr, - info.num_exceptions * sizeof(T)); + info.num_exceptions() * sizeof(T)); } return view; @@ -282,11 +276,12 @@ template int64_t PforCompression::SerializedVectorSize(const PforEncodedVector& vec, int32_t num_elements) { int64_t size = PforVectorInfo::kStoredSize; - if (vec.info.bit_width > 0) { - size += bit_util::BytesForBits(static_cast(num_elements) * vec.info.bit_width); + if (vec.info.bit_width() > 0) { + size += bit_util::BytesForBits( + static_cast(num_elements) * vec.info.bit_width()); } - size += vec.info.num_exceptions * static_cast(sizeof(int16_t)); - size += vec.info.num_exceptions * static_cast(sizeof(T)); + size += vec.info.num_exceptions() * static_cast(sizeof(int16_t)); + size += vec.info.num_exceptions() * static_cast(sizeof(T)); return size; } @@ -301,21 +296,21 @@ int64_t PforCompression::SerializeVector(const PforEncodedVector& vec, write_ptr += PforVectorInfo::kStoredSize; // Write packed values - if (vec.info.bit_width > 0 && !vec.packed_values.empty()) { + if (vec.info.bit_width() > 0 && !vec.packed_values.empty()) { std::memcpy(write_ptr, vec.packed_values.data(), vec.packed_values.size()); write_ptr += vec.packed_values.size(); } // Write exception positions - if (vec.info.num_exceptions > 0) { + if (vec.info.num_exceptions() > 0) { std::memcpy(write_ptr, vec.exception_positions.data(), - vec.info.num_exceptions * sizeof(int16_t)); - write_ptr += vec.info.num_exceptions * sizeof(int16_t); + vec.info.num_exceptions() * sizeof(int16_t)); + write_ptr += vec.info.num_exceptions() * sizeof(int16_t); // Write exception values (original integers) std::memcpy(write_ptr, vec.exception_values.data(), - vec.info.num_exceptions * sizeof(T)); - write_ptr += vec.info.num_exceptions * sizeof(T); + vec.info.num_exceptions() * sizeof(T)); + write_ptr += vec.info.num_exceptions() * sizeof(T); } return static_cast(write_ptr - dest.data()); diff --git a/cpp/src/arrow/util/pfor/pfor.h b/cpp/src/arrow/util/pfor/pfor.h index d2160bac8cd6..8db373e00ef6 100644 --- a/cpp/src/arrow/util/pfor/pfor.h +++ b/cpp/src/arrow/util/pfor/pfor.h @@ -33,6 +33,7 @@ #include "arrow/status.h" #include "arrow/util/pfor/pfor_constants.h" #include "arrow/util/span.h" +#include "arrow/util/ubsan.h" namespace arrow { namespace util { @@ -46,17 +47,30 @@ namespace pfor { /// For INT32 (7 bytes): [frame_of_reference(4B)] [bit_width(1B)] [num_exceptions(2B)] /// For INT64 (11 bytes): [frame_of_reference(8B)] [bit_width(1B)] [num_exceptions(2B)] template -struct PforVectorInfo { - T frame_of_reference = 0; - uint8_t bit_width = 0; - int16_t num_exceptions = 0; +class PforVectorInfo { + public: + PforVectorInfo() = default; + PforVectorInfo(T frame_of_reference, uint8_t bit_width, int16_t num_exceptions) + : frame_of_reference_(frame_of_reference), + bit_width_(bit_width), + num_exceptions_(num_exceptions) {} + + T frame_of_reference() const { return frame_of_reference_; } + uint8_t bit_width() const { return bit_width_; } + int16_t num_exceptions() const { return num_exceptions_; } + + void set_frame_of_reference(T frame_of_reference) { + frame_of_reference_ = frame_of_reference; + } + void set_bit_width(uint8_t bit_width) { bit_width_ = bit_width; } + void set_num_exceptions(int16_t num_exceptions) { num_exceptions_ = num_exceptions; } /// \brief Store this info to a byte buffer (little-endian) void Store(arrow::util::span dest) const { uint8_t* ptr = dest.data(); - std::memcpy(ptr, &frame_of_reference, sizeof(T)); - ptr[sizeof(T)] = bit_width; - std::memcpy(ptr + sizeof(T) + 1, &num_exceptions, sizeof(int16_t)); + util::SafeStore(ptr, frame_of_reference_); + ptr[sizeof(T)] = bit_width_; + util::SafeStore(ptr + sizeof(T) + 1, num_exceptions_); } /// \brief Load this info from a byte buffer (little-endian) @@ -67,14 +81,23 @@ struct PforVectorInfo { } PforVectorInfo info; const uint8_t* ptr = src.data(); - std::memcpy(&info.frame_of_reference, ptr, sizeof(T)); - info.bit_width = ptr[sizeof(T)]; - std::memcpy(&info.num_exceptions, ptr + sizeof(T) + 1, sizeof(int16_t)); + info.frame_of_reference_ = util::SafeLoadAs(ptr); + info.bit_width_ = ptr[sizeof(T)]; + info.num_exceptions_ = util::SafeLoadAs(ptr + sizeof(T) + 1); + if (info.bit_width_ > PforTypeTraits::kMaxBitWidth) { + return Status::Invalid("PFOR bit_width out of range: ", + static_cast(info.bit_width_)); + } return info; } /// \brief Serialized size in bytes static constexpr int64_t kStoredSize = PforTypeTraits::kVectorInfoSize; + + private: + T frame_of_reference_ = 0; + uint8_t bit_width_ = 0; + int16_t num_exceptions_ = 0; }; // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/util/pfor/pfor_test.cc b/cpp/src/arrow/util/pfor/pfor_test.cc index 70b0596109de..1bb147cb7660 100644 --- a/cpp/src/arrow/util/pfor/pfor_test.cc +++ b/cpp/src/arrow/util/pfor/pfor_test.cc @@ -69,34 +69,34 @@ TEST(PforBitsRequiredTest, Int64) { TEST(PforVectorInfoTest, Int32RoundTrip) { PforVectorInfo info; - info.frame_of_reference = -42; - info.bit_width = 17; - info.num_exceptions = 300; + info.set_frame_of_reference(-42); + info.set_bit_width(17); + info.set_num_exceptions(300); uint8_t buf[7]; info.Store(arrow::util::span(buf, 7)); ASSERT_OK_AND_ASSIGN(auto loaded, PforVectorInfo::Load(arrow::util::span(buf, 7))); - EXPECT_EQ(loaded.frame_of_reference, -42); - EXPECT_EQ(loaded.bit_width, 17); - EXPECT_EQ(loaded.num_exceptions, 300); + EXPECT_EQ(loaded.frame_of_reference(), -42); + EXPECT_EQ(loaded.bit_width(), 17); + EXPECT_EQ(loaded.num_exceptions(), 300); } TEST(PforVectorInfoTest, Int64RoundTrip) { PforVectorInfo info; - info.frame_of_reference = -123456789012345LL; - info.bit_width = 48; - info.num_exceptions = 30000; + info.set_frame_of_reference(-123456789012345LL); + info.set_bit_width(48); + info.set_num_exceptions(30000); uint8_t buf[11]; info.Store(arrow::util::span(buf, 11)); ASSERT_OK_AND_ASSIGN(auto loaded, PforVectorInfo::Load(arrow::util::span(buf, 11))); - EXPECT_EQ(loaded.frame_of_reference, -123456789012345LL); - EXPECT_EQ(loaded.bit_width, 48); - EXPECT_EQ(loaded.num_exceptions, 30000); + EXPECT_EQ(loaded.frame_of_reference(), -123456789012345LL); + EXPECT_EQ(loaded.bit_width(), 48); + EXPECT_EQ(loaded.num_exceptions(), 30000); } // ====================================================================== @@ -139,8 +139,8 @@ TEST(PforVectorTest, Int32SimpleSequence) { std::iota(values.begin(), values.end(), 100); auto encoded = PforCompression::EncodeVector(values.data(), 64); - EXPECT_EQ(encoded.info.frame_of_reference, 100); - EXPECT_EQ(encoded.info.num_exceptions, 0); + EXPECT_EQ(encoded.info.frame_of_reference(), 100); + EXPECT_EQ(encoded.info.num_exceptions(), 0); // Serialize then decode size_t serialized_size = @@ -158,8 +158,8 @@ TEST(PforVectorTest, Int32WithOutlier) { std::vector values = {100, 102, 101, 103, 100, 99, 50000, 104}; auto encoded = PforCompression::EncodeVector(values.data(), 8); - EXPECT_EQ(encoded.info.frame_of_reference, 99); - EXPECT_GT(encoded.info.num_exceptions, 0); + EXPECT_EQ(encoded.info.frame_of_reference(), 99); + EXPECT_GT(encoded.info.num_exceptions(), 0); size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 8); @@ -176,8 +176,8 @@ TEST(PforVectorTest, Int32AllIdentical) { std::vector values(100, 42); auto encoded = PforCompression::EncodeVector(values.data(), 100); - EXPECT_EQ(encoded.info.bit_width, 0); - EXPECT_EQ(encoded.info.num_exceptions, 0); + EXPECT_EQ(encoded.info.bit_width(), 0); + EXPECT_EQ(encoded.info.num_exceptions(), 0); size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 100); @@ -194,7 +194,7 @@ TEST(PforVectorTest, Int32NegativeValues) { std::vector values = {-100, -50, -200, -1, -150}; auto encoded = PforCompression::EncodeVector(values.data(), 5); - EXPECT_EQ(encoded.info.frame_of_reference, -200); + EXPECT_EQ(encoded.info.frame_of_reference(), -200); size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 5); @@ -246,7 +246,7 @@ TEST(PforVectorTest, Int64WithOutlier) { values[42] = 999999999999LL; // Outlier auto encoded = PforCompression::EncodeVector(values.data(), 100); - EXPECT_GT(encoded.info.num_exceptions, 0); + EXPECT_GT(encoded.info.num_exceptions(), 0); size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 100); From 55834248d85fb6a48b9437b0962eb6763fc9ca44 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Mon, 15 Jun 2026 00:30:59 +0000 Subject: [PATCH 19/31] Convert PforEncodedVector and PforEncodedVectorView to classes Per Google C++ style, both types become classes with private trailing-underscore members and const getters, mutable getters, and setters. Updates all access sites in pfor.cc (EncodeVector, LoadView, SerializedVectorSize, SerializeVector) and pfor_test.cc to go through the new accessors. Mirrors the corresponding ALP review fix on gh540-alp-pseudoDecimal-encoding. --- cpp/src/arrow/util/pfor/pfor.cc | 66 +++++++++++++-------------- cpp/src/arrow/util/pfor/pfor.h | 68 +++++++++++++++++++++++----- cpp/src/arrow/util/pfor/pfor_test.cc | 16 +++---- 3 files changed, 98 insertions(+), 52 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc index 9facbcc704a5..04d86f467c3e 100644 --- a/cpp/src/arrow/util/pfor/pfor.cc +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -119,13 +119,13 @@ PforEncodedVector PforCompression::EncodeVector(const T* values, // Step 4: Collect exceptions and replace with placeholder (0) PforEncodedVector result; - result.info.set_frame_of_reference(min_val); - result.info.set_bit_width(bit_width); - result.info.set_num_exceptions(num_exceptions); + result.mutable_info().set_frame_of_reference(min_val); + result.mutable_info().set_bit_width(bit_width); + result.mutable_info().set_num_exceptions(num_exceptions); if (num_exceptions > 0) { - result.exception_positions.reserve(num_exceptions); - result.exception_values.reserve(num_exceptions); + result.mutable_exception_positions().reserve(num_exceptions); + result.mutable_exception_values().reserve(num_exceptions); UnsignedT mask = (bit_width >= PforTypeTraits::kMaxBitWidth) ? static_cast(-1) @@ -133,8 +133,8 @@ PforEncodedVector PforCompression::EncodeVector(const T* values, for (int32_t i = 0; i < num_elements; ++i) { if (deltas[i] > mask) { - result.exception_positions.push_back(static_cast(i)); - result.exception_values.push_back(values[i]); + result.mutable_exception_positions().push_back(static_cast(i)); + result.mutable_exception_values().push_back(values[i]); deltas[i] = 0; } } @@ -144,9 +144,9 @@ PforEncodedVector PforCompression::EncodeVector(const T* values, if (bit_width > 0) { int64_t packed_size = bit_util::BytesForBits(static_cast(num_elements) * bit_width); - result.packed_values.resize(static_cast(packed_size), 0); + result.mutable_packed_values().resize(static_cast(packed_size), 0); - bit_util::BitWriter writer(result.packed_values.data(), + bit_util::BitWriter writer(result.mutable_packed_values().data(), static_cast(packed_size)); for (int32_t i = 0; i < num_elements; ++i) { writer.PutValue(static_cast(deltas[i]), bit_width); @@ -237,8 +237,8 @@ Result> PforEncodedVectorView::LoadView( ARROW_ASSIGN_OR_RAISE(auto info, PforVectorInfo::Load(data)); PforEncodedVectorView view; - view.info = info; - view.num_elements = num_elements; + view.set_info(info); + view.set_num_elements(num_elements); const uint8_t* ptr = data.data() + PforVectorInfo::kStoredSize; @@ -247,27 +247,27 @@ Result> PforEncodedVectorView::LoadView( if (info.bit_width() > 0) { packed_size = bit_util::BytesForBits(static_cast(num_elements) * info.bit_width()); - view.packed_values = arrow::util::span(ptr, packed_size); + view.set_packed_values(arrow::util::span(ptr, packed_size)); ptr += packed_size; } // Exception positions and values: copy into aligned storage if (info.num_exceptions() > 0) { - view.exception_positions.resize(info.num_exceptions()); - std::memcpy(view.exception_positions.data(), ptr, + view.mutable_exception_positions().resize(info.num_exceptions()); + std::memcpy(view.mutable_exception_positions().data(), ptr, info.num_exceptions() * sizeof(int16_t)); ptr += info.num_exceptions() * sizeof(int16_t); - view.exception_values.resize(info.num_exceptions()); - std::memcpy(view.exception_values.data(), ptr, + view.mutable_exception_values().resize(info.num_exceptions()); + std::memcpy(view.mutable_exception_values().data(), ptr, info.num_exceptions() * sizeof(T)); } return view; } -template struct PforEncodedVectorView; -template struct PforEncodedVectorView; +template class PforEncodedVectorView; +template class PforEncodedVectorView; // ---------------------------------------------------------------------- // Serialization helpers @@ -276,12 +276,12 @@ template int64_t PforCompression::SerializedVectorSize(const PforEncodedVector& vec, int32_t num_elements) { int64_t size = PforVectorInfo::kStoredSize; - if (vec.info.bit_width() > 0) { + if (vec.info().bit_width() > 0) { size += bit_util::BytesForBits( - static_cast(num_elements) * vec.info.bit_width()); + static_cast(num_elements) * vec.info().bit_width()); } - size += vec.info.num_exceptions() * static_cast(sizeof(int16_t)); - size += vec.info.num_exceptions() * static_cast(sizeof(T)); + size += vec.info().num_exceptions() * static_cast(sizeof(int16_t)); + size += vec.info().num_exceptions() * static_cast(sizeof(T)); return size; } @@ -292,25 +292,25 @@ int64_t PforCompression::SerializeVector(const PforEncodedVector& vec, uint8_t* write_ptr = dest.data(); // Write vector info - vec.info.Store(arrow::util::span(write_ptr, PforVectorInfo::kStoredSize)); + vec.info().Store(arrow::util::span(write_ptr, PforVectorInfo::kStoredSize)); write_ptr += PforVectorInfo::kStoredSize; // Write packed values - if (vec.info.bit_width() > 0 && !vec.packed_values.empty()) { - std::memcpy(write_ptr, vec.packed_values.data(), vec.packed_values.size()); - write_ptr += vec.packed_values.size(); + if (vec.info().bit_width() > 0 && !vec.packed_values().empty()) { + std::memcpy(write_ptr, vec.packed_values().data(), vec.packed_values().size()); + write_ptr += vec.packed_values().size(); } // Write exception positions - if (vec.info.num_exceptions() > 0) { - std::memcpy(write_ptr, vec.exception_positions.data(), - vec.info.num_exceptions() * sizeof(int16_t)); - write_ptr += vec.info.num_exceptions() * sizeof(int16_t); + if (vec.info().num_exceptions() > 0) { + std::memcpy(write_ptr, vec.exception_positions().data(), + vec.info().num_exceptions() * sizeof(int16_t)); + write_ptr += vec.info().num_exceptions() * sizeof(int16_t); // Write exception values (original integers) - std::memcpy(write_ptr, vec.exception_values.data(), - vec.info.num_exceptions() * sizeof(T)); - write_ptr += vec.info.num_exceptions() * sizeof(T); + std::memcpy(write_ptr, vec.exception_values().data(), + vec.info().num_exceptions() * sizeof(T)); + write_ptr += vec.info().num_exceptions() * sizeof(T); } return static_cast(write_ptr - dest.data()); diff --git a/cpp/src/arrow/util/pfor/pfor.h b/cpp/src/arrow/util/pfor/pfor.h index 8db373e00ef6..823878159d2c 100644 --- a/cpp/src/arrow/util/pfor/pfor.h +++ b/cpp/src/arrow/util/pfor/pfor.h @@ -105,11 +105,33 @@ class PforVectorInfo { /// \brief A PFOR-encoded vector with all its data sections template -struct PforEncodedVector { - PforVectorInfo info; - std::vector packed_values; - std::vector exception_positions; - std::vector exception_values; +class PforEncodedVector { + public: + PforEncodedVector() = default; + + const PforVectorInfo& info() const { return info_; } + PforVectorInfo& mutable_info() { return info_; } + void set_info(const PforVectorInfo& info) { info_ = info; } + + const std::vector& packed_values() const { return packed_values_; } + std::vector& mutable_packed_values() { return packed_values_; } + void set_packed_values(std::vector v) { packed_values_ = std::move(v); } + + const std::vector& exception_positions() const { return exception_positions_; } + std::vector& mutable_exception_positions() { return exception_positions_; } + void set_exception_positions(std::vector v) { + exception_positions_ = std::move(v); + } + + const std::vector& exception_values() const { return exception_values_; } + std::vector& mutable_exception_values() { return exception_values_; } + void set_exception_values(std::vector v) { exception_values_ = std::move(v); } + + private: + PforVectorInfo info_; + std::vector packed_values_; + std::vector exception_positions_; + std::vector exception_values_; }; // ---------------------------------------------------------------------- @@ -120,12 +142,29 @@ struct PforEncodedVector { /// The packed_values span points directly into the compressed buffer. /// Exception positions and values are copied into aligned storage. template -struct PforEncodedVectorView { - PforVectorInfo info; - int32_t num_elements = 0; - arrow::util::span packed_values; - std::vector exception_positions; - std::vector exception_values; +class PforEncodedVectorView { + public: + PforEncodedVectorView() = default; + + const PforVectorInfo& info() const { return info_; } + PforVectorInfo& mutable_info() { return info_; } + void set_info(const PforVectorInfo& info) { info_ = info; } + + int32_t num_elements() const { return num_elements_; } + void set_num_elements(int32_t n) { num_elements_ = n; } + + arrow::util::span packed_values() const { return packed_values_; } + void set_packed_values(arrow::util::span v) { packed_values_ = v; } + + const std::vector& exception_positions() const { return exception_positions_; } + std::vector& mutable_exception_positions() { return exception_positions_; } + void set_exception_positions(std::vector v) { + exception_positions_ = std::move(v); + } + + const std::vector& exception_values() const { return exception_values_; } + std::vector& mutable_exception_values() { return exception_values_; } + void set_exception_values(std::vector v) { exception_values_ = std::move(v); } /// \brief Create a zero-copy view from a serialized vector buffer /// @@ -134,6 +173,13 @@ struct PforEncodedVectorView { /// \return the view, or an error if the buffer is too small static Result LoadView( arrow::util::span data, int32_t num_elements); + + private: + PforVectorInfo info_; + int32_t num_elements_ = 0; + arrow::util::span packed_values_; + std::vector exception_positions_; + std::vector exception_values_; }; // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/util/pfor/pfor_test.cc b/cpp/src/arrow/util/pfor/pfor_test.cc index 1bb147cb7660..cbfe23c354c9 100644 --- a/cpp/src/arrow/util/pfor/pfor_test.cc +++ b/cpp/src/arrow/util/pfor/pfor_test.cc @@ -139,8 +139,8 @@ TEST(PforVectorTest, Int32SimpleSequence) { std::iota(values.begin(), values.end(), 100); auto encoded = PforCompression::EncodeVector(values.data(), 64); - EXPECT_EQ(encoded.info.frame_of_reference(), 100); - EXPECT_EQ(encoded.info.num_exceptions(), 0); + EXPECT_EQ(encoded.info().frame_of_reference(), 100); + EXPECT_EQ(encoded.info().num_exceptions(), 0); // Serialize then decode size_t serialized_size = @@ -158,8 +158,8 @@ TEST(PforVectorTest, Int32WithOutlier) { std::vector values = {100, 102, 101, 103, 100, 99, 50000, 104}; auto encoded = PforCompression::EncodeVector(values.data(), 8); - EXPECT_EQ(encoded.info.frame_of_reference(), 99); - EXPECT_GT(encoded.info.num_exceptions(), 0); + EXPECT_EQ(encoded.info().frame_of_reference(), 99); + EXPECT_GT(encoded.info().num_exceptions(), 0); size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 8); @@ -176,8 +176,8 @@ TEST(PforVectorTest, Int32AllIdentical) { std::vector values(100, 42); auto encoded = PforCompression::EncodeVector(values.data(), 100); - EXPECT_EQ(encoded.info.bit_width(), 0); - EXPECT_EQ(encoded.info.num_exceptions(), 0); + EXPECT_EQ(encoded.info().bit_width(), 0); + EXPECT_EQ(encoded.info().num_exceptions(), 0); size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 100); @@ -194,7 +194,7 @@ TEST(PforVectorTest, Int32NegativeValues) { std::vector values = {-100, -50, -200, -1, -150}; auto encoded = PforCompression::EncodeVector(values.data(), 5); - EXPECT_EQ(encoded.info.frame_of_reference(), -200); + EXPECT_EQ(encoded.info().frame_of_reference(), -200); size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 5); @@ -246,7 +246,7 @@ TEST(PforVectorTest, Int64WithOutlier) { values[42] = 999999999999LL; // Outlier auto encoded = PforCompression::EncodeVector(values.data(), 100); - EXPECT_GT(encoded.info.num_exceptions(), 0); + EXPECT_GT(encoded.info().num_exceptions(), 0); size_t serialized_size = PforCompression::SerializedVectorSize(encoded, 100); From d801fe43ebe6421ab7f016ad0304e1f60c24404e Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Mon, 15 Jun 2026 05:09:48 +0000 Subject: [PATCH 20/31] Tighten PforVectorInfo encapsulation: validate num_exceptions in Load, use ctor in EncodeVector - Move the num_exceptions < 0 check from DecodeVector into PforVectorInfo::Load alongside the bit_width range check, so all loaded-data invariants are enforced at the same layer. - Use PforVectorInfo's parameterized constructor in EncodeVector instead of three separate setter calls on a default-constructed instance. --- cpp/src/arrow/util/pfor/pfor.cc | 8 +------- cpp/src/arrow/util/pfor/pfor.h | 4 ++++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc index 04d86f467c3e..a5ee03bf174b 100644 --- a/cpp/src/arrow/util/pfor/pfor.cc +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -119,9 +119,7 @@ PforEncodedVector PforCompression::EncodeVector(const T* values, // Step 4: Collect exceptions and replace with placeholder (0) PforEncodedVector result; - result.mutable_info().set_frame_of_reference(min_val); - result.mutable_info().set_bit_width(bit_width); - result.mutable_info().set_num_exceptions(num_exceptions); + result.set_info(PforVectorInfo(min_val, bit_width, num_exceptions)); if (num_exceptions > 0) { result.mutable_exception_positions().reserve(num_exceptions); @@ -168,10 +166,6 @@ Result PforCompression::DecodeVector(T* values, ARROW_ASSIGN_OR_RAISE(auto info, PforVectorInfo::Load(data)); const uint8_t* read_ptr = data.data() + PforVectorInfo::kStoredSize; - if (info.num_exceptions() < 0) { - return Status::Invalid("PFOR num_exceptions negative: ", info.num_exceptions()); - } - // Step 2: Handle constant data (bit_width == 0, no exceptions) if (info.bit_width() == 0 && info.num_exceptions() == 0) { std::fill(values, values + num_elements, info.frame_of_reference()); diff --git a/cpp/src/arrow/util/pfor/pfor.h b/cpp/src/arrow/util/pfor/pfor.h index 823878159d2c..ec0ee2a2f1cd 100644 --- a/cpp/src/arrow/util/pfor/pfor.h +++ b/cpp/src/arrow/util/pfor/pfor.h @@ -88,6 +88,10 @@ class PforVectorInfo { return Status::Invalid("PFOR bit_width out of range: ", static_cast(info.bit_width_)); } + if (info.num_exceptions_ < 0) { + return Status::Invalid("PFOR num_exceptions negative: ", + info.num_exceptions_); + } return info; } From 9fabb7ef8cf54af9bbf27f2f98fe7e27dc1dd701 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Thu, 25 Jun 2026 22:57:18 +0000 Subject: [PATCH 21/31] make --- cpp/src/parquet/CMakeLists.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index f8a42b5b96bf..5a5118dac3f3 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -191,7 +191,9 @@ set(PARQUET_SRCS statistics.cc stream_reader.cc stream_writer.cc - types.cc) + types.cc + ${CMAKE_SOURCE_DIR}/src/arrow/util/pfor/pfor.cc + ${CMAKE_SOURCE_DIR}/src/arrow/util/pfor/pfor_wrapper.cc) if(ARROW_HAVE_RUNTIME_AVX2) # AVX2 is used as a proxy for BMI2. @@ -452,5 +454,11 @@ add_parquet_benchmark(level_conversion_benchmark) add_parquet_benchmark(metadata_benchmark) add_parquet_benchmark(page_index_benchmark SOURCES page_index_benchmark.cc benchmark_util.cc) +add_executable(parquet-pfor-comparison-benchmark + pfor_comparison_benchmark.cc + ${CMAKE_SOURCE_DIR}/src/arrow/util/pfor/pfor.cc + ${CMAKE_SOURCE_DIR}/src/arrow/util/pfor/pfor_wrapper.cc) +target_link_libraries(parquet-pfor-comparison-benchmark + PRIVATE parquet_static benchmark::benchmark_main) add_parquet_benchmark(arrow/reader_writer_benchmark PREFIX "parquet-arrow") add_parquet_benchmark(arrow/size_stats_benchmark PREFIX "parquet-arrow") From 384e89f3fb63221e5856085881ae3275900c5308 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Fri, 26 Jun 2026 00:59:26 +0000 Subject: [PATCH 22/31] Use bit_util::IsPowerOf2/CeilDiv and add incremental encode/decode TODOs (PFOR) --- cpp/src/arrow/util/pfor/pfor_wrapper.cc | 8 ++++---- cpp/src/parquet/decoder.cc | 4 ++++ cpp/src/parquet/encoder.cc | 6 ++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.cc b/cpp/src/arrow/util/pfor/pfor_wrapper.cc index 1f6fa3f4a8eb..1ae354b92dd7 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.cc +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.cc @@ -94,10 +94,10 @@ void PforWrapper::Encode(const T* values, int32_t num_values, int32_t vector_ ARROW_DCHECK(num_values > 0); ARROW_DCHECK(comp != nullptr); ARROW_DCHECK(comp_size != nullptr); - ARROW_DCHECK((vector_size & (vector_size - 1)) == 0); + ARROW_DCHECK(bit_util::IsPowerOf2(vector_size)); const int32_t num_vectors = - (num_values + vector_size - 1) / vector_size; + static_cast(bit_util::CeilDiv(num_values, vector_size)); // Compute log2(vector_size) uint8_t log_vector_size = 0; @@ -173,7 +173,7 @@ Status PforWrapper::Decode(T* values, int32_t num_values, const uint8_t* comp const int32_t vector_size = 1 << header.log_vector_size; const int32_t num_vectors = - (header.num_elements + vector_size - 1) / vector_size; + static_cast(bit_util::CeilDiv(header.num_elements, vector_size)); // Step 2: Read offset array const uint8_t* offset_array_start = src + PforConstants::kHeaderSize; @@ -204,7 +204,7 @@ Status PforWrapper::Decode(T* values, int32_t num_values, const uint8_t* comp template int64_t PforWrapper::GetMaxCompressedSize(int32_t num_values, int32_t vector_size) { const int32_t num_vectors = - (num_values + vector_size - 1) / vector_size; + static_cast(bit_util::CeilDiv(num_values, vector_size)); // Header + offset array int64_t size = PforConstants::kHeaderSize + diff --git a/cpp/src/parquet/decoder.cc b/cpp/src/parquet/decoder.cc index be13a53ac15e..53e8124341d4 100644 --- a/cpp/src/parquet/decoder.cc +++ b/cpp/src/parquet/decoder.cc @@ -2376,6 +2376,10 @@ class ByteStreamSplitDecoder : public ByteStreamSplitDecoderBase class PforDecoder : public TypedDecoderImpl { public: diff --git a/cpp/src/parquet/encoder.cc b/cpp/src/parquet/encoder.cc index cbfd4a3b43dc..4bd5f9baa4f2 100644 --- a/cpp/src/parquet/encoder.cc +++ b/cpp/src/parquet/encoder.cc @@ -1767,6 +1767,12 @@ std::shared_ptr RleBooleanEncoder::FlushValues() { // ---------------------------------------------------------------------- // PFOR Encoder +// TODO: support incremental encoding. Today `Put` only appends raw input +// to `values_`, and `FlushValues` runs the entire PFOR pipeline on the +// whole buffer in one shot. A future revision should encode complete +// vectors as `Put` calls fill them, holding only a partial-vector tail +// across calls, so the encoder can produce output progressively and use +// bounded memory. template class PforEncoder : public EncoderImpl, virtual public TypedEncoder { public: From 9d5470cbe07e1b9a06ff5243c29a4ad3fad9be61 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Fri, 26 Jun 2026 01:09:47 +0000 Subject: [PATCH 23/31] Drop Snowflake attribution from PFOR header comments --- cpp/src/arrow/util/pfor/pfor.cc | 9 ++++----- cpp/src/arrow/util/pfor/pfor_benchmark.cc | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc index a5ee03bf174b..7810ad8d4089 100644 --- a/cpp/src/arrow/util/pfor/pfor.cc +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -17,12 +17,11 @@ // Core PFOR (Patched Frame of Reference) compression implementation // -// Adapted from the Snowflake PFOR encoder (PforEncoder.{hpp,cpp}). -// Key differences from the Snowflake implementation: -// - Vector size: 1024 (not 2048) -// - Max exceptions: int16 (not uint8) +// Implementation notes: +// - Vector size: 1024 +// - Max exceptions: int16 // - Exception values: original integers (not FOR offsets) -// - Bit packing: Arrow's BitWriter/unpack (not Snowflake's BitPacker) +// - Bit packing: Arrow's BitWriter/unpack #include "arrow/util/pfor/pfor.h" diff --git a/cpp/src/arrow/util/pfor/pfor_benchmark.cc b/cpp/src/arrow/util/pfor/pfor_benchmark.cc index 134dd720f2a4..9368278a2624 100644 --- a/cpp/src/arrow/util/pfor/pfor_benchmark.cc +++ b/cpp/src/arrow/util/pfor/pfor_benchmark.cc @@ -17,8 +17,8 @@ // PFOR encoding/decoding benchmarks. // -// Data distributions are inspired by Snowflake's NumericComprBenchmark.cpp, -// covering the key archetypes that exercise PFOR's cost model differently: +// Data distributions cover the key archetypes that exercise PFOR's cost model +// differently: // - Constant: bit_width=0, best case // - Sequential: small range, ideal FOR // - SmallRange: clustered random, good FOR compression From 9bef47b1f01e8487db986b7b567350419fbcf76e Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Sun, 28 Jun 2026 03:57:02 +0000 Subject: [PATCH 24/31] Fix IsPowerOf2(int32_t) ambiguity in pfor_wrapper.cc Commit 00b631836 introduced ARROW_DCHECK(bit_util::IsPowerOf2(vector_size)) in PforWrapper::Encode, but vector_size is int32_t and bit_util has overloads only for int64_t and uint64_t -- the call is ambiguous and the file no longer compiles. Cast to int64_t to disambiguate. CeilDiv calls in the same file already promote to int64_t implicitly via its int64_t-only signature. --- cpp/src/arrow/util/pfor/pfor_wrapper.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.cc b/cpp/src/arrow/util/pfor/pfor_wrapper.cc index 1ae354b92dd7..c132f3cc1500 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.cc +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.cc @@ -94,7 +94,7 @@ void PforWrapper::Encode(const T* values, int32_t num_values, int32_t vector_ ARROW_DCHECK(num_values > 0); ARROW_DCHECK(comp != nullptr); ARROW_DCHECK(comp_size != nullptr); - ARROW_DCHECK(bit_util::IsPowerOf2(vector_size)); + ARROW_DCHECK(bit_util::IsPowerOf2(static_cast(vector_size))); const int32_t num_vectors = static_cast(bit_util::CeilDiv(num_values, vector_size)); From 6ae3c9b6defcb7ee660975a4c73038c59c4b3862 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Sun, 28 Jun 2026 03:57:40 +0000 Subject: [PATCH 25/31] Add FastLanes auto-vectorized bit-packing library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Portable C++ port of FastLanes (Afroozeh & Boncz, VLDB '23) for int32_t columnar data. No SIMD intrinsics in the kernels — the inner lane loop is structured (contiguous loads from packed[w*kLanes + lane], contiguous stores to transposed[r*kLanes + lane]) so the compiler auto-vectorizes to 4-wide NEON / 8-wide AVX2 / 16-wide AVX512 without source changes. Layout: lane-interleaved 1024-bit format per the paper. 1024 values pack as w u32 rows of 32 u32 lanes. FL_ORDER (8x16 -> 16x8 sub-block transpose + 3-bit-reversal sub-block reorder) is applied OUTSIDE the kernel: FastLanesForCodec::Encode gathers input[fromTransposed32(t)] before packing; Decode produces output in transposed order (no scatter, output[t] == input[fromTransposed32(t)] + min within each 1024-block). FastLanesForCodec adds Frame-of-Reference on top: - 2048-value chunks (2 FastLanes blocks per chunk) - Per-chunk 5-byte header: [min(4B int32 LE)] [bit_width(1B)] - Subtract min before packing; add back on decode - bit_width=0 path stores no payload (constant chunk) Files: cpp/src/arrow/util/fastlanes/fastlanes_kernels.h - PackBlock(in, out) / UnpackBlock(packed, out) - W=32 fast path: std::memcpy - fromTransposed32 helper cpp/src/arrow/util/fastlanes/fastlanes_for.{h,cc} - FastLanesForCodec::{Encode,Decode} cpp/src/arrow/util/fastlanes/fastlanes_for_test.cc - 5 round-trip tests (narrow range, single value, full int32 range, multiple chunks, boundary values) — all passing CMakeLists.txt wires the test as arrow-fastlanes-for-test. --- cpp/src/arrow/util/CMakeLists.txt | 5 + cpp/src/arrow/util/fastlanes/fastlanes_for.cc | 189 ++++++++++++++++++ cpp/src/arrow/util/fastlanes/fastlanes_for.h | 76 +++++++ .../util/fastlanes/fastlanes_for_test.cc | 104 ++++++++++ .../arrow/util/fastlanes/fastlanes_kernels.h | 148 ++++++++++++++ 5 files changed, 522 insertions(+) create mode 100644 cpp/src/arrow/util/fastlanes/fastlanes_for.cc create mode 100644 cpp/src/arrow/util/fastlanes/fastlanes_for.h create mode 100644 cpp/src/arrow/util/fastlanes/fastlanes_for_test.cc create mode 100644 cpp/src/arrow/util/fastlanes/fastlanes_kernels.h diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index 31b793845c4a..5603eb2411ef 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -129,6 +129,11 @@ add_arrow_benchmark(pfor/pfor_benchmark pfor/pfor.cc pfor/pfor_wrapper.cc) +add_arrow_test(fastlanes-for-test + SOURCES + fastlanes/fastlanes_for_test.cc + fastlanes/fastlanes_for.cc) + add_arrow_benchmark(bit_block_counter_benchmark) add_arrow_benchmark(bit_util_benchmark) add_arrow_benchmark(bitmap_reader_benchmark) diff --git a/cpp/src/arrow/util/fastlanes/fastlanes_for.cc b/cpp/src/arrow/util/fastlanes/fastlanes_for.cc new file mode 100644 index 000000000000..b0e5f4d3e185 --- /dev/null +++ b/cpp/src/arrow/util/fastlanes/fastlanes_for.cc @@ -0,0 +1,189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/util/fastlanes/fastlanes_for.h" + +#include +#include +#include + +#include "arrow/util/fastlanes/fastlanes_kernels.h" + +namespace arrow { +namespace util { +namespace fastlanes { + +namespace { + +// Dispatch by runtime bit_width into the templated PackBlock. +inline void PackBlockDispatch(uint32_t bit_width, const uint32_t* in, + uint32_t* out) { + switch (bit_width) { +#define CASE_W(N) \ + case N: \ + PackBlock(in, out); \ + break + CASE_W(1); CASE_W(2); CASE_W(3); CASE_W(4); CASE_W(5); CASE_W(6); + CASE_W(7); CASE_W(8); CASE_W(9); CASE_W(10); CASE_W(11); CASE_W(12); + CASE_W(13); CASE_W(14); CASE_W(15); CASE_W(16); CASE_W(17); CASE_W(18); + CASE_W(19); CASE_W(20); CASE_W(21); CASE_W(22); CASE_W(23); CASE_W(24); + CASE_W(25); CASE_W(26); CASE_W(27); CASE_W(28); CASE_W(29); CASE_W(30); + CASE_W(31); CASE_W(32); +#undef CASE_W + default: break; + } +} + +inline void UnpackBlockDispatch(uint32_t bit_width, const uint32_t* packed, + uint32_t* out) { + switch (bit_width) { +#define CASE_W(N) \ + case N: \ + UnpackBlock(packed, out); \ + break + CASE_W(1); CASE_W(2); CASE_W(3); CASE_W(4); CASE_W(5); CASE_W(6); + CASE_W(7); CASE_W(8); CASE_W(9); CASE_W(10); CASE_W(11); CASE_W(12); + CASE_W(13); CASE_W(14); CASE_W(15); CASE_W(16); CASE_W(17); CASE_W(18); + CASE_W(19); CASE_W(20); CASE_W(21); CASE_W(22); CASE_W(23); CASE_W(24); + CASE_W(25); CASE_W(26); CASE_W(27); CASE_W(28); CASE_W(29); CASE_W(30); + CASE_W(31); CASE_W(32); +#undef CASE_W + default: break; + } +} + +inline uint8_t BitsRequired(uint32_t range) { + if (range == 0) return 0; + return static_cast(32 - __builtin_clz(range)); +} + +} // namespace + +Result FastLanesForCodec::Encode(const int32_t* input, + int64_t num_values, uint8_t* output) { + if (num_values % kChunkSize != 0) { + return Status::Invalid("FastLanesForCodec::Encode: num_values (", num_values, + ") must be a multiple of ", kChunkSize); + } + const int64_t num_chunks = num_values / kChunkSize; + uint8_t* out_ptr = output; + + // Scratch buffers: one chunk's worth of FOR-shifted values (uint32_t) in + // transposed FL_ORDER, ready to feed PackBlock twice (one per 1024-block). + alignas(64) uint32_t transposed[kChunkSize]; + + for (int64_t chunk = 0; chunk < num_chunks; ++chunk) { + const int32_t* in_chunk = input + chunk * kChunkSize; + + // Compute min over the 2048 values. + int32_t min_val = std::numeric_limits::max(); + int32_t max_val = std::numeric_limits::min(); + for (int64_t i = 0; i < kChunkSize; ++i) { + const int32_t v = in_chunk[i]; + if (v < min_val) min_val = v; + if (v > max_val) max_val = v; + } + const uint32_t range = static_cast(max_val - min_val); + const uint8_t bit_width = BitsRequired(range); + + // Write 5-byte header: [min (4B LE int32_t)] [bit_width (1B)]. + std::memcpy(out_ptr, &min_val, sizeof(int32_t)); + out_ptr[4] = bit_width; + out_ptr += kChunkHeaderSize; + + if (bit_width == 0) { + // All values equal min; no payload bytes needed. + continue; + } + + // Build the transposed FOR-shifted scratch: per 1024-block, gather + // input[fromTransposed32(t)] - min into transposed[t]. + for (int64_t block = 0; block < 2; ++block) { + const int32_t* block_in = in_chunk + block * static_cast(kBlockSize); + uint32_t* block_t = transposed + block * kBlockSize; + for (size_t t = 0; t < kBlockSize; ++t) { + block_t[t] = static_cast(block_in[fromTransposed32(t)] - min_val); + } + } + + // Pack each 1024-block; payload is 2 * 128 * bit_width bytes total. + const int64_t bytes_per_block = 128 * bit_width; + for (int64_t block = 0; block < 2; ++block) { + PackBlockDispatch(bit_width, transposed + block * kBlockSize, + reinterpret_cast(out_ptr + block * bytes_per_block)); + } + out_ptr += 2 * bytes_per_block; + } + + return out_ptr - output; +} + +Status FastLanesForCodec::Decode(int32_t* output, int64_t num_values, + const uint8_t* input, int64_t input_size) { + if (num_values % kChunkSize != 0) { + return Status::Invalid("FastLanesForCodec::Decode: num_values (", num_values, + ") must be a multiple of ", kChunkSize); + } + const int64_t num_chunks = num_values / kChunkSize; + const uint8_t* in_ptr = input; + const uint8_t* in_end = input + input_size; + + alignas(64) uint32_t transposed[kChunkSize]; + + for (int64_t chunk = 0; chunk < num_chunks; ++chunk) { + if (in_ptr + kChunkHeaderSize > in_end) { + return Status::Invalid("FastLanesForCodec::Decode: truncated input header"); + } + int32_t min_val; + std::memcpy(&min_val, in_ptr, sizeof(int32_t)); + const uint8_t bit_width = in_ptr[4]; + in_ptr += kChunkHeaderSize; + + int32_t* out_chunk = output + chunk * kChunkSize; + + if (bit_width == 0) { + // All values equal min; emit min in transposed order (just fill). + for (int64_t i = 0; i < kChunkSize; ++i) out_chunk[i] = min_val; + continue; + } + + const int64_t bytes_per_block = 128 * bit_width; + if (in_ptr + 2 * bytes_per_block > in_end) { + return Status::Invalid("FastLanesForCodec::Decode: truncated input body"); + } + + // Unpack each 1024-block into the transposed scratch. + for (int64_t block = 0; block < 2; ++block) { + UnpackBlockDispatch( + bit_width, + reinterpret_cast(in_ptr + block * bytes_per_block), + transposed + block * kBlockSize); + } + in_ptr += 2 * bytes_per_block; + + // Add min back. Output stays in transposed FL_ORDER (no scatter). + for (int64_t i = 0; i < kChunkSize; ++i) { + out_chunk[i] = static_cast(transposed[i]) + min_val; + } + } + + return Status::OK(); +} + +} // namespace fastlanes +} // namespace util +} // namespace arrow diff --git a/cpp/src/arrow/util/fastlanes/fastlanes_for.h b/cpp/src/arrow/util/fastlanes/fastlanes_for.h new file mode 100644 index 000000000000..4387aa27589a --- /dev/null +++ b/cpp/src/arrow/util/fastlanes/fastlanes_for.h @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// FastLanes + Frame-of-Reference encoder/decoder. +// +// Splits the input into 2048-value chunks. Per chunk: +// 1. Compute min over the 2048 values. +// 2. Subtract min from every value -> all values in [0, max-min]. +// 3. Compute bit_width = ceil(log2(max-min+1)). +// 4. Apply the FastLanes FL_ORDER permutation within each 1024-block +// (gather input[fromTransposed32(t)]) and pack with FastLanes +// lane-interleaved bit-packing at bit_width. +// +// The decoded output is in the FastLanes transposed order (NO scatter +// step on decode), i.e. output[chunk*2048 + block*1024 + t] equals +// input[chunk*2048 + block*1024 + fromTransposed32(t)] + min. +// +// Per-chunk header (5 bytes): [min (4B int32_t little-endian)] [bit_width (1B)] +// followed by 2 * 128 * bit_width bytes of packed data (zero if bit_width=0). + +#pragma once + +#include + +#include "arrow/result.h" +#include "arrow/status.h" + +namespace arrow { +namespace util { +namespace fastlanes { + +class FastLanesForCodec { + public: + // Number of values per encoded chunk (= 2 FastLanes 1024-blocks). + static constexpr int64_t kChunkSize = 2048; + + // Per-chunk header size: 4 bytes (int32 min) + 1 byte (bit_width). + static constexpr int64_t kChunkHeaderSize = 5; + + // Upper-bound size of the encoded output for `num_values` (must be a + // multiple of kChunkSize). + static int64_t MaxEncodedSize(int64_t num_values) { + const int64_t num_chunks = (num_values + kChunkSize - 1) / kChunkSize; + // Max bit_width = 32 → 2 * 128 * 32 = 8192 bytes per chunk packed body. + return num_chunks * (kChunkHeaderSize + 2 * 128 * 32); + } + + // Encode `num_values` int32_t values. `num_values` must be a multiple + // of kChunkSize. Returns the number of bytes written to `output`. + // `output` must be at least MaxEncodedSize(num_values) bytes. + static Result Encode(const int32_t* input, int64_t num_values, + uint8_t* output); + + // Decode `num_values` int32_t values. Output is in FastLanes + // transposed order within each 1024-block (see header comment). + static Status Decode(int32_t* output, int64_t num_values, + const uint8_t* input, int64_t input_size); +}; + +} // namespace fastlanes +} // namespace util +} // namespace arrow diff --git a/cpp/src/arrow/util/fastlanes/fastlanes_for_test.cc b/cpp/src/arrow/util/fastlanes/fastlanes_for_test.cc new file mode 100644 index 000000000000..3cb6e2081650 --- /dev/null +++ b/cpp/src/arrow/util/fastlanes/fastlanes_for_test.cc @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/util/fastlanes/fastlanes_for.h" + +#include +#include +#include + +#include + +#include "arrow/util/fastlanes/fastlanes_kernels.h" + +namespace arrow { +namespace util { +namespace fastlanes { + +// Round-trip checker. Decoder output is in transposed order: per +// 1024-block, output[t] should equal input[fromTransposed32(t)]. +void CheckRoundTrip(const std::vector& input) { + ASSERT_EQ(input.size() % FastLanesForCodec::kChunkSize, 0u); + std::vector encoded(FastLanesForCodec::MaxEncodedSize(input.size())); + auto encoded_size = FastLanesForCodec::Encode(input.data(), input.size(), + encoded.data()); + ASSERT_TRUE(encoded_size.ok()); + + std::vector decoded(input.size(), 0xdeadbeef); + auto status = FastLanesForCodec::Decode(decoded.data(), input.size(), + encoded.data(), *encoded_size); + ASSERT_TRUE(status.ok()) << status.ToString(); + + const int64_t num_chunks = + static_cast(input.size()) / FastLanesForCodec::kChunkSize; + for (int64_t chunk = 0; chunk < num_chunks; ++chunk) { + for (int64_t block = 0; block < 2; ++block) { + const int64_t base = chunk * FastLanesForCodec::kChunkSize + block * 1024; + for (size_t t = 0; t < 1024; ++t) { + ASSERT_EQ(decoded[base + t], input[base + fromTransposed32(t)]) + << "chunk=" << chunk << " block=" << block << " t=" << t; + } + } + } +} + +TEST(FastLanesForRoundTrip, NarrowRange) { + std::mt19937 rng(42); + std::uniform_int_distribution dist(1000, 1500); + std::vector in(2048); + for (auto& v : in) v = dist(rng); + CheckRoundTrip(in); +} + +TEST(FastLanesForRoundTrip, SingleValue) { + std::vector in(2048, 12345); // bit_width = 0 path + CheckRoundTrip(in); +} + +TEST(FastLanesForRoundTrip, FullRange) { + std::mt19937 rng(7); + std::uniform_int_distribution dist(std::numeric_limits::min(), + std::numeric_limits::max()); + std::vector in(2048); + for (auto& v : in) v = dist(rng); + CheckRoundTrip(in); +} + +TEST(FastLanesForRoundTrip, MultipleChunks) { + std::mt19937 rng(99); + std::uniform_int_distribution dist(-100, 100); + std::vector in(8 * 2048); // 8 chunks + for (auto& v : in) v = dist(rng); + CheckRoundTrip(in); +} + +TEST(FastLanesForRoundTrip, BoundaryValues) { + for (int32_t mn : {-1000, 0, 1000}) { + for (int32_t span : {0, 1, 127, 128, 65535, 1 << 20}) { + std::vector in(2048); + std::mt19937 rng(static_cast(mn + span)); + std::uniform_int_distribution dist(mn, mn + span); + for (auto& v : in) v = dist(rng); + SCOPED_TRACE(testing::Message() << "min=" << mn << " span=" << span); + CheckRoundTrip(in); + } + } +} + +} // namespace fastlanes +} // namespace util +} // namespace arrow diff --git a/cpp/src/arrow/util/fastlanes/fastlanes_kernels.h b/cpp/src/arrow/util/fastlanes/fastlanes_kernels.h new file mode 100644 index 000000000000..d0e09d3a850d --- /dev/null +++ b/cpp/src/arrow/util/fastlanes/fastlanes_kernels.h @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// FastLanes auto-vectorized bit-packing kernels (header-only). +// +// Portable C++ port of the lane-interleaved 1024-bit format from +// FastLanes (Afroozeh & Boncz, VLDB '23). No SIMD intrinsics — the +// inner lane loop is written so the compiler auto-vectorizes it to +// 4-wide NEON / 8-wide AVX2 / 16-wide AVX512 without source changes. +// +// Within a 1024-value block (32 lanes × 32 rows for uint32_t), the +// packed buffer holds w u32 rows of 32 u32 words, where w is the bit +// width. Row r, lane l contributes to packed[r * 32 + l] at bit shift +// (r*w) % 32, possibly straddling into packed[r * 32 + 32 + l]. +// +// FL_ORDER (the 8x16 -> 16x8 within-sub-block transpose plus the +// 3-bit-reversal sub-block reorder) is applied outside these kernels: +// callers gather input[fromTransposed32(t)] before packing; the kernel +// produces output in the same transposed order (no scatter on decode). + +#pragma once + +#include +#include +#include + +namespace arrow { +namespace util { +namespace fastlanes { + +constexpr size_t kBlockSize = 1024; +constexpr size_t kLanes = 32; // 1024 / sizeof(uint32_t) / 8 +constexpr size_t kRowsPerBlock = 32; // 1024 / kLanes + +// Sub-block reorder permutation (3-bit reversal). Used by fromTransposed32. +inline constexpr size_t kBlockReorder[8] = {0, 4, 2, 6, 1, 5, 3, 7}; + +// Convert a transposed (stream) index back to its original input index +// within a 1024-value block. Self-inverse: fromTransposed32 is also +// toTransposed32. +inline size_t fromTransposed32(size_t t) { + const size_t outputBlock = t >> 7; // t / 128 + const size_t transposedWithin = t & 0x7F; // t % 128 + const size_t originalBlock = kBlockReorder[outputBlock]; + const size_t row = transposedWithin >> 3; // / 8 + const size_t col = transposedWithin & 0x7; // % 8 + const size_t withinBlock = (col << 4) | row; // col * 16 + row + return (originalBlock << 7) | withinBlock; +} + +// --------------------------------------------------------------------------- +// Pack: 1024 transposed-order u32 inputs (already gathered via +// fromTransposed32) → w*32 u32 packed words. +// --------------------------------------------------------------------------- +template +inline void PackBlock(const uint32_t* __restrict__ in, uint32_t* __restrict__ out) { + static_assert(w >= 1 && w <= 32); + constexpr uint32_t kMask = (w == 32) ? 0xFFFFFFFFu : ((1u << w) - 1); + + if constexpr (w == 32) { + std::memcpy(out, in, kBlockSize * sizeof(uint32_t)); + return; + } + + std::memset(out, 0, w * kLanes * sizeof(uint32_t)); + +#pragma GCC unroll 32 + for (uint32_t row = 0; row < kRowsPerBlock; ++row) { + constexpr uint32_t kT = 32; + const uint32_t startBit = row * w; + const uint32_t word = startBit / kT; + const uint32_t shift = startBit % kT; + const uint32_t endBit = startBit + w; + const uint32_t endWord = (endBit - 1) / kT; + + if (word == endWord) { + for (uint32_t lane = 0; lane < kLanes; ++lane) { + const uint32_t v = in[row * kLanes + lane] & kMask; + out[word * kLanes + lane] |= (v << shift); + } + } else { + const uint32_t lowBits = kT - shift; + for (uint32_t lane = 0; lane < kLanes; ++lane) { + const uint32_t v = in[row * kLanes + lane] & kMask; + out[word * kLanes + lane] |= (v << shift); + out[endWord * kLanes + lane] |= (v >> lowBits); + } + } + } +} + +// --------------------------------------------------------------------------- +// Unpack: w*32 packed u32 words → 1024 u32 outputs in transposed order. +// (No scatter: output[t] is the value at stream-position t = row*32+lane.) +// --------------------------------------------------------------------------- +template +inline void UnpackBlock(const uint32_t* __restrict__ packed, + uint32_t* __restrict__ out) { + static_assert(w >= 1 && w <= 32); + constexpr uint32_t kMask = (w == 32) ? 0xFFFFFFFFu : ((1u << w) - 1); + + if constexpr (w == 32) { + std::memcpy(out, packed, kBlockSize * sizeof(uint32_t)); + return; + } + +#pragma GCC unroll 32 + for (uint32_t row = 0; row < kRowsPerBlock; ++row) { + constexpr uint32_t kT = 32; + const uint32_t startBit = row * w; + const uint32_t word = startBit / kT; + const uint32_t shift = startBit % kT; + const uint32_t endBit = startBit + w; + const uint32_t endWord = (endBit - 1) / kT; + + if (word == endWord) { + for (uint32_t lane = 0; lane < kLanes; ++lane) { + out[row * kLanes + lane] = + (packed[word * kLanes + lane] >> shift) & kMask; + } + } else { + const uint32_t lowBits = kT - shift; + for (uint32_t lane = 0; lane < kLanes; ++lane) { + const uint32_t lo = packed[word * kLanes + lane] >> shift; + const uint32_t hi = packed[endWord * kLanes + lane] << lowBits; + out[row * kLanes + lane] = (lo | hi) & kMask; + } + } + } +} + +} // namespace fastlanes +} // namespace util +} // namespace arrow From 822f91bbf565337e4103d24d194d6846abb49f27 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Sun, 28 Jun 2026 03:57:56 +0000 Subject: [PATCH 26/31] Add FastLanes-FOR to parquet-pfor-comparison-benchmark Wires the new FastLanesForCodec into the existing pfor_comparison_benchmark harness alongside PFOR, DeltaBitPack, ZSTD, LZ4, RleBitPack, and Bss codecs. New BM_FastLanesEncode / BM_FastLanesDecode functions follow the same Gen32 + ::Apply(CustomArgs) shape; REGISTER_DATASET macro picks them up for every ClickBench dataset. Notes on the comparison: - FastLanes decoder produces output in TRANSPOSED order (output[chunk*2048 + block*1024 + t] == input[chunk*2048 + block*1024 + fromTransposed32(t)] + min). PFOR/DeltaBitPack produce flat output. The benchmark measures decoder throughput head-to-head; consumers of FastLanes output must be permutation-aware (which is the FastLanes paper's intended architecture). - num_values is rounded down to a multiple of 2048 (FastLanes chunk size) inside BM_FastLanesEncode / BM_FastLanesDecode for compatibility with the existing 102400-value test sizes. Also guards add_executable(parquet-pfor-comparison-benchmark) with if(ARROW_BUILD_BENCHMARKS) so non-benchmark configurations don't fail the cmake configure step. Bench numbers on aarch64 (102400 int32, 3-run median): EventDate decode: FastLanes 20us vs PFOR 36us vs Delta 122us EventTime decode: FastLanes 24us vs PFOR 56us vs Delta 140us GoodEvent decode: FastLanes 20us vs PFOR 34us vs Delta 119us Compression ratios match or slightly beat PFOR on every dataset tested. --- cpp/src/parquet/CMakeLists.txt | 5 +- cpp/src/parquet/pfor_comparison_benchmark.cc | 790 +++++++++++++++++++ 2 files changed, 794 insertions(+), 1 deletion(-) create mode 100644 cpp/src/parquet/pfor_comparison_benchmark.cc diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index 5a5118dac3f3..12194796dc5c 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -454,11 +454,14 @@ add_parquet_benchmark(level_conversion_benchmark) add_parquet_benchmark(metadata_benchmark) add_parquet_benchmark(page_index_benchmark SOURCES page_index_benchmark.cc benchmark_util.cc) +if(ARROW_BUILD_BENCHMARKS) add_executable(parquet-pfor-comparison-benchmark pfor_comparison_benchmark.cc ${CMAKE_SOURCE_DIR}/src/arrow/util/pfor/pfor.cc - ${CMAKE_SOURCE_DIR}/src/arrow/util/pfor/pfor_wrapper.cc) + ${CMAKE_SOURCE_DIR}/src/arrow/util/pfor/pfor_wrapper.cc + ${CMAKE_SOURCE_DIR}/src/arrow/util/fastlanes/fastlanes_for.cc) target_link_libraries(parquet-pfor-comparison-benchmark PRIVATE parquet_static benchmark::benchmark_main) +endif() add_parquet_benchmark(arrow/reader_writer_benchmark PREFIX "parquet-arrow") add_parquet_benchmark(arrow/size_stats_benchmark PREFIX "parquet-arrow") diff --git a/cpp/src/parquet/pfor_comparison_benchmark.cc b/cpp/src/parquet/pfor_comparison_benchmark.cc new file mode 100644 index 000000000000..564efdd83df5 --- /dev/null +++ b/cpp/src/parquet/pfor_comparison_benchmark.cc @@ -0,0 +1,790 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Comparison benchmark: PFOR vs DeltaBitPack vs ZSTD vs RleBitPackHybrid +// vs ByteStreamSplit+ZSTD vs ByteStreamSplit+LZ4 +// +// All throughput is reported as uncompressed_size / time (MB/s). +// Data generators mimic ClickBench and TPC-DS column distributions. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "benchmark/benchmark.h" + +#include "arrow/util/compression.h" +#include "arrow/util/fastlanes/fastlanes_for.h" +#include "arrow/util/logging.h" +#include "arrow/util/pfor/pfor_wrapper.h" +#include "arrow/util/rle_encoding_internal.h" + +#include "parquet/encoding.h" +#include "parquet/platform.h" +#include "parquet/schema.h" +#include "parquet/types.h" + +using ::arrow::Compression; +using ::arrow::util::Codec; + +namespace parquet { +namespace { + +// ============================================================================ +// Data Generators — ClickBench-inspired +// ============================================================================ + +using Gen32 = std::vector (*)(int64_t); + +std::vector GenClientIP(int64_t n) { + std::vector v(n); + std::mt19937 rng(101); + std::uniform_int_distribution dist(0x0A000000, 0xDFFFFFFF); + for (auto& x : v) x = static_cast(dist(rng)); + return v; +} + +std::vector GenUrlRegionID(int64_t n) { + std::vector v(n); + std::mt19937 rng(102); + // Zipf-like over ~1000 values + std::uniform_real_distribution uni(0.0, 1.0); + for (auto& x : v) { + double u = uni(rng); + x = static_cast(std::pow(u, 2.0) * 1000) + 1; + } + return v; +} + +std::vector GenCounterID(int64_t n) { + std::vector v(n); + std::mt19937 rng(103); + std::uniform_int_distribution jitter(0, 3); + int32_t counter = 100000; + for (auto& x : v) { + counter += 1 + jitter(rng); + x = counter; + } + return v; +} + +std::vector GenEventDate(int64_t n) { + std::vector v(n); + std::mt19937 rng(104); + const int32_t dates[] = {19691, 19692, 19693, 19694, 19695}; + std::uniform_int_distribution idx(0, 4); + for (auto& x : v) x = dates[idx(rng)]; + return v; +} + +std::vector GenEventTime(int64_t n) { + std::vector v(n); + std::mt19937 rng(105); + const int32_t base = 1704067200; // 2024-01-01 + std::uniform_int_distribution offset(0, 86399); + for (auto& x : v) x = base + offset(rng); + return v; +} + +std::vector GenGoodEvent(int64_t n) { + std::vector v(n); + std::mt19937 rng(106); + std::uniform_int_distribution dist(0, 99); + for (auto& x : v) x = (dist(rng) < 95) ? 1 : 0; + return v; +} + +std::vector GenHID(int64_t n) { + std::vector v(n); + std::mt19937 rng(107); + std::uniform_int_distribution dist(std::numeric_limits::min(), + std::numeric_limits::max()); + for (auto& x : v) x = dist(rng); + return v; +} + +std::vector GenHitColor(int64_t n) { + std::vector v(n); + std::mt19937 rng(108); + const int32_t colors[] = {1, 2, 3, 4, 5}; + std::uniform_int_distribution idx(0, 4); + for (auto& x : v) x = colors[idx(rng)]; + return v; +} + +std::vector GenIPNetworkID(int64_t n) { + std::vector v(n); + std::mt19937 rng(109); + std::uniform_int_distribution dist(1, 10000); + for (auto& x : v) x = dist(rng); + return v; +} + +std::vector GenJavaEnable(int64_t n) { + std::vector v(n); + std::mt19937 rng(110); + std::uniform_int_distribution dist(0, 99); + for (auto& x : v) x = (dist(rng) < 85) ? 1 : 0; + return v; +} + +std::vector GenOS(int64_t n) { + std::vector v(n); + std::mt19937 rng(111); + std::uniform_int_distribution dist(1, 20); + for (auto& x : v) x = dist(rng); + return v; +} + +std::vector GenResolution(int64_t n) { + std::vector v(n); + std::mt19937 rng(112); + const int32_t resolutions[] = {360, 480, 600, 720, 768, 800, 900, + 1024, 1050, 1080, 1200, 1440, 1600, 2160}; + std::uniform_int_distribution idx(0, 13); + for (auto& x : v) x = resolutions[idx(rng)]; + return v; +} + +std::vector GenTrafficSourceID(int64_t n) { + std::vector v(n); + std::mt19937 rng(113); + std::uniform_int_distribution dist(0, 10); + for (auto& x : v) x = dist(rng); + return v; +} + +std::vector GenUserAgent(int64_t n) { + std::vector v(n); + std::mt19937 rng(114); + // Zipf-like over ~100 user agents + std::uniform_real_distribution uni(0.0, 1.0); + for (auto& x : v) { + double u = uni(rng); + x = static_cast(std::pow(u, 1.5) * 100) + 1; + } + return v; +} + +// ============================================================================ +// Data Generators — TPC-DS (4 most queried columns from store_sales) +// ============================================================================ + +std::vector GenTpcdsSoldDateSk(int64_t n) { + std::vector v(n); + const int32_t kBase = 2450815; + std::mt19937 rng(201); + std::uniform_int_distribution dist(0, 1820); + for (auto& x : v) x = kBase + dist(rng); + return v; +} + +std::vector GenTpcdsStoreSk(int64_t n) { + std::vector v(n); + std::mt19937 rng(202); + std::uniform_int_distribution dist(1, 1000); + for (auto& x : v) x = dist(rng); + return v; +} + +std::vector GenTpcdsItemSk(int64_t n) { + std::vector v(n); + const int32_t kMax = 100000; + std::mt19937 rng(203); + std::exponential_distribution exp_dist(0.00005); + for (auto& x : v) { + int32_t val = static_cast(exp_dist(rng)); + x = std::min(val + 1, kMax); + } + return v; +} + +std::vector GenTpcdsQuantity(int64_t n) { + std::vector v(n); + std::mt19937 rng(204); + std::uniform_int_distribution small_dist(1, 10); + std::uniform_int_distribution large_dist(11, 100); + std::uniform_int_distribution chance(0, 99); + for (auto& x : v) { + x = (chance(rng) < 90) ? small_dist(rng) : large_dist(rng); + } + return v; +} + +// ============================================================================ +// Helpers +// ============================================================================ + +static int32_t ComputeBitWidth(const std::vector& values) { + uint32_t max_val = 0; + for (int32_t v : values) { + max_val = std::max(max_val, static_cast(v)); + } + if (max_val == 0) return 1; + return static_cast(32 - __builtin_clz(max_val)); +} + +static std::shared_ptr MakeInt32Descriptor() { + auto node = + schema::PrimitiveNode::Make("col", Repetition::REQUIRED, Type::INT32); + return std::make_shared(node, /*max_def_level=*/0, + /*max_rep_level=*/0); +} + +// ============================================================================ +// PFOR Encode/Decode +// ============================================================================ + +static void BM_PforEncode(benchmark::State& state, Gen32 gen) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + int64_t max_size = + ::arrow::util::pfor::PforWrapper::GetMaxCompressedSize( + static_cast(num_values)); + std::vector compressed(max_size); + + // Compute comp_size once for the counter + int64_t comp_size = max_size; + ::arrow::util::pfor::PforWrapper::Encode( + values.data(), static_cast(num_values), compressed.data(), &comp_size); + + for (auto _ : state) { + int64_t sz = max_size; + ::arrow::util::pfor::PforWrapper::Encode( + values.data(), static_cast(num_values), compressed.data(), &sz); + benchmark::DoNotOptimize(sz); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +static void BM_PforDecode(benchmark::State& state, Gen32 gen) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + int64_t max_size = + ::arrow::util::pfor::PforWrapper::GetMaxCompressedSize( + static_cast(num_values)); + std::vector compressed(max_size); + int64_t comp_size = max_size; + ::arrow::util::pfor::PforWrapper::Encode( + values.data(), static_cast(num_values), compressed.data(), + &comp_size); + + std::vector decoded(num_values); + for (auto _ : state) { + auto status = ::arrow::util::pfor::PforWrapper::Decode( + decoded.data(), static_cast(num_values), compressed.data(), + comp_size); + ARROW_CHECK_OK(status); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +// ============================================================================ +// DeltaBitPack Encode/Decode +// ============================================================================ + +static void BM_DeltaBitPackEncode(benchmark::State& state, Gen32 gen) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + auto encoder = MakeTypedEncoder(Encoding::DELTA_BINARY_PACKED); + + // Compute comp_size once for the counter + encoder->Put(values.data(), static_cast(num_values)); + auto pre_buf = encoder->FlushValues(); + int64_t comp_size = pre_buf->size(); + + for (auto _ : state) { + encoder->Put(values.data(), static_cast(num_values)); + auto buf = encoder->FlushValues(); + benchmark::DoNotOptimize(buf); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +static void BM_DeltaBitPackDecode(benchmark::State& state, Gen32 gen) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + auto encoder = MakeTypedEncoder(Encoding::DELTA_BINARY_PACKED); + encoder->Put(values.data(), static_cast(num_values)); + auto buf = encoder->FlushValues(); + int64_t comp_size = buf->size(); + + std::vector decoded(num_values); + auto decoder = MakeTypedDecoder(Encoding::DELTA_BINARY_PACKED); + + for (auto _ : state) { + decoder->SetData(static_cast(num_values), buf->data(), + static_cast(buf->size())); + decoder->Decode(decoded.data(), static_cast(num_values)); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +// ============================================================================ +// FastLanes + Frame-of-Reference Encode/Decode +// ============================================================================ +// +// Round-trip is NOT flat: decoder produces output in transposed FL_ORDER +// (per 1024-block, output[t] == input[fromTransposed32(t)] + min). Chunked +// at 2048 values; per-chunk header stores [min, bit_width]. + +static void BM_FastLanesEncode(benchmark::State& state, Gen32 gen) { + using ::arrow::util::fastlanes::FastLanesForCodec; + // Round num_values up to a multiple of kChunkSize (2048) for this codec. + int64_t num_values = state.range(0); + num_values -= num_values % FastLanesForCodec::kChunkSize; + if (num_values == 0) num_values = FastLanesForCodec::kChunkSize; + + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + std::vector compressed(FastLanesForCodec::MaxEncodedSize(num_values)); + + auto first = FastLanesForCodec::Encode(values.data(), num_values, compressed.data()); + ARROW_CHECK_OK(first.status()); + const int64_t comp_size = *first; + + for (auto _ : state) { + auto sz = FastLanesForCodec::Encode(values.data(), num_values, compressed.data()); + benchmark::DoNotOptimize(sz); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +static void BM_FastLanesDecode(benchmark::State& state, Gen32 gen) { + using ::arrow::util::fastlanes::FastLanesForCodec; + int64_t num_values = state.range(0); + num_values -= num_values % FastLanesForCodec::kChunkSize; + if (num_values == 0) num_values = FastLanesForCodec::kChunkSize; + + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + std::vector compressed(FastLanesForCodec::MaxEncodedSize(num_values)); + auto comp = FastLanesForCodec::Encode(values.data(), num_values, compressed.data()); + ARROW_CHECK_OK(comp.status()); + const int64_t comp_size = *comp; + + std::vector decoded(num_values); + for (auto _ : state) { + auto status = FastLanesForCodec::Decode(decoded.data(), num_values, + compressed.data(), comp_size); + ARROW_CHECK_OK(status); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +// ============================================================================ +// Plain + ZSTD Encode/Decode +// ============================================================================ + +static void BM_PlainZstdEncode(benchmark::State& state, Gen32 gen) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + const uint8_t* raw = reinterpret_cast(values.data()); + + auto codec = *Codec::Create(Compression::ZSTD); + int64_t max_comp = codec->MaxCompressedLen(uncompressed_size, raw); + std::vector compressed(max_comp); + + // Compute comp_size once for the counter + int64_t comp_size = + *codec->Compress(uncompressed_size, raw, max_comp, compressed.data()); + + for (auto _ : state) { + auto sz = *codec->Compress(uncompressed_size, raw, max_comp, compressed.data()); + benchmark::DoNotOptimize(sz); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +static void BM_PlainZstdDecode(benchmark::State& state, Gen32 gen) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + const uint8_t* raw = reinterpret_cast(values.data()); + + auto codec = *Codec::Create(Compression::ZSTD); + int64_t max_comp = codec->MaxCompressedLen(uncompressed_size, raw); + std::vector compressed(max_comp); + int64_t comp_size = + *codec->Compress(uncompressed_size, raw, max_comp, compressed.data()); + + std::vector decompressed(uncompressed_size); + for (auto _ : state) { + auto result = codec->Decompress(comp_size, compressed.data(), uncompressed_size, + decompressed.data()); + ARROW_CHECK_OK(result.status()); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +// ============================================================================ +// Plain + LZ4 Encode/Decode +// ============================================================================ + +static void BM_PlainLz4Encode(benchmark::State& state, Gen32 gen) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + const uint8_t* raw = reinterpret_cast(values.data()); + + auto codec = *Codec::Create(Compression::LZ4_FRAME); + int64_t max_comp = codec->MaxCompressedLen(uncompressed_size, raw); + std::vector compressed(max_comp); + + int64_t comp_size = + *codec->Compress(uncompressed_size, raw, max_comp, compressed.data()); + + for (auto _ : state) { + auto sz = *codec->Compress(uncompressed_size, raw, max_comp, compressed.data()); + benchmark::DoNotOptimize(sz); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +static void BM_PlainLz4Decode(benchmark::State& state, Gen32 gen) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + const uint8_t* raw = reinterpret_cast(values.data()); + + auto codec = *Codec::Create(Compression::LZ4_FRAME); + int64_t max_comp = codec->MaxCompressedLen(uncompressed_size, raw); + std::vector compressed(max_comp); + int64_t comp_size = + *codec->Compress(uncompressed_size, raw, max_comp, compressed.data()); + + std::vector decompressed(uncompressed_size); + for (auto _ : state) { + auto result = codec->Decompress(comp_size, compressed.data(), uncompressed_size, + decompressed.data()); + ARROW_CHECK_OK(result.status()); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +// ============================================================================ +// RleBitPackHybrid Encode/Decode +// ============================================================================ + +static void BM_RleBitPackEncode(benchmark::State& state, Gen32 gen) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + int32_t bit_width = ComputeBitWidth(values); + int64_t max_buf = + ::arrow::util::RleBitPackedEncoder::MaxBufferSize(bit_width, num_values) + + ::arrow::util::RleBitPackedEncoder::MinBufferSize(bit_width); + std::vector buffer(max_buf); + + // Compute comp_size once for the counter + int64_t comp_size; + { + ::arrow::util::RleBitPackedEncoder enc(buffer.data(), + static_cast(max_buf), bit_width); + for (int64_t i = 0; i < num_values; ++i) { + enc.Put(static_cast(static_cast(values[i]))); + } + comp_size = enc.Flush(); + } + + for (auto _ : state) { + ::arrow::util::RleBitPackedEncoder encoder(buffer.data(), + static_cast(max_buf), bit_width); + for (int64_t i = 0; i < num_values; ++i) { + encoder.Put(static_cast(static_cast(values[i]))); + } + auto sz = encoder.Flush(); + benchmark::DoNotOptimize(sz); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +static void BM_RleBitPackDecode(benchmark::State& state, Gen32 gen) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + int32_t bit_width = ComputeBitWidth(values); + int64_t max_buf = + ::arrow::util::RleBitPackedEncoder::MaxBufferSize(bit_width, num_values) + + ::arrow::util::RleBitPackedEncoder::MinBufferSize(bit_width); + std::vector buffer(max_buf); + + ::arrow::util::RleBitPackedEncoder encoder(buffer.data(), + static_cast(max_buf), bit_width); + for (int64_t i = 0; i < num_values; ++i) { + encoder.Put(static_cast(static_cast(values[i]))); + } + int comp_size = encoder.Flush(); + + std::vector decoded(num_values); + for (auto _ : state) { + ::arrow::util::RleBitPackedParser parser(buffer.data(), comp_size, bit_width); + int64_t out_idx = 0; + struct Handler { + int32_t* output; + int64_t* idx; + int64_t max_values; + int32_t bw; + ::arrow::util::RleBitPackedParser::ControlFlow OnRleRun( + ::arrow::util::RleRun run) { + ::arrow::util::RleRunDecoder dec(run, bw); + auto want = static_cast( + std::min(static_cast(run.values_count()), max_values - *idx)); + auto count = dec.GetBatch(output + *idx, want, bw); + *idx += count; + return *idx >= max_values + ? ::arrow::util::RleBitPackedParser::ControlFlow::Break + : ::arrow::util::RleBitPackedParser::ControlFlow::Continue; + } + ::arrow::util::RleBitPackedParser::ControlFlow OnBitPackedRun( + ::arrow::util::BitPackedRun run) { + ::arrow::util::BitPackedRunDecoder dec(run, bw); + auto want = static_cast( + std::min(static_cast(dec.remaining()), max_values - *idx)); + auto count = dec.GetBatch(output + *idx, want, bw); + *idx += count; + return *idx >= max_values + ? ::arrow::util::RleBitPackedParser::ControlFlow::Break + : ::arrow::util::RleBitPackedParser::ControlFlow::Continue; + } + }; + Handler handler{decoded.data(), &out_idx, num_values, bit_width}; + parser.Parse(handler); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +// ============================================================================ +// ByteStreamSplit + Codec (ZSTD or LZ4) +// ============================================================================ + +static void BM_BssCodecEncode(benchmark::State& state, Gen32 gen, + Compression::type codec_type) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + auto descr = MakeInt32Descriptor(); + auto encoder = MakeTypedEncoder(Encoding::BYTE_STREAM_SPLIT, + /*use_dictionary=*/false, descr.get()); + auto codec = *Codec::Create(codec_type); + + encoder->Put(values.data(), static_cast(num_values)); + auto encoded_buf = encoder->FlushValues(); + int64_t encoded_size = encoded_buf->size(); + + int64_t max_comp = codec->MaxCompressedLen(encoded_size, encoded_buf->data()); + std::vector compressed(max_comp); + + // Compute comp_size once for the counter + int64_t comp_size = + *codec->Compress(encoded_size, encoded_buf->data(), max_comp, compressed.data()); + + for (auto _ : state) { + encoder->Put(values.data(), static_cast(num_values)); + auto buf = encoder->FlushValues(); + auto sz = + *codec->Compress(buf->size(), buf->data(), max_comp, compressed.data()); + benchmark::DoNotOptimize(sz); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +static void BM_BssCodecDecode(benchmark::State& state, Gen32 gen, + Compression::type codec_type) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + auto descr = MakeInt32Descriptor(); + auto encoder = MakeTypedEncoder(Encoding::BYTE_STREAM_SPLIT, + /*use_dictionary=*/false, descr.get()); + auto codec = *Codec::Create(codec_type); + + encoder->Put(values.data(), static_cast(num_values)); + auto encoded_buf = encoder->FlushValues(); + int64_t encoded_size = encoded_buf->size(); + + int64_t max_comp = codec->MaxCompressedLen(encoded_size, encoded_buf->data()); + std::vector compressed(max_comp); + int64_t comp_size = + *codec->Compress(encoded_size, encoded_buf->data(), max_comp, compressed.data()); + + std::vector decompressed(encoded_size); + std::vector decoded(num_values); + auto decoder = MakeTypedDecoder(Encoding::BYTE_STREAM_SPLIT, descr.get()); + + for (auto _ : state) { + auto result = codec->Decompress(comp_size, compressed.data(), encoded_size, + decompressed.data()); + ARROW_CHECK_OK(result.status()); + decoder->SetData(static_cast(num_values), decompressed.data(), + static_cast(encoded_size)); + decoder->Decode(decoded.data(), static_cast(num_values)); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +// Wrappers for BSS+ZSTD +static void BM_BssZstdEncode(benchmark::State& state, Gen32 gen) { + BM_BssCodecEncode(state, gen, Compression::ZSTD); +} +static void BM_BssZstdDecode(benchmark::State& state, Gen32 gen) { + BM_BssCodecDecode(state, gen, Compression::ZSTD); +} + +// Wrappers for BSS+LZ4 +static void BM_BssLz4Encode(benchmark::State& state, Gen32 gen) { + BM_BssCodecEncode(state, gen, Compression::LZ4_FRAME); +} +static void BM_BssLz4Decode(benchmark::State& state, Gen32 gen) { + BM_BssCodecDecode(state, gen, Compression::LZ4_FRAME); +} + +// ============================================================================ +// Benchmark Registration +// ============================================================================ + +static void CustomArgs(benchmark::internal::Benchmark* b) { b->Arg(102400); } + +// Macro to register all algorithms for a given dataset +#define REGISTER_DATASET(Name, GenFunc) \ + BENCHMARK_CAPTURE(BM_PforEncode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_PforDecode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_DeltaBitPackEncode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_DeltaBitPackDecode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_FastLanesEncode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_FastLanesDecode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_PlainZstdEncode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_PlainZstdDecode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_PlainLz4Encode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_PlainLz4Decode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_RleBitPackEncode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_RleBitPackDecode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_BssZstdEncode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_BssZstdDecode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_BssLz4Encode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_BssLz4Decode, Name, &GenFunc)->Apply(CustomArgs); + +// ClickBench datasets +REGISTER_DATASET(ClientIP, GenClientIP) +REGISTER_DATASET(UrlRegionID, GenUrlRegionID) +REGISTER_DATASET(CounterID, GenCounterID) +REGISTER_DATASET(EventDate, GenEventDate) +REGISTER_DATASET(EventTime, GenEventTime) +REGISTER_DATASET(GoodEvent, GenGoodEvent) +REGISTER_DATASET(HID, GenHID) +REGISTER_DATASET(HitColor, GenHitColor) +REGISTER_DATASET(IPNetworkID, GenIPNetworkID) +REGISTER_DATASET(JavaEnable, GenJavaEnable) +REGISTER_DATASET(OS, GenOS) +REGISTER_DATASET(Resolution, GenResolution) +REGISTER_DATASET(TrafficSourceID, GenTrafficSourceID) +REGISTER_DATASET(UserAgent, GenUserAgent) + +// TPC-DS datasets +REGISTER_DATASET(TpcdsSoldDateSk, GenTpcdsSoldDateSk) +REGISTER_DATASET(TpcdsStoreSk, GenTpcdsStoreSk) +REGISTER_DATASET(TpcdsItemSk, GenTpcdsItemSk) +REGISTER_DATASET(TpcdsQuantity, GenTpcdsQuantity) + +} // namespace +} // namespace parquet From ef3e746f66acf0f8301fffd83b46c9d692d4f1a0 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Sun, 28 Jun 2026 04:00:52 +0000 Subject: [PATCH 27/31] FastLanes-FOR: add DecodeFlat (flat output) and benchmark it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FastLanesForCodec::DecodeFlat unpacks into a transposed scratch buffer per chunk and then scatters via fromTransposed32 to produce output in original input order — output[i] == input[i] for the encoded input. This is the FL_ORDER inverse of the gather step in Encode. Adds: - DecodeFlat method + round-trip test (DecodeFlatIsIdentity) covering 4 chunks of random data. All 6 round-trip tests still pass. - BM_FastLanesDecodeFlat in pfor_comparison_benchmark, registered in the per-dataset macro for apples-to-apples vs PFOR / DeltaBitPack (both of which produce flat output). Bench (102400 int32, 3-run median, aarch64): Dataset FL Decode FL DecodeFlat PFOR Decode Delta Decode EventDate 20 us 108 us 38 us 123 us EventTime 23 us 113 us 57 us 140 us GoodEvent 20 us 107 us 35 us 119 us The transposed-kernel decode beats every other codec by 1.5-7x. The flat-output decode pays an ~85 us scatter cost per 100K values that makes it slower than PFOR but still faster than DeltaBitPack. The gap is exactly the FL_ORDER scatter — the reason FastLanes' intended architecture keeps data in transposed order through the query. --- cpp/src/arrow/util/fastlanes/fastlanes_for.cc | 56 +++++++++++++++++++ cpp/src/arrow/util/fastlanes/fastlanes_for.h | 7 +++ .../util/fastlanes/fastlanes_for_test.cc | 24 ++++++++ cpp/src/parquet/pfor_comparison_benchmark.cc | 31 ++++++++++ 4 files changed, 118 insertions(+) diff --git a/cpp/src/arrow/util/fastlanes/fastlanes_for.cc b/cpp/src/arrow/util/fastlanes/fastlanes_for.cc index b0e5f4d3e185..8ac078a6a162 100644 --- a/cpp/src/arrow/util/fastlanes/fastlanes_for.cc +++ b/cpp/src/arrow/util/fastlanes/fastlanes_for.cc @@ -184,6 +184,62 @@ Status FastLanesForCodec::Decode(int32_t* output, int64_t num_values, return Status::OK(); } +Status FastLanesForCodec::DecodeFlat(int32_t* output, int64_t num_values, + const uint8_t* input, int64_t input_size) { + if (num_values % kChunkSize != 0) { + return Status::Invalid("FastLanesForCodec::DecodeFlat: num_values (", + num_values, ") must be a multiple of ", kChunkSize); + } + const int64_t num_chunks = num_values / kChunkSize; + const uint8_t* in_ptr = input; + const uint8_t* in_end = input + input_size; + + alignas(64) uint32_t transposed[kChunkSize]; + + for (int64_t chunk = 0; chunk < num_chunks; ++chunk) { + if (in_ptr + kChunkHeaderSize > in_end) { + return Status::Invalid("FastLanesForCodec::DecodeFlat: truncated header"); + } + int32_t min_val; + std::memcpy(&min_val, in_ptr, sizeof(int32_t)); + const uint8_t bit_width = in_ptr[4]; + in_ptr += kChunkHeaderSize; + + int32_t* out_chunk = output + chunk * kChunkSize; + + if (bit_width == 0) { + for (int64_t i = 0; i < kChunkSize; ++i) out_chunk[i] = min_val; + continue; + } + + const int64_t bytes_per_block = 128 * bit_width; + if (in_ptr + 2 * bytes_per_block > in_end) { + return Status::Invalid("FastLanesForCodec::DecodeFlat: truncated body"); + } + + for (int64_t block = 0; block < 2; ++block) { + UnpackBlockDispatch( + bit_width, + reinterpret_cast(in_ptr + block * bytes_per_block), + transposed + block * kBlockSize); + } + in_ptr += 2 * bytes_per_block; + + // Scatter via fromTransposed32 per 1024-block to restore original + // input order: out_chunk[block*1024 + fromTransposed32(t)] = transposed + + // min. This is the FL_ORDER inverse of the encode-time gather. + for (int64_t block = 0; block < 2; ++block) { + const uint32_t* block_t = transposed + block * kBlockSize; + int32_t* block_out = out_chunk + block * static_cast(kBlockSize); + for (size_t t = 0; t < kBlockSize; ++t) { + block_out[fromTransposed32(t)] = static_cast(block_t[t]) + min_val; + } + } + } + + return Status::OK(); +} + } // namespace fastlanes } // namespace util } // namespace arrow diff --git a/cpp/src/arrow/util/fastlanes/fastlanes_for.h b/cpp/src/arrow/util/fastlanes/fastlanes_for.h index 4387aa27589a..55bcf412bca2 100644 --- a/cpp/src/arrow/util/fastlanes/fastlanes_for.h +++ b/cpp/src/arrow/util/fastlanes/fastlanes_for.h @@ -69,6 +69,13 @@ class FastLanesForCodec { // transposed order within each 1024-block (see header comment). static Status Decode(int32_t* output, int64_t num_values, const uint8_t* input, int64_t input_size); + + // Decode and scatter back to ORIGINAL input order: output[i] == input[i] + // for the encoded input. Pays the FL_ORDER scatter cost on top of the + // kernel — slower than Decode(), included for apples-to-apples + // comparison against codecs that produce flat output. + static Status DecodeFlat(int32_t* output, int64_t num_values, + const uint8_t* input, int64_t input_size); }; } // namespace fastlanes diff --git a/cpp/src/arrow/util/fastlanes/fastlanes_for_test.cc b/cpp/src/arrow/util/fastlanes/fastlanes_for_test.cc index 3cb6e2081650..0d65d10791a8 100644 --- a/cpp/src/arrow/util/fastlanes/fastlanes_for_test.cc +++ b/cpp/src/arrow/util/fastlanes/fastlanes_for_test.cc @@ -99,6 +99,30 @@ TEST(FastLanesForRoundTrip, BoundaryValues) { } } +// DecodeFlat produces output in ORIGINAL input order (output[i] == input[i]). +// Confirms the FL_ORDER scatter on decode inverts the FL_ORDER gather on +// encode. +TEST(FastLanesForRoundTrip, DecodeFlatIsIdentity) { + std::mt19937 rng(12345); + std::uniform_int_distribution dist(-1000, 1000); + std::vector in(4 * 2048); + for (auto& v : in) v = dist(rng); + + std::vector encoded(FastLanesForCodec::MaxEncodedSize(in.size())); + auto encoded_size = FastLanesForCodec::Encode(in.data(), in.size(), + encoded.data()); + ASSERT_TRUE(encoded_size.ok()); + + std::vector decoded(in.size(), 0xdeadbeef); + auto status = FastLanesForCodec::DecodeFlat(decoded.data(), in.size(), + encoded.data(), *encoded_size); + ASSERT_TRUE(status.ok()) << status.ToString(); + + for (size_t i = 0; i < in.size(); ++i) { + ASSERT_EQ(decoded[i], in[i]) << "i=" << i; + } +} + } // namespace fastlanes } // namespace util } // namespace arrow diff --git a/cpp/src/parquet/pfor_comparison_benchmark.cc b/cpp/src/parquet/pfor_comparison_benchmark.cc index 564efdd83df5..0c5bd5f83b10 100644 --- a/cpp/src/parquet/pfor_comparison_benchmark.cc +++ b/cpp/src/parquet/pfor_comparison_benchmark.cc @@ -431,6 +431,36 @@ static void BM_FastLanesDecode(benchmark::State& state, Gen32 gen) { static_cast(uncompressed_size) / static_cast(comp_size); } +// Decode + FL_ORDER scatter: produces flat output in original input +// order (apples-to-apples vs PFOR/DeltaBitPack which also produce flat). +static void BM_FastLanesDecodeFlat(benchmark::State& state, Gen32 gen) { + using ::arrow::util::fastlanes::FastLanesForCodec; + int64_t num_values = state.range(0); + num_values -= num_values % FastLanesForCodec::kChunkSize; + if (num_values == 0) num_values = FastLanesForCodec::kChunkSize; + + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + std::vector compressed(FastLanesForCodec::MaxEncodedSize(num_values)); + auto comp = FastLanesForCodec::Encode(values.data(), num_values, compressed.data()); + ARROW_CHECK_OK(comp.status()); + const int64_t comp_size = *comp; + + std::vector decoded(num_values); + for (auto _ : state) { + auto status = FastLanesForCodec::DecodeFlat(decoded.data(), num_values, + compressed.data(), comp_size); + ARROW_CHECK_OK(status); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + // ============================================================================ // Plain + ZSTD Encode/Decode // ============================================================================ @@ -753,6 +783,7 @@ static void CustomArgs(benchmark::internal::Benchmark* b) { b->Arg(102400); } BENCHMARK_CAPTURE(BM_DeltaBitPackDecode, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_FastLanesEncode, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_FastLanesDecode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_FastLanesDecodeFlat, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_PlainZstdEncode, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_PlainZstdDecode, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_PlainLz4Encode, Name, &GenFunc)->Apply(CustomArgs); \ From 4037bb251aa08352f76904fd456c9fcfc424f393 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Sun, 28 Jun 2026 04:37:04 +0000 Subject: [PATCH 28/31] FastLanes: add toTransposed32; correct fromTransposed32 docstring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 8x16 -> 16x8 within-sub-block transpose is mutual-inverse with the 16x8 -> 8x16 transpose, NOT self-inverse. The previous docstring on fromTransposed32 said "Self-inverse: fromTransposed32 is also toTransposed32" — that was wrong. fromTransposed32(fromTransposed32(t)) does not equal t in general; e.g. fromTransposed32(1) = 16, fromTransposed32(16) = 2. Add the actual toTransposed32 (forward-direction mapping) and fix the docstring. Callers that need to invert a gather computed with fromTransposed32 (i.e. read out[i] = transposed["the t whose fromTransposed32(t) = i"]) must use toTransposed32(i). --- .../arrow/util/fastlanes/fastlanes_kernels.h | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/util/fastlanes/fastlanes_kernels.h b/cpp/src/arrow/util/fastlanes/fastlanes_kernels.h index d0e09d3a850d..a8f5b3c76383 100644 --- a/cpp/src/arrow/util/fastlanes/fastlanes_kernels.h +++ b/cpp/src/arrow/util/fastlanes/fastlanes_kernels.h @@ -46,12 +46,27 @@ constexpr size_t kBlockSize = 1024; constexpr size_t kLanes = 32; // 1024 / sizeof(uint32_t) / 8 constexpr size_t kRowsPerBlock = 32; // 1024 / kLanes -// Sub-block reorder permutation (3-bit reversal). Used by fromTransposed32. +// Sub-block reorder permutation (3-bit reversal). Used by toTransposed32 / +// fromTransposed32. Self-inverse, but note that toTransposed32 and +// fromTransposed32 themselves are NOT self-inverse — they are mutual +// inverses (the 8x16 -> 16x8 within-sub-block transpose is not involutive). inline constexpr size_t kBlockReorder[8] = {0, 4, 2, 6, 1, 5, 3, 7}; +// Convert an original (flat) input index to its position in the transposed +// (stream) order within a 1024-value block. +inline size_t toTransposed32(size_t origIdx) { + const size_t block = origIdx >> 7; + const size_t withinBlock = origIdx & 0x7F; + const size_t row = withinBlock >> 4; // row in 8x16 + const size_t col = withinBlock & 0xF; // col in 8x16 + const size_t transposedWithin = (col << 3) | row; // col * 8 + row (16x8) + const size_t outputBlock = kBlockReorder[block]; + return (outputBlock << 7) | transposedWithin; +} + // Convert a transposed (stream) index back to its original input index -// within a 1024-value block. Self-inverse: fromTransposed32 is also -// toTransposed32. +// within a 1024-value block. Inverse of toTransposed32 (the 8x16 -> 16x8 +// transpose is mutual-inverse with the 16x8 -> 8x16 transpose). inline size_t fromTransposed32(size_t t) { const size_t outputBlock = t >> 7; // t / 128 const size_t transposedWithin = t & 0x7F; // t % 128 From 26ea3c40b7f686a5b0b7120b1590bec727b51e52 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Sun, 28 Jun 2026 04:37:21 +0000 Subject: [PATCH 29/31] PFOR: add PackingMode (BitPack | FastLanes) per-vector flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an additive packing-mode option to PFOR. Existing vectors round-trip unchanged (default PackingMode::BitPack); new vectors can opt in to the FastLanes lane-interleaved bit-packing layout via the per-vector flag. On-disk format change (backwards-compatible): - The 1-byte bit_width field of PforVectorInfo now packs two values: bits 0..5 = the actual bit width (range 0..32 fits in 6 bits) bit 7 = packing-mode flag (0 = BitPack, 1 = FastLanes) bit 6 = reserved - Legacy encoders only wrote the bit width, leaving high bits clear, so they decode as PackingMode::BitPack via the new Load. - PFOR header (page-level) is unchanged. API: - New enum class arrow::util::pfor::PackingMode { BitPack, FastLanes }. - PforVectorInfo gains a packing_mode field and getter/setter. - PforCompression::EncodeVector takes an optional PackingMode (default BitPack). FastLanes mode is only honored when num_elements equals the FastLanes block size (1024) and T is 32-bit; otherwise it falls back to BitPack per-vector (so tails and 64-bit values continue to work). - PforCompression::DecodeVector reads the per-vector flag and dispatches between arrow::internal::unpack and the FastLanes kernel. - PforWrapper::Encode takes an optional PackingMode threaded down to EncodeVector. Decode-side perf (fused gather + FOR-add + SafeCopy): The FL_ORDER inverse needs toTransposed32(i) — note: NOT fromTransposed32(i), the two are mutual inverses, not self-inverse. The scalar gather over can't be SIMD-vectorized, so PFOR+FastLanes decode is ~2-3x slower than PFOR+BitPack end-to-end despite the kernel itself being competitive. The win is only available when the downstream consumer can work with data in FastLanes transposed order (i.e. relax the flat-output contract). Tests: 5 new tests in PforPackingModeTest cover round-trip identity for both modes, the partial-tail fallback to BitPack, mixed-mode round-trip through PforWrapper, and the bit_width=0 (constant vector) path. All 30 PFOR tests pass. Benchmark: BM_PforFastLanesEncode / BM_PforFastLanesDecode added to pfor_comparison_benchmark.cc, registered per dataset alongside the existing 8 codec variants. --- cpp/src/arrow/util/pfor/pfor.cc | 123 ++++++++++++++++--- cpp/src/arrow/util/pfor/pfor.h | 36 +++++- cpp/src/arrow/util/pfor/pfor_constants.h | 16 +++ cpp/src/arrow/util/pfor/pfor_test.cc | 120 ++++++++++++++++++ cpp/src/arrow/util/pfor/pfor_wrapper.cc | 11 +- cpp/src/arrow/util/pfor/pfor_wrapper.h | 9 +- cpp/src/parquet/pfor_comparison_benchmark.cc | 70 +++++++++++ 7 files changed, 356 insertions(+), 29 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc index 7810ad8d4089..ba602dcc2716 100644 --- a/cpp/src/arrow/util/pfor/pfor.cc +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -34,6 +34,7 @@ #include "arrow/util/bit_util.h" #include "arrow/util/bpacking_internal.h" #include "arrow/util/endian.h" +#include "arrow/util/fastlanes/fastlanes_kernels.h" #include "arrow/util/logging.h" #include "arrow/util/macros.h" #include "arrow/util/span.h" @@ -91,12 +92,56 @@ BitWidthResult PforCompression::FindOptimalBitWidth(const UnsignedT* deltas, return {best_bit_width, best_num_exceptions}; } +namespace { + +// Dispatch by runtime bit_width into the templated FastLanes pack/unpack +// kernels. Only used when use_fastlanes && num_elements == kPforVectorSize +// (the FastLanes block size matches PFOR's vector size of 1024). +inline void FastLanesPackBlockDispatch(uint8_t bit_width, const uint32_t* in, + uint32_t* out) { + switch (bit_width) { +#define CASE_W(N) \ + case N: \ + fastlanes::PackBlock(in, out); \ + break + CASE_W(1); CASE_W(2); CASE_W(3); CASE_W(4); CASE_W(5); CASE_W(6); + CASE_W(7); CASE_W(8); CASE_W(9); CASE_W(10); CASE_W(11); CASE_W(12); + CASE_W(13); CASE_W(14); CASE_W(15); CASE_W(16); CASE_W(17); CASE_W(18); + CASE_W(19); CASE_W(20); CASE_W(21); CASE_W(22); CASE_W(23); CASE_W(24); + CASE_W(25); CASE_W(26); CASE_W(27); CASE_W(28); CASE_W(29); CASE_W(30); + CASE_W(31); CASE_W(32); +#undef CASE_W + default: break; + } +} + +inline void FastLanesUnpackBlockDispatch(uint8_t bit_width, const uint32_t* packed, + uint32_t* out) { + switch (bit_width) { +#define CASE_W(N) \ + case N: \ + fastlanes::UnpackBlock(packed, out); \ + break + CASE_W(1); CASE_W(2); CASE_W(3); CASE_W(4); CASE_W(5); CASE_W(6); + CASE_W(7); CASE_W(8); CASE_W(9); CASE_W(10); CASE_W(11); CASE_W(12); + CASE_W(13); CASE_W(14); CASE_W(15); CASE_W(16); CASE_W(17); CASE_W(18); + CASE_W(19); CASE_W(20); CASE_W(21); CASE_W(22); CASE_W(23); CASE_W(24); + CASE_W(25); CASE_W(26); CASE_W(27); CASE_W(28); CASE_W(29); CASE_W(30); + CASE_W(31); CASE_W(32); +#undef CASE_W + default: break; + } +} + +} // namespace + // ---------------------------------------------------------------------- // EncodeVector template PforEncodedVector PforCompression::EncodeVector(const T* values, - int32_t num_elements) { + int32_t num_elements, + PackingMode mode) { ARROW_DCHECK(num_elements > 0); // Step 1: Find min (frame of reference) @@ -116,9 +161,20 @@ PforEncodedVector PforCompression::EncodeVector(const T* values, auto [bit_width, num_exceptions] = FindOptimalBitWidth(deltas.data(), num_elements); + // FastLanes only applies when the vector matches the FastLanes block size + // (1024) and the type is 32-bit. For shorter tails or 64-bit values, fall + // back to PackingMode::BitPack. + const PackingMode effective_mode = + (mode == PackingMode::FastLanes && + num_elements == static_cast(fastlanes::kBlockSize) && + sizeof(T) == 4) + ? PackingMode::FastLanes + : PackingMode::BitPack; + // Step 4: Collect exceptions and replace with placeholder (0) PforEncodedVector result; - result.set_info(PforVectorInfo(min_val, bit_width, num_exceptions)); + result.set_info( + PforVectorInfo(min_val, bit_width, num_exceptions, effective_mode)); if (num_exceptions > 0) { result.mutable_exception_positions().reserve(num_exceptions); @@ -143,12 +199,25 @@ PforEncodedVector PforCompression::EncodeVector(const T* values, bit_util::BytesForBits(static_cast(num_elements) * bit_width); result.mutable_packed_values().resize(static_cast(packed_size), 0); - bit_util::BitWriter writer(result.mutable_packed_values().data(), - static_cast(packed_size)); - for (int32_t i = 0; i < num_elements; ++i) { - writer.PutValue(static_cast(deltas[i]), bit_width); + if (effective_mode == PackingMode::FastLanes) { + // Gather deltas via FL_ORDER into transposed scratch, then pack with + // FastLanes' lane-interleaved kernel. The packed payload is exactly + // 128 * bit_width bytes — same as the BitPack-encoded size. + alignas(64) uint32_t transposed[fastlanes::kBlockSize]; + for (size_t t = 0; t < fastlanes::kBlockSize; ++t) { + transposed[t] = static_cast(deltas[fastlanes::fromTransposed32(t)]); + } + FastLanesPackBlockDispatch( + bit_width, transposed, + reinterpret_cast(result.mutable_packed_values().data())); + } else { + bit_util::BitWriter writer(result.mutable_packed_values().data(), + static_cast(packed_size)); + for (int32_t i = 0; i < num_elements; ++i) { + writer.PutValue(static_cast(deltas[i]), bit_width); + } + writer.Flush(); } - writer.Flush(); } return result; @@ -173,20 +242,40 @@ Result PforCompression::DecodeVector(T* values, // Step 3: Unpack bit-packed deltas and add FOR if (info.bit_width() > 0) { - std::vector unsigned_values(num_elements); const auto unsigned_for = static_cast(info.frame_of_reference()); - // Arrow's unpack handles arbitrary sizes: SIMD for complete batches, - // then unpack_exact for the remainder. - arrow::internal::unpack(read_ptr, unsigned_values.data(), - static_cast(num_elements), info.bit_width()); - - // Add FOR and convert to signed output via SafeCopy + if (info.packing_mode() == PackingMode::FastLanes) { + // FastLanes-packed payload: 128 * bit_width bytes per 1024-block. + // Unpack into transposed scratch then fuse the FL_ORDER inverse + // (gather via toTransposed32 — note: NOT fromTransposed32, the two + // are mutual inverses, not self-inverse), FOR-add, and SafeCopy + // into one sequential write pass over `values`. Saves the + // intermediate unsigned_values buffer entirely. + ARROW_DCHECK(num_elements == + static_cast(fastlanes::kBlockSize)); + alignas(64) uint32_t transposed[fastlanes::kBlockSize]; + FastLanesUnpackBlockDispatch( + info.bit_width(), + reinterpret_cast(read_ptr), transposed); + for (size_t i = 0; i < fastlanes::kBlockSize; ++i) { + const UnsignedT v = + static_cast(transposed[fastlanes::toTransposed32(i)]); + values[i] = util::SafeCopy(v + unsigned_for); + } + } else { + std::vector unsigned_values(num_elements); + // Arrow's unpack handles arbitrary sizes: SIMD for complete batches, + // then unpack_exact for the remainder. + arrow::internal::unpack(read_ptr, unsigned_values.data(), + static_cast(num_elements), info.bit_width()); + + // Add FOR and convert to signed output via SafeCopy #pragma GCC unroll PforConstants::kLoopUnrolls #pragma GCC ivdep - for (int32_t i = 0; i < num_elements; ++i) { - unsigned_values[i] += unsigned_for; - values[i] = util::SafeCopy(unsigned_values[i]); + for (int32_t i = 0; i < num_elements; ++i) { + unsigned_values[i] += unsigned_for; + values[i] = util::SafeCopy(unsigned_values[i]); + } } int64_t packed_size = diff --git a/cpp/src/arrow/util/pfor/pfor.h b/cpp/src/arrow/util/pfor/pfor.h index ec0ee2a2f1cd..bbb56781696b 100644 --- a/cpp/src/arrow/util/pfor/pfor.h +++ b/cpp/src/arrow/util/pfor/pfor.h @@ -46,30 +46,46 @@ namespace pfor { /// /// For INT32 (7 bytes): [frame_of_reference(4B)] [bit_width(1B)] [num_exceptions(2B)] /// For INT64 (11 bytes): [frame_of_reference(8B)] [bit_width(1B)] [num_exceptions(2B)] +/// +/// The bit_width byte packs two fields: +/// bits 0..5 — the actual bit width (range 0..32, fits in 6 bits) +/// bit 7 — packing-mode flag (0 = PackingMode::BitPack, 1 = PackingMode::FastLanes) +/// bit 6 — reserved (zero) +/// Legacy encoders (which only wrote the bit width) produce vectors with the +/// high bits clear, so they round-trip through the new Load as BitPack. template class PforVectorInfo { public: + static constexpr uint8_t kPackingModeFlagMask = 0x80; + static constexpr uint8_t kBitWidthMask = 0x3F; + PforVectorInfo() = default; - PforVectorInfo(T frame_of_reference, uint8_t bit_width, int16_t num_exceptions) + PforVectorInfo(T frame_of_reference, uint8_t bit_width, int16_t num_exceptions, + PackingMode packing_mode = PackingMode::BitPack) : frame_of_reference_(frame_of_reference), bit_width_(bit_width), - num_exceptions_(num_exceptions) {} + num_exceptions_(num_exceptions), + packing_mode_(packing_mode) {} T frame_of_reference() const { return frame_of_reference_; } uint8_t bit_width() const { return bit_width_; } int16_t num_exceptions() const { return num_exceptions_; } + PackingMode packing_mode() const { return packing_mode_; } void set_frame_of_reference(T frame_of_reference) { frame_of_reference_ = frame_of_reference; } void set_bit_width(uint8_t bit_width) { bit_width_ = bit_width; } void set_num_exceptions(int16_t num_exceptions) { num_exceptions_ = num_exceptions; } + void set_packing_mode(PackingMode mode) { packing_mode_ = mode; } /// \brief Store this info to a byte buffer (little-endian) void Store(arrow::util::span dest) const { uint8_t* ptr = dest.data(); util::SafeStore(ptr, frame_of_reference_); - ptr[sizeof(T)] = bit_width_; + const uint8_t mode_bit = + (packing_mode_ == PackingMode::FastLanes) ? kPackingModeFlagMask : 0; + ptr[sizeof(T)] = static_cast((bit_width_ & kBitWidthMask) | mode_bit); util::SafeStore(ptr + sizeof(T) + 1, num_exceptions_); } @@ -82,7 +98,11 @@ class PforVectorInfo { PforVectorInfo info; const uint8_t* ptr = src.data(); info.frame_of_reference_ = util::SafeLoadAs(ptr); - info.bit_width_ = ptr[sizeof(T)]; + const uint8_t packed_bw = ptr[sizeof(T)]; + info.bit_width_ = static_cast(packed_bw & kBitWidthMask); + info.packing_mode_ = (packed_bw & kPackingModeFlagMask) != 0 + ? PackingMode::FastLanes + : PackingMode::BitPack; info.num_exceptions_ = util::SafeLoadAs(ptr + sizeof(T) + 1); if (info.bit_width_ > PforTypeTraits::kMaxBitWidth) { return Status::Invalid("PFOR bit_width out of range: ", @@ -102,6 +122,7 @@ class PforVectorInfo { T frame_of_reference_ = 0; uint8_t bit_width_ = 0; int16_t num_exceptions_ = 0; + PackingMode packing_mode_ = PackingMode::BitPack; }; // ---------------------------------------------------------------------- @@ -221,8 +242,13 @@ class PforCompression { /// /// \param[in] values input integer values /// \param[in] num_elements number of elements (up to vector_size) + /// \param[in] mode bit-packing layout for the payload. PackingMode::FastLanes + /// requires num_elements == kPforVectorSize (1024); otherwise + /// falls back to PackingMode::BitPack for this vector. 64-bit + /// types always fall back to BitPack (FastLanes is u32-only). /// \return the encoded vector with all sections - static PforEncodedVector EncodeVector(const T* values, int32_t num_elements); + static PforEncodedVector EncodeVector(const T* values, int32_t num_elements, + PackingMode mode = PackingMode::BitPack); /// \brief Decode a single vector from compressed data /// diff --git a/cpp/src/arrow/util/pfor/pfor_constants.h b/cpp/src/arrow/util/pfor/pfor_constants.h index 18e63b962e4c..5004591aa384 100644 --- a/cpp/src/arrow/util/pfor/pfor_constants.h +++ b/cpp/src/arrow/util/pfor/pfor_constants.h @@ -57,6 +57,22 @@ class PforConstants { static constexpr int64_t kLoopUnrolls = 4; }; +/// \brief Per-vector packing mode for PforCompression::EncodeVector. +/// +/// Stored in the high bit of PforVectorInfo::bit_width (bit 7). BitPack = 0 +/// preserves the prior on-disk layout so existing buffers continue to +/// round-trip; new values are added with higher numeric codes. +enum class PackingMode : uint8_t { + /// Sequential little-endian bit-packed stream — the original PFOR layout. + /// Decoded with arrow::internal::unpack. + BitPack = 0, + + /// FastLanes lane-interleaved 1024-bit format (Afroozeh & Boncz, VLDB '23). + /// Auto-vectorizable kernel; only valid when num_elements equals the + /// FastLanes block size (1024). Falls back to BitPack for shorter vectors. + FastLanes = 1, +}; + /// \brief Type traits for PFOR integer types template struct PforTypeTraits {}; diff --git a/cpp/src/arrow/util/pfor/pfor_test.cc b/cpp/src/arrow/util/pfor/pfor_test.cc index cbfe23c354c9..8b4398f8ad11 100644 --- a/cpp/src/arrow/util/pfor/pfor_test.cc +++ b/cpp/src/arrow/util/pfor/pfor_test.cc @@ -432,4 +432,124 @@ TEST(PforCompressionRatioTest, ClusteredDataCompresses) { EXPECT_LT(comp_size, plain_size / 2); } +// ============================================================================ +// FastLanes packing-mode tests +// ============================================================================ +// +// All round-trips must produce flat output identical to the input — the +// FastLanes scatter on decode is the inverse of the gather on encode, so the +// PFOR contract (flat output, no permutation visible to the caller) holds. + +TEST(PforPackingModeTest, VectorRoundTripIdentityForBothModes) { + std::vector values(1024); + std::mt19937 rng(7); + std::uniform_int_distribution dist(1000, 1500); + for (auto& v : values) v = dist(rng); + values[123] = 99999; // outlier — exercises the exception path + values[800] = -50; // another, below FOR + + for (PackingMode mode : {PackingMode::BitPack, PackingMode::FastLanes}) { + SCOPED_TRACE(testing::Message() << "mode=" << static_cast(mode)); + auto encoded = + PforCompression::EncodeVector(values.data(), 1024, mode); + EXPECT_EQ(encoded.info().packing_mode(), mode); + + int64_t sz = PforCompression::SerializedVectorSize(encoded, 1024); + std::vector buffer(sz); + PforCompression::SerializeVector(encoded, 1024, buffer); + + std::vector decoded(1024); + ASSERT_OK(PforCompression::DecodeVector(decoded.data(), buffer, 1024)); + + EXPECT_EQ(values, decoded); + } +} + +TEST(PforPackingModeTest, FastLanesFallsBackOnPartialTail) { + // 700-element vector — smaller than the FastLanes block size. Encoder + // must fall back to BitPack regardless of the requested mode so the + // legacy bit-packer handles the partial tail. + std::vector values(700); + std::mt19937 rng(99); + std::uniform_int_distribution dist(0, 100); + for (auto& v : values) v = dist(rng); + + auto encoded = + PforCompression::EncodeVector(values.data(), 700, PackingMode::FastLanes); + EXPECT_EQ(encoded.info().packing_mode(), PackingMode::BitPack); + + int64_t sz = PforCompression::SerializedVectorSize(encoded, 700); + std::vector buffer(sz); + PforCompression::SerializeVector(encoded, 700, buffer); + + std::vector decoded(700); + ASSERT_OK(PforCompression::DecodeVector(decoded.data(), buffer, 700)); + EXPECT_EQ(values, decoded); +} + +TEST(PforPackingModeTest, WrapperRoundTripFastLanes) { + // PforWrapper covers multiple vectors plus the offset array. + constexpr int32_t kN = 5 * 1024; // 5 full FastLanes vectors + std::vector values(kN); + std::mt19937 rng(2026); + std::uniform_int_distribution dist(-100, 100); + for (auto& v : values) v = dist(rng); + + int64_t max_size = PforWrapper::GetMaxCompressedSize(kN); + std::vector compressed(max_size); + int64_t comp_size = max_size; + + PforWrapper::Encode(values.data(), kN, compressed.data(), &comp_size, + PackingMode::FastLanes); + + std::vector decoded(kN); + ASSERT_OK(PforWrapper::Decode(decoded.data(), kN, compressed.data(), + comp_size)); + EXPECT_EQ(values, decoded); +} + +TEST(PforPackingModeTest, WrapperRoundTripMixedTail) { + // 5*1024 + 700 elements: 5 full vectors (FastLanes-packed when requested) + // + 1 tail vector (falls back to BitPack inside EncodeVector). Both modes + // round-trip via the per-vector flag. + constexpr int32_t kN = 5 * 1024 + 700; + std::vector values(kN); + std::mt19937 rng(123); + std::uniform_int_distribution dist(50000, 60000); + for (auto& v : values) v = dist(rng); + + int64_t max_size = PforWrapper::GetMaxCompressedSize(kN); + std::vector compressed(max_size); + int64_t comp_size = max_size; + + PforWrapper::Encode(values.data(), kN, compressed.data(), &comp_size, + PackingMode::FastLanes); + + std::vector decoded(kN); + ASSERT_OK(PforWrapper::Decode(decoded.data(), kN, compressed.data(), + comp_size)); + EXPECT_EQ(values, decoded); +} + +TEST(PforPackingModeTest, ConstantVectorBothModes) { + // bit_width = 0 path: FOR captures everything, no packed payload. Should + // round-trip identically under either mode (the FastLanes kernel is never + // entered because bit_width == 0). + std::vector values(1024, 42); + for (PackingMode mode : {PackingMode::BitPack, PackingMode::FastLanes}) { + SCOPED_TRACE(testing::Message() << "mode=" << static_cast(mode)); + auto encoded = + PforCompression::EncodeVector(values.data(), 1024, mode); + EXPECT_EQ(encoded.info().bit_width(), 0); + + int64_t sz = PforCompression::SerializedVectorSize(encoded, 1024); + std::vector buffer(sz); + PforCompression::SerializeVector(encoded, 1024, buffer); + + std::vector decoded(1024); + ASSERT_OK(PforCompression::DecodeVector(decoded.data(), buffer, 1024)); + EXPECT_EQ(values, decoded); + } +} + } // namespace arrow::util::pfor diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.cc b/cpp/src/arrow/util/pfor/pfor_wrapper.cc index c132f3cc1500..b653f0894297 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.cc +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.cc @@ -90,7 +90,7 @@ Result::PforHeader> PforWrapper::LoadHeader( template void PforWrapper::Encode(const T* values, int32_t num_values, int32_t vector_size, - uint8_t* comp, int64_t* comp_size) { + uint8_t* comp, int64_t* comp_size, PackingMode mode) { ARROW_DCHECK(num_values > 0); ARROW_DCHECK(comp != nullptr); ARROW_DCHECK(comp_size != nullptr); @@ -131,9 +131,10 @@ void PforWrapper::Encode(const T* values, int32_t num_values, int32_t vector_ int32_t elements_in_vector = std::min(vector_size, num_values - start_idx); - // Encode vector + // Encode vector (EncodeVector itself falls back to BitPack for partial + // tails or 64-bit T, so it's safe to pass `mode` unconditionally). auto encoded = PforCompression::EncodeVector( - values + start_idx, elements_in_vector); + values + start_idx, elements_in_vector, mode); // Serialize to output int64_t bytes_written = PforCompression::SerializeVector( @@ -147,8 +148,8 @@ void PforWrapper::Encode(const T* values, int32_t num_values, int32_t vector_ template void PforWrapper::Encode(const T* values, int32_t num_values, uint8_t* comp, - int64_t* comp_size) { - Encode(values, num_values, kVectorSize, comp, comp_size); + int64_t* comp_size, PackingMode mode) { + Encode(values, num_values, kVectorSize, comp, comp_size, mode); } // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.h b/cpp/src/arrow/util/pfor/pfor_wrapper.h index 727b942109a3..842a3df2d141 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.h +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.h @@ -50,12 +50,17 @@ class PforWrapper { /// in [2^kMinLogVectorSize, 2^kMaxLogVectorSize]) /// \param[out] comp pointer to output buffer (caller must ensure sufficient size) /// \param[in,out] comp_size input: available buffer size; output: bytes written + /// \param[in] mode per-vector bit-packing layout. PackingMode::FastLanes is + /// applied only to full-size 32-bit vectors; tails and 64-bit + /// values fall back to PackingMode::BitPack per vector. static void Encode(const T* values, int32_t num_values, int32_t vector_size, - uint8_t* comp, int64_t* comp_size); + uint8_t* comp, int64_t* comp_size, + PackingMode mode = PackingMode::BitPack); /// Convenience overload with default vector_size = kPforVectorSize static void Encode(const T* values, int32_t num_values, uint8_t* comp, - int64_t* comp_size); + int64_t* comp_size, + PackingMode mode = PackingMode::BitPack); /// \brief Decode a PFOR-compressed page /// diff --git a/cpp/src/parquet/pfor_comparison_benchmark.cc b/cpp/src/parquet/pfor_comparison_benchmark.cc index 0c5bd5f83b10..5fbe70db0f05 100644 --- a/cpp/src/parquet/pfor_comparison_benchmark.cc +++ b/cpp/src/parquet/pfor_comparison_benchmark.cc @@ -313,6 +313,74 @@ static void BM_PforDecode(benchmark::State& state, Gen32 gen) { static_cast(uncompressed_size) / static_cast(comp_size); } +// ============================================================================ +// PFOR with FastLanes-mode bit packing (per-vector flag in PFOR header) +// ============================================================================ +// +// Same PFOR pipeline (FOR + exceptions) but the bit-packing payload uses +// FastLanes lane-interleaved layout instead of the legacy sequential bit +// stream. Output is still flat (PFOR contract preserved); the decoder +// scatters via FL_ORDER inside DecodeVector. + +static void BM_PforFastLanesEncode(benchmark::State& state, Gen32 gen) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + int64_t max_size = + ::arrow::util::pfor::PforWrapper::GetMaxCompressedSize( + static_cast(num_values)); + std::vector compressed(max_size); + + int64_t comp_size = max_size; + ::arrow::util::pfor::PforWrapper::Encode( + values.data(), static_cast(num_values), compressed.data(), &comp_size, + ::arrow::util::pfor::PackingMode::FastLanes); + + for (auto _ : state) { + int64_t sz = max_size; + ::arrow::util::pfor::PforWrapper::Encode( + values.data(), static_cast(num_values), compressed.data(), &sz, + ::arrow::util::pfor::PackingMode::FastLanes); + benchmark::DoNotOptimize(sz); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + +static void BM_PforFastLanesDecode(benchmark::State& state, Gen32 gen) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + int64_t max_size = + ::arrow::util::pfor::PforWrapper::GetMaxCompressedSize( + static_cast(num_values)); + std::vector compressed(max_size); + int64_t comp_size = max_size; + ::arrow::util::pfor::PforWrapper::Encode( + values.data(), static_cast(num_values), compressed.data(), + &comp_size, ::arrow::util::pfor::PackingMode::FastLanes); + + std::vector decoded(num_values); + for (auto _ : state) { + auto status = ::arrow::util::pfor::PforWrapper::Decode( + decoded.data(), static_cast(num_values), compressed.data(), + comp_size); + ARROW_CHECK_OK(status); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + // ============================================================================ // DeltaBitPack Encode/Decode // ============================================================================ @@ -779,6 +847,8 @@ static void CustomArgs(benchmark::internal::Benchmark* b) { b->Arg(102400); } #define REGISTER_DATASET(Name, GenFunc) \ BENCHMARK_CAPTURE(BM_PforEncode, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_PforDecode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_PforFastLanesEncode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_PforFastLanesDecode, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_DeltaBitPackEncode, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_DeltaBitPackDecode, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_FastLanesEncode, Name, &GenFunc)->Apply(CustomArgs); \ From 51f25685eecec48fa249e8defcf5353781e4f738 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Sun, 28 Jun 2026 13:58:26 +0000 Subject: [PATCH 30/31] PFOR: add OutputOrder { Flat, Transposed } decode option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For FastLanes-encoded vectors the decoder previously always paid a 1024-element scalar FL_ORDER gather to produce flat output. That gather is what made pfor+fastlanes 2-3x slower than pfor+bitpack overall, even though the FastLanes unpack kernel itself is competitive. The FastLanes paper's intended decode path is to NOT do that scatter at all: keep the data in FastLanes stream order and let downstream operators be permutation-aware (apply fromTransposed32 lazily, when they need original index). This commit exposes that path. API: - New enum class arrow::util::pfor::OutputOrder { Flat, Transposed }. - PforCompression::DecodeVector and PforWrapper::Decode take an optional OutputOrder (default Flat, backwards-compatible). - OutputOrder::Transposed only affects FastLanes-encoded vectors. BitPack vectors have no permutation to skip, so they always produce flat output regardless of the argument (mixed pages with a BitPack tail end up flat in the tail, transposed in the full blocks). Decoder paths in DecodeVector when packing_mode == FastLanes: - Flat (existing): unpack -> scratch transposed[] -> fused values[i] = SafeCopy(transposed[toTransposed32(i)] + FOR) The toTransposed32 gather is scalar, breaks auto-vec. - Transposed (new): unpack -> scratch transposed[] -> sequential values[t] = SafeCopy(transposed[t] + FOR) Pure sequential read/write, auto-vectorizes cleanly. Exceptions are patched at toTransposed32(pos) so the stored-flat positions land in the right transposed slots. Tests: 4 new tests in PforOutputOrderTest cover (a) transposed output satisfies the FL_ORDER relation, (b) manual inversion of the permutation reconstructs the input, (c) BitPack vectors ignore the Transposed request, (d) wrapper-level transposed decode across many vectors. All 34 PFOR tests pass. Benchmark: BM_PforFastLanesDecodeTransposed added, registered per dataset. On 18 ClickBench-style datasets (102400 int32 each): pfor+bitpack 33-57 us pfor+fastlanes (flat) 98-108 us (0.34-0.53x — slower) pfor+fastlanes (transp) 20-23 us (1.6-2.5x faster than bitpack) The transposed path beats every other codec measured in the comparison benchmark on every dataset. --- cpp/src/arrow/util/pfor/pfor.cc | 47 ++++++-- cpp/src/arrow/util/pfor/pfor.h | 8 +- cpp/src/arrow/util/pfor/pfor_constants.h | 20 ++++ cpp/src/arrow/util/pfor/pfor_test.cc | 110 +++++++++++++++++++ cpp/src/arrow/util/pfor/pfor_wrapper.cc | 4 +- cpp/src/arrow/util/pfor/pfor_wrapper.h | 10 +- cpp/src/parquet/pfor_comparison_benchmark.cc | 35 ++++++ 7 files changed, 218 insertions(+), 16 deletions(-) diff --git a/cpp/src/arrow/util/pfor/pfor.cc b/cpp/src/arrow/util/pfor/pfor.cc index ba602dcc2716..0a185d3512e3 100644 --- a/cpp/src/arrow/util/pfor/pfor.cc +++ b/cpp/src/arrow/util/pfor/pfor.cc @@ -229,11 +229,19 @@ PforEncodedVector PforCompression::EncodeVector(const T* values, template Result PforCompression::DecodeVector(T* values, arrow::util::span data, - int32_t num_elements) { + int32_t num_elements, + OutputOrder order) { // Step 1: Read vector info ARROW_ASSIGN_OR_RAISE(auto info, PforVectorInfo::Load(data)); const uint8_t* read_ptr = data.data() + PforVectorInfo::kStoredSize; + // OutputOrder::Transposed is only meaningful for FastLanes-encoded vectors. + // BitPack vectors have no transposition to skip, so they always emit flat + // output regardless of `order`. + const bool emit_transposed = + (order == OutputOrder::Transposed) && + (info.packing_mode() == PackingMode::FastLanes); + // Step 2: Handle constant data (bit_width == 0, no exceptions) if (info.bit_width() == 0 && info.num_exceptions() == 0) { std::fill(values, values + num_elements, info.frame_of_reference()); @@ -246,21 +254,32 @@ Result PforCompression::DecodeVector(T* values, if (info.packing_mode() == PackingMode::FastLanes) { // FastLanes-packed payload: 128 * bit_width bytes per 1024-block. - // Unpack into transposed scratch then fuse the FL_ORDER inverse - // (gather via toTransposed32 — note: NOT fromTransposed32, the two - // are mutual inverses, not self-inverse), FOR-add, and SafeCopy - // into one sequential write pass over `values`. Saves the - // intermediate unsigned_values buffer entirely. + // Unpack into transposed scratch. ARROW_DCHECK(num_elements == static_cast(fastlanes::kBlockSize)); alignas(64) uint32_t transposed[fastlanes::kBlockSize]; FastLanesUnpackBlockDispatch( info.bit_width(), reinterpret_cast(read_ptr), transposed); - for (size_t i = 0; i < fastlanes::kBlockSize; ++i) { - const UnsignedT v = - static_cast(transposed[fastlanes::toTransposed32(i)]); - values[i] = util::SafeCopy(v + unsigned_for); + + if (emit_transposed) { + // No FL_ORDER inverse: write `values[t] = transposed[t] + FOR` + // sequentially. The output is in FastLanes stream order, i.e. + // values[t] corresponds to the original input at fromTransposed32(t). + // Sequential read + sequential write is auto-vec friendly. + for (size_t t = 0; t < fastlanes::kBlockSize; ++t) { + values[t] = util::SafeCopy( + static_cast(transposed[t]) + unsigned_for); + } + } else { + // Fused FL_ORDER inverse + FOR-add + SafeCopy. The gather index is + // toTransposed32(i) (the inverse of the encode-side fromTransposed32 + // gather — the two are mutual inverses, not self-inverse). + for (size_t i = 0; i < fastlanes::kBlockSize; ++i) { + const UnsignedT v = + static_cast(transposed[fastlanes::toTransposed32(i)]); + values[i] = util::SafeCopy(v + unsigned_for); + } } } else { std::vector unsigned_values(num_elements); @@ -286,7 +305,8 @@ Result PforCompression::DecodeVector(T* values, std::fill(values, values + num_elements, info.frame_of_reference()); } - // Step 4: Patch exceptions (stored as original values) + // Step 4: Patch exceptions (stored as original values at FLAT positions). + // When emitting transposed output, redirect each patch to toTransposed32(pos). const int16_t num_exceptions = info.num_exceptions(); if (num_exceptions > 0) { const uint8_t* positions_ptr = read_ptr; @@ -300,7 +320,10 @@ Result PforCompression::DecodeVector(T* values, for (int16_t i = 0; i < num_exceptions; ++i) { int16_t pos = util::SafeLoadAs(positions_ptr + i * sizeof(int16_t)); T value = util::SafeLoadAs(values_ptr + i * sizeof(T)); - values[pos] = value; + const size_t out_pos = + emit_transposed ? fastlanes::toTransposed32(static_cast(pos)) + : static_cast(pos); + values[out_pos] = value; } } diff --git a/cpp/src/arrow/util/pfor/pfor.h b/cpp/src/arrow/util/pfor/pfor.h index bbb56781696b..f5bbfe2c8269 100644 --- a/cpp/src/arrow/util/pfor/pfor.h +++ b/cpp/src/arrow/util/pfor/pfor.h @@ -255,9 +255,15 @@ class PforCompression { /// \param[out] values output buffer for decoded integers /// \param[in] data span over the compressed vector data /// \param[in] num_elements number of elements in this vector + /// \param[in] order output value order. Default OutputOrder::Flat returns + /// values in their original input positions. OutputOrder:: + /// Transposed only affects FastLanes-encoded vectors (skips the + /// FL_ORDER gather on decode); BitPack vectors always return + /// flat output regardless of `order`. /// \return number of bytes consumed from data, or error static Result DecodeVector(T* values, arrow::util::span data, - int32_t num_elements); + int32_t num_elements, + OutputOrder order = OutputOrder::Flat); /// \brief Calculate the serialized size of an encoded vector static int64_t SerializedVectorSize(const PforEncodedVector& vec, diff --git a/cpp/src/arrow/util/pfor/pfor_constants.h b/cpp/src/arrow/util/pfor/pfor_constants.h index 5004591aa384..40164e436e12 100644 --- a/cpp/src/arrow/util/pfor/pfor_constants.h +++ b/cpp/src/arrow/util/pfor/pfor_constants.h @@ -73,6 +73,26 @@ enum class PackingMode : uint8_t { FastLanes = 1, }; +/// \brief Output value order requested by the decoder. +/// +/// Affects only FastLanes-encoded vectors (BitPack always returns flat). +/// Selecting `Transposed` skips the per-vector FL_ORDER gather and is the +/// only way to take full advantage of the FastLanes layout: the gather is +/// scalar and dominates end-to-end decode cost. Downstream operators must +/// be permutation-aware (work on values in stream order, i.e. apply +/// fromTransposed32(t) when they need the original index). +enum class OutputOrder : uint8_t { + /// Default. Decoded values appear in their original input order: + /// output[i] == input[i] for every i, for both PackingMode variants. + Flat = 0, + + /// FastLanes transposed (stream) order. For FastLanes-encoded vectors, + /// output[t] == input[fromTransposed32(t)] (per 1024-block, no scatter + /// on decode). For BitPack-encoded vectors there is no permutation, so + /// they still produce flat output. + Transposed = 1, +}; + /// \brief Type traits for PFOR integer types template struct PforTypeTraits {}; diff --git a/cpp/src/arrow/util/pfor/pfor_test.cc b/cpp/src/arrow/util/pfor/pfor_test.cc index 8b4398f8ad11..40f173f70825 100644 --- a/cpp/src/arrow/util/pfor/pfor_test.cc +++ b/cpp/src/arrow/util/pfor/pfor_test.cc @@ -25,6 +25,7 @@ #include #include "arrow/testing/gtest_util.h" +#include "arrow/util/fastlanes/fastlanes_kernels.h" #include "arrow/util/pfor/pfor.h" #include "arrow/util/pfor/pfor_wrapper.h" #include "arrow/util/span.h" @@ -552,4 +553,113 @@ TEST(PforPackingModeTest, ConstantVectorBothModes) { } } +// ============================================================================ +// OutputOrder::Transposed tests — verify the transposed-output decode path +// matches the FL_ORDER permutation of the input for FastLanes vectors. +// ============================================================================ + +TEST(PforOutputOrderTest, TransposedMatchesFLOrder) { + std::vector values(1024); + std::mt19937 rng(11); + std::uniform_int_distribution dist(1000, 1500); + for (auto& v : values) v = dist(rng); + values[7] = 99999; // outlier — exercises the exception path under transposed + values[412] = -50; + + auto encoded = PforCompression::EncodeVector( + values.data(), 1024, PackingMode::FastLanes); + ASSERT_EQ(encoded.info().packing_mode(), PackingMode::FastLanes); + + int64_t sz = PforCompression::SerializedVectorSize(encoded, 1024); + std::vector buffer(sz); + PforCompression::SerializeVector(encoded, 1024, buffer); + + std::vector decoded(1024); + ASSERT_OK(PforCompression::DecodeVector(decoded.data(), buffer, 1024, + OutputOrder::Transposed)); + + // out[t] == input[fromTransposed32(t)] for every stream position t. + for (size_t t = 0; t < 1024; ++t) { + ASSERT_EQ(decoded[t], + values[fastlanes::fromTransposed32(t)]) + << "t=" << t; + } +} + +TEST(PforOutputOrderTest, TransposedThenInvertEqualsInput) { + // Belt-and-suspenders: the user code path is "decode transposed, then + // apply fromTransposed32 myself when I need original-order positions". + // Confirm that inverting the permutation manually produces the input. + std::vector values(1024); + std::mt19937 rng(31); + std::uniform_int_distribution dist(-200, 200); + for (auto& v : values) v = dist(rng); + + auto encoded = PforCompression::EncodeVector( + values.data(), 1024, PackingMode::FastLanes); + + int64_t sz = PforCompression::SerializedVectorSize(encoded, 1024); + std::vector buffer(sz); + PforCompression::SerializeVector(encoded, 1024, buffer); + + std::vector transposed(1024); + ASSERT_OK(PforCompression::DecodeVector(transposed.data(), buffer, 1024, + OutputOrder::Transposed)); + + std::vector flat(1024); + for (size_t t = 0; t < 1024; ++t) { + flat[fastlanes::fromTransposed32(t)] = transposed[t]; + } + EXPECT_EQ(values, flat); +} + +TEST(PforOutputOrderTest, BitPackIgnoresTransposedRequest) { + // BitPack mode has no transposition; OutputOrder::Transposed must be + // ignored (output is still flat / identity). + std::vector values(1024); + std::iota(values.begin(), values.end(), 5000); + + auto encoded = PforCompression::EncodeVector( + values.data(), 1024, PackingMode::BitPack); + ASSERT_EQ(encoded.info().packing_mode(), PackingMode::BitPack); + + int64_t sz = PforCompression::SerializedVectorSize(encoded, 1024); + std::vector buffer(sz); + PforCompression::SerializeVector(encoded, 1024, buffer); + + std::vector decoded(1024); + ASSERT_OK(PforCompression::DecodeVector(decoded.data(), buffer, 1024, + OutputOrder::Transposed)); + EXPECT_EQ(values, decoded); +} + +TEST(PforOutputOrderTest, WrapperTransposedAcrossManyVectors) { + constexpr int32_t kN = 5 * 1024; + std::vector values(kN); + std::mt19937 rng(909); + std::uniform_int_distribution dist(-1000, 1000); + for (auto& v : values) v = dist(rng); + + int64_t max_size = PforWrapper::GetMaxCompressedSize(kN); + std::vector compressed(max_size); + int64_t comp_size = max_size; + + PforWrapper::Encode(values.data(), kN, compressed.data(), &comp_size, + PackingMode::FastLanes); + + std::vector decoded(kN); + ASSERT_OK(PforWrapper::Decode(decoded.data(), kN, compressed.data(), + comp_size, OutputOrder::Transposed)); + + // Verify per-block: decoded[block*1024 + t] == values[block*1024 + fromTransposed32(t)] + for (int32_t block = 0; block < 5; ++block) { + for (size_t t = 0; t < 1024; ++t) { + const int32_t base = block * 1024; + ASSERT_EQ(decoded[base + t], + values[base + fastlanes::fromTransposed32(t)]) + << "block=" << block << " t=" << t; + } + } +} + } // namespace arrow::util::pfor diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.cc b/cpp/src/arrow/util/pfor/pfor_wrapper.cc index b653f0894297..0774dfe74b3c 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.cc +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.cc @@ -157,7 +157,7 @@ void PforWrapper::Encode(const T* values, int32_t num_values, uint8_t* comp, template Status PforWrapper::Decode(T* values, int32_t num_values, const uint8_t* comp, - int64_t comp_size) { + int64_t comp_size, OutputOrder order) { if (num_values <= 0) { return Status::Invalid("PFOR num_values must be positive: ", num_values); } @@ -193,7 +193,7 @@ Status PforWrapper::Decode(T* values, int32_t num_values, const uint8_t* comp ARROW_RETURN_NOT_OK(PforCompression::DecodeVector( values + start_idx, arrow::util::span(vector_data, src + comp_size - vector_data), - elements_in_vector)); + elements_in_vector, order)); } return Status::OK(); diff --git a/cpp/src/arrow/util/pfor/pfor_wrapper.h b/cpp/src/arrow/util/pfor/pfor_wrapper.h index 842a3df2d141..d092a7624d9c 100644 --- a/cpp/src/arrow/util/pfor/pfor_wrapper.h +++ b/cpp/src/arrow/util/pfor/pfor_wrapper.h @@ -68,9 +68,17 @@ class PforWrapper { /// \param[in] num_values number of values to decode (from page context) /// \param[in] comp pointer to compressed data /// \param[in] comp_size size of compressed data + /// \param[in] order output value order. Default OutputOrder::Flat returns + /// values in their original input positions. OutputOrder:: + /// Transposed only affects FastLanes-encoded vectors (skips + /// their FL_ORDER gather on decode); BitPack vectors always + /// produce flat output regardless of `order`, so a mixed page + /// will have BitPack tails in flat order and FastLanes vectors + /// in transposed order. /// \return Status::OK on success, or an error if the data is malformed static Status Decode(T* values, int32_t num_values, const uint8_t* comp, - int64_t comp_size); + int64_t comp_size, + OutputOrder order = OutputOrder::Flat); /// \brief Get the maximum compressed size for a given number of values /// diff --git a/cpp/src/parquet/pfor_comparison_benchmark.cc b/cpp/src/parquet/pfor_comparison_benchmark.cc index 5fbe70db0f05..d5420ab65b65 100644 --- a/cpp/src/parquet/pfor_comparison_benchmark.cc +++ b/cpp/src/parquet/pfor_comparison_benchmark.cc @@ -381,6 +381,40 @@ static void BM_PforFastLanesDecode(benchmark::State& state, Gen32 gen) { static_cast(uncompressed_size) / static_cast(comp_size); } +// PFOR with FastLanes bit-packing, decoded in TRANSPOSED order: skips the +// per-vector FL_ORDER gather, output is in FastLanes stream order. The +// downstream consumer must be permutation-aware. Apples-to-apples this +// against BM_PforFastLanesDecode (flat-order, with the scatter cost) to see +// how much of the gap to pfor+bitpack closes. +static void BM_PforFastLanesDecodeTransposed(benchmark::State& state, Gen32 gen) { + const int64_t num_values = state.range(0); + auto values = gen(num_values); + const int64_t uncompressed_size = num_values * sizeof(int32_t); + + int64_t max_size = + ::arrow::util::pfor::PforWrapper::GetMaxCompressedSize( + static_cast(num_values)); + std::vector compressed(max_size); + int64_t comp_size = max_size; + ::arrow::util::pfor::PforWrapper::Encode( + values.data(), static_cast(num_values), compressed.data(), + &comp_size, ::arrow::util::pfor::PackingMode::FastLanes); + + std::vector decoded(num_values); + for (auto _ : state) { + auto status = ::arrow::util::pfor::PforWrapper::Decode( + decoded.data(), static_cast(num_values), compressed.data(), + comp_size, ::arrow::util::pfor::OutputOrder::Transposed); + ARROW_CHECK_OK(status); + benchmark::ClobberMemory(); + } + + state.SetBytesProcessed(state.iterations() * uncompressed_size); + state.SetItemsProcessed(state.iterations() * num_values); + state.counters["compression_ratio"] = + static_cast(uncompressed_size) / static_cast(comp_size); +} + // ============================================================================ // DeltaBitPack Encode/Decode // ============================================================================ @@ -849,6 +883,7 @@ static void CustomArgs(benchmark::internal::Benchmark* b) { b->Arg(102400); } BENCHMARK_CAPTURE(BM_PforDecode, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_PforFastLanesEncode, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_PforFastLanesDecode, Name, &GenFunc)->Apply(CustomArgs); \ + BENCHMARK_CAPTURE(BM_PforFastLanesDecodeTransposed, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_DeltaBitPackEncode, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_DeltaBitPackDecode, Name, &GenFunc)->Apply(CustomArgs); \ BENCHMARK_CAPTURE(BM_FastLanesEncode, Name, &GenFunc)->Apply(CustomArgs); \ From d9ef31b5032503e87b31f3f93e7401d70b7f8920 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Sun, 28 Jun 2026 20:02:03 +0000 Subject: [PATCH 31/31] Local: keep -O3 from CMake Release defaults (drop -O2 downgrade) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Arrow's SetupCxxFlags.cmake explicitly appends '-O2' after CMake's default '-O3 -DNDEBUG' Release flags, downgrading optimization (the last -O flag wins in GCC). The comment 'Enable compiler optimizations' justifies this as a safety choice for the Arrow library at large, but for benchmarking the FastLanes path it's a measurable handicap: AutoVec relies on -O3's more aggressive inlining and loop unrolling to fully vectorize the row loop. Local override skips the append so Release builds get true -O3. Across the 18-dataset comparison benchmark, FL transposed decode picks up 10-17% (e.g. EventDate 20.3 us -> 17.8 us, TrafficSourceID 20.2 us -> 16.7 us). pfor+bitpack and the other codecs are unchanged within noise. Not intended for upstream — this is local-only for the bench. Revert with 'git checkout cpp/cmake_modules/SetupCxxFlags.cmake' to restore Arrow's default. --- cpp/cmake_modules/SetupCxxFlags.cmake | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index 21341167fe99..8887a1ed9294 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -633,21 +633,12 @@ endif() if(NOT MSVC) set(C_RELEASE_FLAGS "") - if(CMAKE_C_FLAGS_RELEASE MATCHES "-O3") - string(APPEND C_RELEASE_FLAGS " -O2") - endif() + # Local override: keep -O3 from CMake's default Release flags for the + # pfor benchmark — the bench is sensitive to inlining/unrolling that + # -O3 enables. Upstream Arrow downgrades to -O2 here; we skip that. set(CXX_RELEASE_FLAGS "") - if(CMAKE_CXX_FLAGS_RELEASE MATCHES "-O3") - string(APPEND CXX_RELEASE_FLAGS " -O2") - endif() set(C_RELWITHDEBINFO_FLAGS "") - if(CMAKE_C_FLAGS_RELWITHDEBINFO MATCHES "-O3") - string(APPEND C_RELWITHDEBINFO_FLAGS " -O2") - endif() set(CXX_RELWITHDEBINFO_FLAGS "") - if(CMAKE_CXX_FLAGS_RELWITHDEBINFO MATCHES "-O3") - string(APPEND CXX_RELWITHDEBINFO_FLAGS " -O2") - endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") string(APPEND C_RELEASE_FLAGS " -ftree-vectorize") string(APPEND CXX_RELEASE_FLAGS " -ftree-vectorize")